Merge pull request #2892 from umami-software/dev

v2.13.0

Dockerfile

@@ -37,7 +37,7 @@ RUN adduser --system --uid 1001 nextjs
 
 RUN set -x \
     && apk add --no-cache curl \
-    && yarn add npm-run-all dotenv prisma semver
+    && yarn add npm-run-all dotenv semver prisma@5.17.0
 
 # You only need to copy next.config.js if you are NOT using the default configuration
 COPY --from=builder /app/next.config.js .

db/clickhouse/schema.sql

@@ -29,9 +29,11 @@ CREATE TABLE umami.website_event
     created_at DateTime('UTC'),
     job_id Nullable(UUID)
 )
-    engine = MergeTree
-        ORDER BY (website_id, session_id, created_at)
-        SETTINGS index_granularity = 8192;
+ENGINE = MergeTree
+    PARTITION BY toYYYYMM(created_at)
+    ORDER BY (toStartOfHour(created_at), website_id, session_id, visit_id, created_at)
+    PRIMARY KEY (toStartOfHour(created_at), website_id, session_id, visit_id)
+    SETTINGS index_granularity = 8192;
 
 CREATE TABLE umami.event_data
 (
@@ -48,9 +50,9 @@ CREATE TABLE umami.event_data
     created_at DateTime('UTC'),
     job_id Nullable(UUID)
 )
-    engine = MergeTree
-        ORDER BY (website_id, event_id, data_key, created_at)
-        SETTINGS index_granularity = 8192;
+ENGINE = MergeTree
+    ORDER BY (website_id, event_id, data_key, created_at)
+    SETTINGS index_granularity = 8192;
 
 CREATE TABLE umami.session_data
 (
@@ -64,6 +66,129 @@ CREATE TABLE umami.session_data
     created_at DateTime('UTC'),
     job_id Nullable(UUID)
 )
-    engine = MergeTree
-        ORDER BY (website_id, session_id, data_key, created_at)
-        SETTINGS index_granularity = 8192;
+ENGINE = ReplacingMergeTree
+    ORDER BY (website_id, session_id, data_key)
+    SETTINGS index_granularity = 8192;
+
+-- stats hourly
+CREATE TABLE umami.website_event_stats_hourly
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    city String,
+    entry_url AggregateFunction(argMin, String, DateTime('UTC')),
+    exit_url AggregateFunction(argMax, String, DateTime('UTC')),
+    url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
+    url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
+    referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
+    page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
+    event_type UInt32,
+    event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
+    views SimpleAggregateFunction(sum, UInt64),
+    min_time SimpleAggregateFunction(min, DateTime('UTC')),
+    max_time SimpleAggregateFunction(max, DateTime('UTC')),
+    created_at Datetime('UTC')
+)
+ENGINE = AggregatingMergeTree
+    PARTITION BY toYYYYMM(created_at)
+    ORDER BY (
+        website_id,
+        event_type,
+        toStartOfHour(created_at),
+        cityHash64(visit_id),
+        visit_id
+    )
+    SAMPLE BY cityHash64(visit_id);
+
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
+TO umami.website_event_stats_hourly
+AS
+SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    city,
+    entry_url,
+    exit_url,
+    url_paths as url_path,
+    url_query,
+    referrer_domain,
+    page_title,
+    event_type,
+    event_name,
+    views,
+    min_time,
+    max_time,
+    timestamp as created_at
+FROM (SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    city,
+    argMinState(url_path, created_at) entry_url,
+    argMaxState(url_path, created_at) exit_url,
+    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+    arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+    event_type,
+    if(event_type = 2, groupArray(event_name), []) event_name,
+    sumIf(1, event_type = 1) views,
+    min(created_at) min_time,
+    max(created_at) max_time,
+    toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    city,
+    event_type,
+    timestamp);
+
+-- projections
+ALTER TABLE umami.website_event 
+ADD PROJECTION website_event_url_path_projection (
+SELECT * ORDER BY toStartOfDay(created_at), website_id, url_path, created_at
+);
+
+ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_url_path_projection;
+
+ALTER TABLE umami.website_event 
+ADD PROJECTION website_event_referrer_domain_projection (
+SELECT * ORDER BY toStartOfDay(created_at), website_id, referrer_domain, created_at
+);
+
+ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_referrer_domain_projection;

next.config.js

@@ -17,7 +17,7 @@ const trackerScriptName = process.env.TRACKER_SCRIPT_NAME;
 
 const contentSecurityPolicy = [
   `default-src 'self'`,
-  `img-src *`,
+  `img-src * data:`,
   `script-src 'self' 'unsafe-eval' 'unsafe-inline'`,
   `style-src 'self' 'unsafe-inline'`,
   `connect-src 'self' api.umami.is cloud.umami.is`,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "umami",
-  "version": "2.12.1",
+  "version": "2.13.0",
   "description": "A simple, fast, privacy-focused alternative to Google Analytics.",
   "author": "Umami Software, Inc. <hello@umami.is>",
   "license": "MIT",
@@ -64,9 +64,12 @@
     ".next/cache"
   ],
   "dependencies": {
-    "@clickhouse/client": "^1.0.2",
+    "@clickhouse/client": "^1.4.1",
+    "@date-fns/utc": "^1.2.0",
+    "@dicebear/collection": "^9.2.1",
+    "@dicebear/core": "^9.2.1",
     "@fontsource/inter": "^4.5.15",
-    "@prisma/client": "5.14.0",
+    "@prisma/client": "5.17",
     "@prisma/extension-read-replicas": "^0.3.0",
     "@react-spring/web": "^9.7.3",
     "@tanstack/react-query": "^5.28.6",
@@ -81,7 +84,6 @@
     "cross-spawn": "^7.0.3",
     "date-fns": "^2.23.0",
     "date-fns-tz": "^1.1.4",
-    "dateformat": "^5.0.3",
     "debug": "^4.3.4",
     "del": "^6.0.0",
     "detect-browser": "^5.2.0",
@@ -93,18 +95,18 @@
     "is-ci": "^3.0.1",
     "is-docker": "^3.0.0",
     "is-localhost-ip": "^1.4.0",
-    "isbot": "^5.1.1",
+    "isbot": "^5.1.16",
     "kafkajs": "^2.1.0",
     "maxmind": "^4.3.6",
     "md5": "^2.3.0",
     "moment-timezone": "^0.5.35",
-    "next": "14.2.3",
+    "next": "14.2.5",
     "next-basics": "^0.39.0",
     "node-fetch": "^3.2.8",
     "npm-run-all": "^4.1.5",
-    "prisma": "5.14.0",
+    "prisma": "5.17",
     "react": "^18.2.0",
-    "react-basics": "^0.123.0",
+    "react-basics": "^0.125.0",
     "react-beautiful-dnd": "^13.1.0",
     "react-dom": "^18.2.0",
     "react-error-boundary": "^4.0.4",
@@ -117,7 +119,7 @@
     "thenby": "^1.3.4",
     "uuid": "^9.0.0",
     "yup": "^0.32.11",
-    "zustand": "^4.3.8"
+    "zustand": "^4.5.5"
   },
   "devDependencies": {
     "@formatjs/cli": "^4.2.29",
@@ -175,6 +177,6 @@
     "tar": "^6.1.2",
     "ts-jest": "^29.1.2",
     "ts-node": "^10.9.1",
-    "typescript": "^5.4.3"
+    "typescript": "^5.5.3"
   }
 }