From cb4368e12ca16c2c1c658d715c60383089424ec5 Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Wed, 31 Jul 2024 09:35:29 -0700
Subject: [PATCH] template multiple queries for filtering

---
 db/clickhouse/schema.sql                      | 16 +++++-------
 src/queries/analytics/getWebsiteStats.ts      | 25 +++++++++++++++++--
 .../analytics/pageviews/getPageviewMetrics.ts | 18 +++++++++++++
 .../analytics/pageviews/getPageviewStats.ts   | 21 ++++++++++++++--
 4 files changed, 66 insertions(+), 14 deletions(-)

diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index 6179bfb0..aadc5402 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -30,7 +30,9 @@ CREATE TABLE umami.website_event
     job_id Nullable(UUID)
 )
     engine = MergeTree
-        ORDER BY (website_id, session_id, created_at)
+        PARTITION BY toYYYYMM(created_at)
+        ORDER BY (toStartOfHour(created_at), website_id, session_id, visit_id, created_at)
+        PRIMARY KEY (toStartOfHour(created_at), website_id, session_id, visit_id)
         SETTINGS index_granularity = 8192;
 
 CREATE TABLE umami.event_data
@@ -97,15 +99,9 @@ CREATE TABLE umami.website_event_stats_hourly
     created_at Datetime('UTC')
 )
 ENGINE = AggregatingMergeTree
-PARTITION BY toYYYYMM(created_at)
-ORDER BY (
-    website_id,
-    event_type,
-    toStartOfHour(created_at),
-    cityHash64(visit_id),
-    visit_id
-)
-SAMPLE BY cityHash64(visit_id);
+        PARTITION BY toYYYYMM(created_at)
+        ORDER BY (toStartOfDay(created_at), website_id, session_id, visit_id, created_at)
+        PRIMARY KEY (toStartOfDay(created_at), website_id, session_id, visit_id)
 
 CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
 TO umami.website_event_stats_hourly
diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts
index ebe711a0..7d8a76f6 100644
--- a/src/queries/analytics/getWebsiteStats.ts
+++ b/src/queries/analytics/getWebsiteStats.ts
@@ -1,4 +1,3 @@
-/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
 import clickhouse from 'lib/clickhouse';
 import { EVENT_TYPE } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
@@ -69,8 +68,30 @@ async function clickhouseQuery(
   });
 
   return rawQuery(
+    // `
+    // select
+    //   sum(t.c) as "pageviews",
+    //   count(distinct t.session_id) as "visitors",
+    //   count(distinct t.visit_id) as "visits",
+    //   sum(if(t.c = 1, 1, 0)) as "bounces",
+    //   sum(max_time-min_time) as "totaltime"
+    // from (
+    //   select
+    //     session_id,
+    //     visit_id,
+    //     count(*) c,
+    //     min(created_at) min_time,
+    //     max(created_at) max_time
+    //   from website_event
+    //   where website_id = {websiteId:UUID}
+    //     and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    //     and event_type = {eventType:UInt32}
+    //     ${filterQuery}
+    //   group by session_id, visit_id
+    // ) as t;
+    // `,
     `
-    select 
+    select
       sum(views) as "pageviews",
       uniq(session_id) as "visitors",
       uniq(visit_id) as "visits",
diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts
index ccfe4ef0..4ac101b2 100644
--- a/src/queries/analytics/pageviews/getPageviewMetrics.ts
+++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts
@@ -111,7 +111,25 @@ async function clickhouseQuery(
     groupByQuery = 'group by x';
   }
 
+  // let excludeDomain = '';
+  // if (column === 'referrer_domain') {
+  //   excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
+  // }
+
   return rawQuery(
+    // `
+    //   select ${column} x, count(*) y
+    //   from website_event
+    //   where website_id = {websiteId:UUID}
+    //     and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    //     and event_type = {eventType:UInt32}
+    //     ${excludeDomain}
+    //     ${filterQuery}
+    //   group by x
+    //   order by y desc
+    //   limit ${limit}
+    //   offset ${offset}
+    //   `,
     `
     select g.t as x,
       count(*) as y
diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts
index f6ea1e08..c378d48a 100644
--- a/src/queries/analytics/pageviews/getPageviewStats.ts
+++ b/src/queries/analytics/pageviews/getPageviewStats.ts
@@ -51,12 +51,29 @@ async function clickhouseQuery(
   const columnQuery = unit === 'minute' ? 'count(*)' : 'sum(views)';
 
   return rawQuery(
+    // `
+    // select
+    //   ${getDateStringSQL('g.t', unit)} as x,
+    //   g.y as y
+    // from (
+    //   select
+    //     ${getDateSQL('created_at', unit, timezone)} as t,
+    //     count(*) as y
+    //   from website_event
+    //   where website_id = {websiteId:UUID}
+    //     and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    //     and event_type = {eventType:UInt32}
+    //     ${filterQuery}
+    //   group by t
+    // ) as g
+    // order by t
+    // `,
     `
     select
-      ${getDateStringSQL('g.t', unit)} as x, 
+      ${getDateStringSQL('g.t', unit)} as x,
       g.y as y
     from (
-      select 
+      select
         ${getDateSQL('created_at', unit, timezone)} as t,
         ${columnQuery} as y
       from ${table} website_event