From 14e4a090bb79ef61c33ea57ea8d4be21ac34bc51 Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Mon, 27 Mar 2023 11:25:16 -0700
Subject: [PATCH 1/6] update schema and queries to implement reset_at

---
 db/clickhouse/schema.sql                      |  6 ------
 db/mysql/schema.prisma                        |  2 +-
 db/postgresql/schema.prisma                   |  2 +-
 lib/types.ts                                  |  2 +-
 queries/admin/website.ts                      |  9 +++++---
 queries/analytics/event/getEventMetrics.ts    | 14 ++++++++-----
 queries/analytics/event/saveEvent.ts          |  6 ------
 queries/analytics/eventData/getEventData.ts   | 17 +++++++++------
 queries/analytics/eventData/saveEventData.ts  |  5 +----
 .../analytics/pageview/getPageviewMetrics.ts  | 14 ++++++++-----
 .../analytics/pageview/getPageviewStats.ts    | 21 ++++++++++++++-----
 queries/analytics/pageview/savePageView.ts    |  3 ---
 queries/analytics/session/createSession.ts    |  2 --
 .../analytics/session/getSessionMetrics.ts    | 14 ++++++++-----
 queries/analytics/stats/getWebsiteStats.ts    | 16 ++++++++------
 15 files changed, 74 insertions(+), 59 deletions(-)

diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index 3fd99aa3..904cd377 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -6,7 +6,6 @@ CREATE TABLE umami.event
     website_id UUID,
     session_id UUID,
     event_id UUID,
-    rev_id UInt32,
     --session
     hostname LowCardinality(String),
     browser LowCardinality(String),
@@ -38,7 +37,6 @@ CREATE TABLE umami.event_queue (
     website_id UUID,
     session_id UUID,
     event_id UUID,
-    rev_id UInt32,
     --session
     hostname LowCardinality(String),
     browser LowCardinality(String),
@@ -74,7 +72,6 @@ CREATE MATERIALIZED VIEW umami.event_queue_mv TO umami.event AS
 SELECT website_id,
     session_id,
     event_id,
-    rev_id,
     hostname,
     browser,
     os,
@@ -101,7 +98,6 @@ CREATE TABLE umami.event_data
     website_id UUID,
     session_id UUID,
     event_id UUID,
-    rev_id UInt32,
     url_path String,
     event_name String,
     event_key String,
@@ -119,7 +115,6 @@ CREATE TABLE umami.event_data_queue (
     website_id UUID,
     session_id UUID,
     event_id UUID,
-    rev_id UInt32,
     url_path String,
     event_name String,
     event_key String,
@@ -141,7 +136,6 @@ CREATE MATERIALIZED VIEW umami.event_data_queue_mv TO umami.event_data AS
 SELECT website_id,
     session_id,
     event_id,
-    rev_id,
     url_path,
     event_name,
     event_key,
diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma
index 13dcdcbc..af939c0c 100644
--- a/db/mysql/schema.prisma
+++ b/db/mysql/schema.prisma
@@ -50,7 +50,7 @@ model Website {
   name      String    @db.VarChar(100)
   domain    String?   @db.VarChar(500)
   shareId   String?   @unique @map("share_id") @db.VarChar(50)
-  revId     Int       @default(0) @map("rev_id") @db.UnsignedInt
+  resetAt   DateTime? @map("reset_at") @db.Timestamp(0)
   userId    String?   @map("user_id") @db.VarChar(36)
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
   updatedAt DateTime? @map("updated_at") @db.Timestamp(0)
diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma
index bb26a807..57f7a78f 100644
--- a/db/postgresql/schema.prisma
+++ b/db/postgresql/schema.prisma
@@ -50,7 +50,7 @@ model Website {
   name      String    @db.VarChar(100)
   domain    String?   @db.VarChar(500)
   shareId   String?   @unique @map("share_id") @db.VarChar(50)
-  revId     Int       @default(0) @map("rev_id") @db.Integer
+  resetAt   DateTime? @map("reset_at") @db.Timestamptz(6)
   userId    String?   @map("user_id") @db.Uuid
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
   updatedAt DateTime? @map("updated_at") @db.Timestamptz(6)
diff --git a/lib/types.ts b/lib/types.ts
index c818d6ff..88a3907c 100644
--- a/lib/types.ts
+++ b/lib/types.ts
@@ -48,7 +48,7 @@ export interface User {
 export interface Website {
   id: string;
   userId: string;
-  revId: number;
+  resetAt: Date;
   name: string;
   domain: string;
   shareId: string;
diff --git a/queries/admin/website.ts b/queries/admin/website.ts
index 7bcb66b2..49c97691 100644
--- a/queries/admin/website.ts
+++ b/queries/admin/website.ts
@@ -49,8 +49,6 @@ export async function resetWebsite(
 ): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {
   const { client, transaction } = prisma;
 
-  const { revId } = await getWebsite({ id: websiteId });
-
   return transaction([
     client.websiteEvent.deleteMany({
       where: { websiteId },
@@ -58,7 +56,12 @@ export async function resetWebsite(
     client.session.deleteMany({
       where: { websiteId },
     }),
-    client.website.update({ where: { id: websiteId }, data: { revId: revId + 1 } }),
+    client.website.update({
+      where: { id: websiteId },
+      data: {
+        resetAt: new Date(),
+      },
+    }),
   ]).then(async data => {
     if (cache.enabled) {
       await cache.storeWebsite(data[2]);
diff --git a/queries/analytics/event/getEventMetrics.ts b/queries/analytics/event/getEventMetrics.ts
index 31cfe327..35eda8f6 100644
--- a/queries/analytics/event/getEventMetrics.ts
+++ b/queries/analytics/event/getEventMetrics.ts
@@ -46,7 +46,9 @@ async function relationalQuery(
   },
 ) {
   const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma;
-  const params: any = [websiteId, startDate, endDate];
+  const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params: any = [websiteId, resetDate, startDate, endDate];
 
   return rawQuery(
     `select
@@ -55,7 +57,8 @@ async function relationalQuery(
       count(*) y
     from website_event
     where website_id = $1${toUuid()}
-      and created_at between $2 and $3
+      and created_at >= $2
+      and created_at between $3 and $4
       and event_type = ${EVENT_TYPE.customEvent}
       ${getFilterQuery(filters, params)}
     group by 1, 2
@@ -83,9 +86,10 @@ async function clickhouseQuery(
     };
   },
 ) {
-  const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse;
+  const { rawQuery, getDateQuery, getDateFormat, getBetweenDates, getFilterQuery } = clickhouse;
   const website = await cache.fetchWebsite(websiteId);
-  const params = { websiteId, revId: website?.revId || 0 };
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params = { websiteId };
 
   return rawQuery(
     `select
@@ -94,8 +98,8 @@ async function clickhouseQuery(
       count(*) y
     from event
     where website_id = {websiteId:UUID}
-    and rev_id = {revId:UInt32}
       and event_type = ${EVENT_TYPE.customEvent}
+      and created_at >= ${getDateFormat(resetDate)}
       and ${getBetweenDates('created_at', startDate, endDate)}
       ${getFilterQuery(filters, params)}
     group by x, t
diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts
index d1f124ea..cbb48826 100644
--- a/queries/analytics/event/saveEvent.ts
+++ b/queries/analytics/event/saveEvent.ts
@@ -3,7 +3,6 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import kafka from 'lib/kafka';
 import prisma from 'lib/prisma';
 import { uuid } from 'lib/crypto';
-import cache from 'lib/cache';
 import { saveEventData } from '../eventData/saveEventData';
 
 export async function saveEvent(args: {
@@ -41,7 +40,6 @@ async function relationalQuery(data: {
   eventData?: any;
 }) {
   const { websiteId, id: sessionId, urlPath, urlQuery, eventName, eventData, pageTitle } = data;
-  const website = await cache.fetchWebsite(websiteId);
   const websiteEventId = uuid();
 
   const websiteEvent = prisma.client.websiteEvent.create({
@@ -62,7 +60,6 @@ async function relationalQuery(data: {
       websiteId,
       sessionId,
       eventId: websiteEventId,
-      revId: website?.revId,
       urlPath: urlPath?.substring(0, URL_LENGTH),
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
       eventData,
@@ -106,7 +103,6 @@ async function clickhouseQuery(data: {
     ...args
   } = data;
   const { getDateFormat, sendMessage } = kafka;
-  const website = await cache.fetchWebsite(websiteId);
   const eventId = uuid();
   const createdAt = getDateFormat(new Date());
 
@@ -123,7 +119,6 @@ async function clickhouseQuery(data: {
     page_title: pageTitle,
     event_type: EVENT_TYPE.customEvent,
     event_name: eventName?.substring(0, EVENT_NAME_LENGTH),
-    rev_id: website?.revId || 0,
     created_at: createdAt,
     ...args,
   };
@@ -135,7 +130,6 @@ async function clickhouseQuery(data: {
       websiteId,
       sessionId,
       eventId,
-      revId: website?.revId,
       urlPath: urlPath?.substring(0, URL_LENGTH),
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
       eventData,
diff --git a/queries/analytics/eventData/getEventData.ts b/queries/analytics/eventData/getEventData.ts
index 1c34a733..f8e2febb 100644
--- a/queries/analytics/eventData/getEventData.ts
+++ b/queries/analytics/eventData/getEventData.ts
@@ -48,7 +48,9 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
   const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
-  const params: any = [websiteId, startDate, endDate, eventName || ''];
+  const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params: any = [websiteId, resetDate, startDate, endDate, eventName || ''];
 
   return rawQuery(
     `select
@@ -65,8 +67,9 @@ async function relationalQuery(
           : ''
       }
     where website_id = $1${toUuid()}
-      and created_at between $2 and $3
-      ${eventName ? `and eventName = $4` : ''}
+      and created_at >= $2
+      and created_at between $3 and $4
+      ${eventName ? `and eventName = $5` : ''}
       ${getEventDataFilterQuery(filters, params)}
       ${timeSeries ? 'group by t' : ''}`,
     params,
@@ -93,9 +96,11 @@ async function clickhouseQuery(
   },
 ) {
   const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
-  const { rawQuery, getBetweenDates, getDateQuery, getEventDataFilterQuery } = clickhouse;
+  const { rawQuery, getDateFormat, getBetweenDates, getDateQuery, getEventDataFilterQuery } =
+    clickhouse;
   const website = await cache.fetchWebsite(websiteId);
-  const params = { websiteId, revId: website?.revId || 0 };
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params = { websiteId };
 
   return rawQuery(
     `select
@@ -107,8 +112,8 @@ async function clickhouseQuery(
         }
     from event_data
     where website_id = {websiteId:UUID}
-      and rev_id = {revId:UInt32}
       ${eventName ? `and eventName = ${eventName}` : ''}
+      and created_at >= ${getDateFormat(resetDate)}
       and ${getBetweenDates('created_at', startDate, endDate)}
       ${getEventDataFilterQuery(filters, params)}
       ${timeSeries ? 'group by t' : ''}`,
diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts
index c7ccb1b1..90e63565 100644
--- a/queries/analytics/eventData/saveEventData.ts
+++ b/queries/analytics/eventData/saveEventData.ts
@@ -11,7 +11,6 @@ export async function saveEventData(args: {
   websiteId: string;
   eventId: string;
   sessionId?: string;
-  revId?: number;
   urlPath?: string;
   eventName?: string;
   eventData: EventData;
@@ -58,13 +57,12 @@ async function clickhouseQuery(data: {
   websiteId: string;
   eventId: string;
   sessionId?: string;
-  revId?: number;
   urlPath?: string;
   eventName?: string;
   eventData: EventData;
   createdAt?: string;
 }) {
-  const { websiteId, sessionId, eventId, revId, urlPath, eventName, eventData, createdAt } = data;
+  const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
 
   const { getDateFormat, sendMessages } = kafka;
 
@@ -74,7 +72,6 @@ async function clickhouseQuery(data: {
     website_id: websiteId,
     session_id: sessionId,
     event_id: eventId,
-    rev_id: revId,
     url_path: urlPath,
     event_name: eventName,
     event_key: a.key,
diff --git a/queries/analytics/pageview/getPageviewMetrics.ts b/queries/analytics/pageview/getPageviewMetrics.ts
index 0475207c..09524047 100644
--- a/queries/analytics/pageview/getPageviewMetrics.ts
+++ b/queries/analytics/pageview/getPageviewMetrics.ts
@@ -35,8 +35,11 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, column, filters = {}, type } = data;
   const { rawQuery, parseFilters, toUuid } = prisma;
+  const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
   const params: any = [
     websiteId,
+    resetDate,
     startDate,
     endDate,
     type === 'event' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
@@ -48,8 +51,9 @@ async function relationalQuery(
     from website_event
       ${joinSession}
     where website_event.website_id = $1${toUuid()}
-      and website_event.created_at between $2 and $3
-      and event_type = $4
+      and website_event.created_at >= $2
+      and website_event.created_at between $3 and $4
+      and event_type = $5
       ${filterQuery}
     group by 1
     order by 2 desc
@@ -69,11 +73,11 @@ async function clickhouseQuery(
   },
 ) {
   const { startDate, endDate, column, filters = {}, type } = data;
-  const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
+  const { rawQuery, getDateFormat, parseFilters, getBetweenDates } = clickhouse;
   const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
   const params = {
     websiteId,
-    revId: website?.revId || 0,
     eventType: type === 'event' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
   };
   const { filterQuery } = parseFilters(filters, params);
@@ -82,8 +86,8 @@ async function clickhouseQuery(
     `select ${column} x, count(*) y
     from event
     where website_id = {websiteId:UUID}
-      and rev_id = {revId:UInt32}
       and event_type = {eventType:UInt32}
+      and created_at >= ${getDateFormat(resetDate)}
       and ${getBetweenDates('created_at', startDate, endDate)}
       ${filterQuery}
     group by x
diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts
index 273151aa..c6197312 100644
--- a/queries/analytics/pageview/getPageviewStats.ts
+++ b/queries/analytics/pageview/getPageviewStats.ts
@@ -46,7 +46,9 @@ async function relationalQuery(
     sessionKey = 'session_id',
   } = data;
   const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma;
-  const params: any = [websiteId, startDate, endDate];
+  const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params: any = [websiteId, resetDate, startDate, endDate];
   const { filterQuery, joinSession } = parseFilters(filters, params);
 
   return rawQuery(
@@ -55,7 +57,8 @@ async function relationalQuery(
       from website_event
         ${joinSession}
       where website_event.website_id = $1${toUuid()}
-        and website_event.created_at between $2 and $3
+        and website_event.created_at >= $2
+        and website_event.created_at between $3 and $4
         and event_type = ${EVENT_TYPE.pageView}
         ${filterQuery}
       group by 1`,
@@ -76,9 +79,17 @@ async function clickhouseQuery(
   },
 ) {
   const { startDate, endDate, timezone = 'UTC', unit = 'day', count = '*', filters = {} } = data;
-  const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
+  const {
+    parseFilters,
+    getDateFormat,
+    rawQuery,
+    getDateStringQuery,
+    getDateQuery,
+    getBetweenDates,
+  } = clickhouse;
   const website = await cache.fetchWebsite(websiteId);
-  const params = { websiteId, revId: website?.revId || 0 };
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params = { websiteId };
   const { filterQuery } = parseFilters(filters, params);
 
   return rawQuery(
@@ -91,8 +102,8 @@ async function clickhouseQuery(
         count(${count !== '*' ? 'distinct session_id' : count}) y
       from event
       where website_id = {websiteId:UUID}
-        and rev_id = {revId:UInt32}
         and event_type = ${EVENT_TYPE.pageView}
+        and created_at >= ${getDateFormat(resetDate)}
         and ${getBetweenDates('created_at', startDate, endDate)}
         ${filterQuery}
       group by t) g
diff --git a/queries/analytics/pageview/savePageView.ts b/queries/analytics/pageview/savePageView.ts
index 1b0239ca..41742774 100644
--- a/queries/analytics/pageview/savePageView.ts
+++ b/queries/analytics/pageview/savePageView.ts
@@ -2,7 +2,6 @@ import { URL_LENGTH, EVENT_TYPE } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import kafka from 'lib/kafka';
 import prisma from 'lib/prisma';
-import cache from 'lib/cache';
 import { uuid } from 'lib/crypto';
 
 export async function savePageView(args: {
@@ -104,13 +103,11 @@ async function clickhouseQuery(data: {
     ...args
   } = data;
   const { getDateFormat, sendMessage } = kafka;
-  const website = await cache.fetchWebsite(websiteId);
 
   const message = {
     website_id: websiteId,
     session_id: sessionId,
     event_id: uuid(),
-    rev_id: website?.revId || 0,
     country: country ? country : null,
     subdivision1: subdivision1 ? subdivision1 : null,
     subdivision2: subdivision2 ? subdivision2 : null,
diff --git a/queries/analytics/session/createSession.ts b/queries/analytics/session/createSession.ts
index 3afa9c9a..d2657d60 100644
--- a/queries/analytics/session/createSession.ts
+++ b/queries/analytics/session/createSession.ts
@@ -50,7 +50,6 @@ async function clickhouseQuery(data: {
     city,
   } = data;
   const { getDateFormat, sendMessage } = kafka;
-  const website = await cache.fetchWebsite(websiteId);
 
   const msg = {
     session_id: id,
@@ -65,7 +64,6 @@ async function clickhouseQuery(data: {
     subdivision1,
     subdivision2,
     city,
-    rev_id: website?.revId || 0,
     created_at: getDateFormat(new Date()),
   };
 
diff --git a/queries/analytics/session/getSessionMetrics.ts b/queries/analytics/session/getSessionMetrics.ts
index 9f427e53..073f93f6 100644
--- a/queries/analytics/session/getSessionMetrics.ts
+++ b/queries/analytics/session/getSessionMetrics.ts
@@ -20,9 +20,11 @@ async function relationalQuery(
   websiteId: string,
   data: { startDate: Date; endDate: Date; field: string; filters: object },
 ) {
+  const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
   const { startDate, endDate, field, filters = {} } = data;
   const { toUuid, parseFilters, rawQuery } = prisma;
-  const params: any = [websiteId, startDate, endDate];
+  const params: any = [websiteId, resetDate, startDate, endDate];
   const { filterQuery, joinSession } = parseFilters(filters, params);
 
   return rawQuery(
@@ -35,7 +37,8 @@ async function relationalQuery(
           on website_event.website_id = website.website_id
         ${joinSession}
       where website.website_id = $1${toUuid()}
-      and website_event.created_at between $2 and $3
+        and website_event.created_at >= $2
+        and website_event.created_at between $3 and $4
       ${filterQuery}
     )
     group by 1
@@ -50,17 +53,18 @@ async function clickhouseQuery(
   data: { startDate: Date; endDate: Date; field: string; filters: object },
 ) {
   const { startDate, endDate, field, filters = {} } = data;
-  const { parseFilters, getBetweenDates, rawQuery } = clickhouse;
+  const { getDateFormat, parseFilters, getBetweenDates, rawQuery } = clickhouse;
   const website = await cache.fetchWebsite(websiteId);
-  const params = { websiteId, revId: website?.revId || 0 };
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params = { websiteId };
   const { filterQuery } = parseFilters(filters, params);
 
   return rawQuery(
     `select ${field} x, count(distinct session_id) y
     from event as x
     where website_id = {websiteId:UUID}
-    and rev_id = {revId:UInt32}
     and event_type = ${EVENT_TYPE.pageView}
+      and created_at >= ${getDateFormat(resetDate)}
       and ${getBetweenDates('created_at', startDate, endDate)}
       ${filterQuery}
     group by x
diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts
index 73d28fb6..d11de565 100644
--- a/queries/analytics/stats/getWebsiteStats.ts
+++ b/queries/analytics/stats/getWebsiteStats.ts
@@ -19,7 +19,9 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, filters = {} } = data;
   const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;
-  const params: any = [websiteId, startDate, endDate];
+  const website = await cache.fetchWebsite(websiteId);
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params: any = [websiteId, resetDate, startDate, endDate];
   const { filterQuery, joinSession } = parseFilters(filters, params);
 
   return rawQuery(
@@ -37,7 +39,8 @@ async function relationalQuery(
             on website_event.website_id = website.website_id
           ${joinSession}
         where website.website_id = $1${toUuid()}
-          and website_event.created_at between $2 and $3
+          and website_event.created_at >= $2
+          and website_event.created_at between $3 and $4
           ${filterQuery}
         group by 1, 2
      ) t`,
@@ -50,9 +53,10 @@ async function clickhouseQuery(
   data: { startDate: Date; endDate: Date; filters: object },
 ) {
   const { startDate, endDate, filters = {} } = data;
-  const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
+  const { rawQuery, getDateFormat, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
   const website = await cache.fetchWebsite(websiteId);
-  const params = { websiteId, revId: website?.revId || 0 };
+  const resetDate = website?.resetAt || website?.createdAt;
+  const params = { websiteId };
   const { filterQuery } = parseFilters(filters, params);
 
   return rawQuery(
@@ -70,8 +74,8 @@ async function clickhouseQuery(
        from event
        where event_type = ${EVENT_TYPE.pageView}
         and website_id = {websiteId:UUID}
-        and rev_id = {revId:UInt32}
-         and ${getBetweenDates('created_at', startDate, endDate)}
+          and created_at >= ${getDateFormat(resetDate)}
+          and ${getBetweenDates('created_at', startDate, endDate)}
          ${filterQuery}
        group by session_id, time_series
      ) t;`,

From cb79d806e892dc1c9820fd6af834b23f212cd8ba Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Mon, 27 Mar 2023 11:36:21 -0700
Subject: [PATCH 2/6] update postgres / mysql migrations

---
 db/mysql/migrations/01_init/migration.sql              | 5 +----
 db/postgresql/migrations/10_add_reset_at/migration.sql | 9 +++++++++
 2 files changed, 10 insertions(+), 4 deletions(-)
 create mode 100644 db/postgresql/migrations/10_add_reset_at/migration.sql

diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql
index 58b7e698..aea2574f 100644
--- a/db/mysql/migrations/01_init/migration.sql
+++ b/db/mysql/migrations/01_init/migration.sql
@@ -41,7 +41,7 @@ CREATE TABLE `website` (
     `name` VARCHAR(100) NOT NULL,
     `domain` VARCHAR(500) NULL,
     `share_id` VARCHAR(50) NULL,
-    `rev_id` INTEGER UNSIGNED NOT NULL DEFAULT 0,
+    `reset_at` TIMESTAMP(0) NULL,
     `user_id` VARCHAR(36) NULL,
     `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
     `updated_at` TIMESTAMP(0) NULL,
@@ -138,6 +138,3 @@ CREATE TABLE `team_website` (
     INDEX `team_website_website_id_idx`(`website_id`),
     PRIMARY KEY (`team_website_id`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-
--- AddSystemUser
-INSERT INTO "user" (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');
\ No newline at end of file
diff --git a/db/postgresql/migrations/10_add_reset_at/migration.sql b/db/postgresql/migrations/10_add_reset_at/migration.sql
new file mode 100644
index 00000000..ba185e1b
--- /dev/null
+++ b/db/postgresql/migrations/10_add_reset_at/migration.sql
@@ -0,0 +1,9 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `rev_id` on the `website` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "website" DROP COLUMN "rev_id",
+ADD COLUMN     "reset_at" TIMESTAMPTZ(6);

From fbe35f4c97a3fd44b9bff5725166edde0b8ef3e5 Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Mon, 27 Mar 2023 12:11:06 -0700
Subject: [PATCH 3/6] fixed relational query to get reset_at

---
 queries/analytics/event/getEventMetrics.ts       | 3 ++-
 queries/analytics/eventData/getEventData.ts      | 3 ++-
 queries/analytics/pageview/getPageviewMetrics.ts | 3 ++-
 queries/analytics/pageview/getPageviewStats.ts   | 3 ++-
 queries/analytics/session/getSessionMetrics.ts   | 3 ++-
 queries/analytics/stats/getWebsiteStats.ts       | 3 ++-
 6 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/queries/analytics/event/getEventMetrics.ts b/queries/analytics/event/getEventMetrics.ts
index 35eda8f6..f69d9bb1 100644
--- a/queries/analytics/event/getEventMetrics.ts
+++ b/queries/analytics/event/getEventMetrics.ts
@@ -4,6 +4,7 @@ import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
 import cache from 'lib/cache';
 import { WebsiteEventMetric } from 'lib/types';
 import { EVENT_TYPE } from 'lib/constants';
+import { getWebsite } from 'queries';
 
 export async function getEventMetrics(
   ...args: [
@@ -46,7 +47,7 @@ async function relationalQuery(
   },
 ) {
   const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma;
-  const website = await cache.fetchWebsite(websiteId);
+  const website = await getWebsite({ id: websiteId });
   const resetDate = website?.resetAt || website?.createdAt;
   const params: any = [websiteId, resetDate, startDate, endDate];
 
diff --git a/queries/analytics/eventData/getEventData.ts b/queries/analytics/eventData/getEventData.ts
index f8e2febb..cd44d29a 100644
--- a/queries/analytics/eventData/getEventData.ts
+++ b/queries/analytics/eventData/getEventData.ts
@@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import { WebsiteEventDataMetric } from 'lib/types';
+import { getWebsite } from 'queries';
 
 export async function getEventData(
   ...args: [
@@ -48,7 +49,7 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
   const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
-  const website = await cache.fetchWebsite(websiteId);
+  const website = await getWebsite({ id: websiteId });
   const resetDate = website?.resetAt || website?.createdAt;
   const params: any = [websiteId, resetDate, startDate, endDate, eventName || ''];
 
diff --git a/queries/analytics/pageview/getPageviewMetrics.ts b/queries/analytics/pageview/getPageviewMetrics.ts
index 09524047..53464591 100644
--- a/queries/analytics/pageview/getPageviewMetrics.ts
+++ b/queries/analytics/pageview/getPageviewMetrics.ts
@@ -4,6 +4,7 @@ import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
 import cache from 'lib/cache';
 import { Prisma } from '@prisma/client';
 import { EVENT_TYPE } from 'lib/constants';
+import { getWebsite } from 'queries';
 
 export async function getPageviewMetrics(
   ...args: [
@@ -35,7 +36,7 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, column, filters = {}, type } = data;
   const { rawQuery, parseFilters, toUuid } = prisma;
-  const website = await cache.fetchWebsite(websiteId);
+  const website = await getWebsite({ id: websiteId });
   const resetDate = website?.resetAt || website?.createdAt;
   const params: any = [
     websiteId,
diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts
index c6197312..894f512b 100644
--- a/queries/analytics/pageview/getPageviewStats.ts
+++ b/queries/analytics/pageview/getPageviewStats.ts
@@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import { EVENT_TYPE } from 'lib/constants';
+import { getWebsite } from 'queries';
 
 export async function getPageviewStats(
   ...args: [
@@ -46,7 +47,7 @@ async function relationalQuery(
     sessionKey = 'session_id',
   } = data;
   const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma;
-  const website = await cache.fetchWebsite(websiteId);
+  const website = await getWebsite({ id: websiteId });
   const resetDate = website?.resetAt || website?.createdAt;
   const params: any = [websiteId, resetDate, startDate, endDate];
   const { filterQuery, joinSession } = parseFilters(filters, params);
diff --git a/queries/analytics/session/getSessionMetrics.ts b/queries/analytics/session/getSessionMetrics.ts
index 073f93f6..dc391d84 100644
--- a/queries/analytics/session/getSessionMetrics.ts
+++ b/queries/analytics/session/getSessionMetrics.ts
@@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
 import cache from 'lib/cache';
 import { EVENT_TYPE } from 'lib/constants';
+import { getWebsite } from 'queries';
 
 export async function getSessionMetrics(
   ...args: [
@@ -20,7 +21,7 @@ async function relationalQuery(
   websiteId: string,
   data: { startDate: Date; endDate: Date; field: string; filters: object },
 ) {
-  const website = await cache.fetchWebsite(websiteId);
+  const website = await getWebsite({ id: websiteId });
   const resetDate = website?.resetAt || website?.createdAt;
   const { startDate, endDate, field, filters = {} } = data;
   const { toUuid, parseFilters, rawQuery } = prisma;
diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts
index d11de565..39a89cbc 100644
--- a/queries/analytics/stats/getWebsiteStats.ts
+++ b/queries/analytics/stats/getWebsiteStats.ts
@@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
 import cache from 'lib/cache';
 import { EVENT_TYPE } from 'lib/constants';
+import { getWebsite } from 'queries';
 
 export async function getWebsiteStats(
   ...args: [websiteId: string, data: { startDate: Date; endDate: Date; filters: object }]
@@ -19,7 +20,7 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, filters = {} } = data;
   const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;
-  const website = await cache.fetchWebsite(websiteId);
+  const website = await getWebsite({ id: websiteId });
   const resetDate = website?.resetAt || website?.createdAt;
   const params: any = [websiteId, resetDate, startDate, endDate];
   const { filterQuery, joinSession } = parseFilters(filters, params);

From c38da5bfd7339e38eeaad1a83f9d33aab42e16e2 Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Mon, 27 Mar 2023 12:15:22 -0700
Subject: [PATCH 4/6] readd system user insert for mysql

---
 db/mysql/migrations/01_init/migration.sql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql
index aea2574f..6e310932 100644
--- a/db/mysql/migrations/01_init/migration.sql
+++ b/db/mysql/migrations/01_init/migration.sql
@@ -138,3 +138,6 @@ CREATE TABLE `team_website` (
     INDEX `team_website_website_id_idx`(`website_id`),
     PRIMARY KEY (`team_website_id`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+-- AddSystemUser
+INSERT INTO "user" (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');
\ No newline at end of file

From 077fad20ea88d00411a2b13762da337db7f801d9 Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Mon, 27 Mar 2023 12:44:59 -0700
Subject: [PATCH 5/6] update skip_broken_messages

---
 db/clickhouse/schema.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index 904cd377..d754ac8b 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -66,7 +66,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro
        kafka_group_name = 'event_consumer_group',
        kafka_format = 'JSONEachRow',
        kafka_max_block_size = 1048576,
-       kafka_skip_broken_messages = 1;
+       kafka_skip_broken_messages = 100;
 
 CREATE MATERIALIZED VIEW umami.event_queue_mv TO umami.event AS
 SELECT website_id,
@@ -130,7 +130,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro
        kafka_group_name = 'event_data_consumer_group',
        kafka_format = 'JSONEachRow',
        kafka_max_block_size = 1048576,
-       kafka_skip_broken_messages = 1;
+       kafka_skip_broken_messages = 100;
 
 CREATE MATERIALIZED VIEW umami.event_data_queue_mv TO umami.event_data AS
 SELECT website_id,

From 4d6abc45293798e69d19726441af35f693e4783d Mon Sep 17 00:00:00 2001
From: Francis Cao <franciscao@gmail.com>
Date: Mon, 27 Mar 2023 15:54:37 -0700
Subject: [PATCH 6/6] remove unused ref columns

---
 queries/analytics/event/saveEvent.ts | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts
index 1749dca5..9c0d9cc6 100644
--- a/queries/analytics/event/saveEvent.ts
+++ b/queries/analytics/event/saveEvent.ts
@@ -10,9 +10,6 @@ export async function saveEvent(args: {
   websiteId: string;
   urlPath: string;
   urlQuery?: string;
-  referrerPath?: string;
-  referrerQuery?: string;
-  referrerDomain?: string;
   pageTitle?: string;
   eventName?: string;
   eventData?: any;