diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index d754ac8b..77176413 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -1,7 +1,7 @@ SET allow_experimental_object_type = 1; -- Create Event -CREATE TABLE umami.event +CREATE TABLE umami.website_event ( website_id UUID, session_id UUID, @@ -33,7 +33,7 @@ CREATE TABLE umami.event ORDER BY (website_id, session_id, created_at) SETTINGS index_granularity = 8192; -CREATE TABLE umami.event_queue ( +CREATE TABLE umami.website_event_queue ( website_id UUID, session_id UUID, event_id UUID, @@ -68,7 +68,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro kafka_max_block_size = 1048576, kafka_skip_broken_messages = 100; -CREATE MATERIALIZED VIEW umami.event_queue_mv TO umami.event AS +CREATE MATERIALIZED VIEW umami.website_event_queue_mv TO umami.website_event AS SELECT website_id, session_id, event_id, @@ -91,7 +91,7 @@ SELECT website_id, event_type, event_name, created_at -FROM umami.event_queue; +FROM umami.website_event_queue; CREATE TABLE umami.event_data ( diff --git a/queries/analytics/event/getEventMetrics.ts b/queries/analytics/event/getEventMetrics.ts index f69d9bb1..80dcd8b8 100644 --- a/queries/analytics/event/getEventMetrics.ts +++ b/queries/analytics/event/getEventMetrics.ts @@ -97,7 +97,7 @@ async function clickhouseQuery( event_name x, ${getDateQuery('created_at', unit, timezone)} t, count(*) y - from event + from website_event where website_id = {websiteId:UUID} and event_type = ${EVENT_TYPE.customEvent} and created_at >= ${getDateFormat(resetDate)} diff --git a/queries/analytics/event/getEvents.ts b/queries/analytics/event/getEvents.ts index 3c175216..a5b1be8f 100644 --- a/queries/analytics/event/getEvents.ts +++ b/queries/analytics/event/getEvents.ts @@ -33,7 +33,7 @@ function clickhouseQuery(websiteId: string, startAt: Date) { toUnixTimestamp(created_at) as timestamp, url_path, event_name as eventName - from event + from website_event where event_type = ${EVENT_TYPE.customEvent} and website_id = {websiteId:UUID} and created_at >= {startAt:DateTime('UTC')}`, diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts index 01ef5221..332b349c 100644 --- a/queries/analytics/event/saveEvent.ts +++ b/queries/analytics/event/saveEvent.ts @@ -126,6 +126,7 @@ async function clickhouseQuery(data: { subdivision1, subdivision2, city, + ...args } = data; const { getDateFormat, sendMessage } = kafka; const eventId = uuid(); @@ -148,6 +149,7 @@ async function clickhouseQuery(data: { event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, created_at: createdAt, + ...args, }; await sendMessage(message, 'event'); diff --git a/queries/analytics/pageview/getPageviewMetrics.ts b/queries/analytics/pageview/getPageviewMetrics.ts index 53464591..214dbaab 100644 --- a/queries/analytics/pageview/getPageviewMetrics.ts +++ b/queries/analytics/pageview/getPageviewMetrics.ts @@ -85,7 +85,7 @@ async function clickhouseQuery( return rawQuery( `select ${column} x, count(*) y - from event + from website_event where website_id = {websiteId:UUID} and event_type = {eventType:UInt32} and created_at >= ${getDateFormat(resetDate)} diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index 894f512b..8a224428 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -101,7 +101,7 @@ async function clickhouseQuery( (select ${getDateQuery('created_at', unit, timezone)} t, count(${count !== '*' ? 'distinct session_id' : count}) y - from event + from website_event where website_id = {websiteId:UUID} and event_type = ${EVENT_TYPE.pageView} and created_at >= ${getDateFormat(resetDate)} diff --git a/queries/analytics/pageview/getPageviews.ts b/queries/analytics/pageview/getPageviews.ts index d23669a7..4e7730be 100644 --- a/queries/analytics/pageview/getPageviews.ts +++ b/queries/analytics/pageview/getPageviews.ts @@ -31,7 +31,7 @@ async function clickhouseQuery(websiteId: string, startAt: Date) { created_at as createdAt, toUnixTimestamp(created_at) as timestamp, url_path - from event + from website_event where event_type = ${EVENT_TYPE.pageView} and website_id = {websiteId:UUID} and created_at >= {startAt:DateTime('UTC')}`, diff --git a/queries/analytics/pageview/savePageView.ts b/queries/analytics/pageview/savePageView.ts deleted file mode 100644 index 41742774..00000000 --- a/queries/analytics/pageview/savePageView.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { URL_LENGTH, EVENT_TYPE } from 'lib/constants'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import kafka from 'lib/kafka'; -import prisma from 'lib/prisma'; -import { uuid } from 'lib/crypto'; - -export async function savePageView(args: { - id: string; - websiteId: string; - urlPath: string; - urlQuery?: string; - referrerPath?: string; - referrerQuery?: string; - referrerDomain?: string; - pageTitle?: string; - hostname?: string; - browser?: string; - os?: string; - device?: string; - screen?: string; - language?: string; - country?: string; - subdivision1?: string; - subdivision2?: string; - city?: string; -}) { - return runQuery({ - [PRISMA]: () => relationalQuery(args), - [CLICKHOUSE]: () => clickhouseQuery(args), - }); -} - -async function relationalQuery(data: { - id: string; - websiteId: string; - urlPath: string; - urlQuery?: string; - referrerPath?: string; - referrerQuery?: string; - referrerDomain?: string; - pageTitle?: string; -}) { - const { - websiteId, - id: sessionId, - urlPath, - urlQuery, - referrerPath, - referrerQuery, - referrerDomain, - pageTitle, - } = data; - - return prisma.client.websiteEvent.create({ - data: { - id: uuid(), - websiteId, - sessionId, - urlPath: urlPath?.substring(0, URL_LENGTH), - urlQuery: urlQuery?.substring(0, URL_LENGTH), - referrerPath: referrerPath?.substring(0, URL_LENGTH), - referrerQuery: referrerQuery?.substring(0, URL_LENGTH), - referrerDomain: referrerDomain?.substring(0, URL_LENGTH), - pageTitle: pageTitle, - eventType: EVENT_TYPE.pageView, - }, - }); -} - -async function clickhouseQuery(data: { - id: string; - websiteId: string; - urlPath: string; - urlQuery?: string; - referrerPath?: string; - referrerQuery?: string; - referrerDomain?: string; - pageTitle?: string; - hostname?: string; - browser?: string; - os?: string; - device?: string; - screen?: string; - language?: string; - country?: string; - subdivision1?: string; - subdivision2?: string; - city?: string; -}) { - const { - websiteId, - id: sessionId, - urlPath, - urlQuery, - referrerPath, - referrerQuery, - referrerDomain, - pageTitle, - country, - subdivision1, - subdivision2, - city, - ...args - } = data; - const { getDateFormat, sendMessage } = kafka; - - const message = { - website_id: websiteId, - session_id: sessionId, - event_id: uuid(), - country: country ? country : null, - subdivision1: subdivision1 ? subdivision1 : null, - subdivision2: subdivision2 ? subdivision2 : null, - city: city ? city : null, - url_path: urlPath?.substring(0, URL_LENGTH), - url_query: urlQuery?.substring(0, URL_LENGTH), - referrer_path: referrerPath?.substring(0, URL_LENGTH), - referrer_query: referrerQuery?.substring(0, URL_LENGTH), - referrer_domain: referrerDomain?.substring(0, URL_LENGTH), - page_title: pageTitle, - event_type: EVENT_TYPE.pageView, - created_at: getDateFormat(new Date()), - ...args, - }; - - await sendMessage(message, 'event'); - - return data; -} diff --git a/queries/analytics/session/getSession.ts b/queries/analytics/session/getSession.ts index a3400d26..d226e832 100644 --- a/queries/analytics/session/getSession.ts +++ b/queries/analytics/session/getSession.ts @@ -35,7 +35,7 @@ async function clickhouseQuery({ id: sessionId }: { id: string }) { subdivision1, subdivision2, city - from event + from website_event where session_id = {sessionId:UUID} limit 1`, params, diff --git a/queries/analytics/session/getSessionMetrics.ts b/queries/analytics/session/getSessionMetrics.ts index dc391d84..703ca871 100644 --- a/queries/analytics/session/getSessionMetrics.ts +++ b/queries/analytics/session/getSessionMetrics.ts @@ -62,7 +62,7 @@ async function clickhouseQuery( return rawQuery( `select ${field} x, count(distinct session_id) y - from event as x + from website_event as x where website_id = {websiteId:UUID} and event_type = ${EVENT_TYPE.pageView} and created_at >= ${getDateFormat(resetDate)} diff --git a/queries/analytics/session/getSessions.ts b/queries/analytics/session/getSessions.ts index aefc49ca..4d09cc72 100644 --- a/queries/analytics/session/getSessions.ts +++ b/queries/analytics/session/getSessions.ts @@ -39,7 +39,7 @@ async function clickhouseQuery(websiteId: string, startAt: Date) { subdivision1, subdivision2, city - from event + from website_event where website_id = {websiteId:UUID} and created_at >= {startAt:DateTime('UTC')}`, { diff --git a/queries/analytics/stats/getActiveVisitors.ts b/queries/analytics/stats/getActiveVisitors.ts index 60a1e5c1..89f092c1 100644 --- a/queries/analytics/stats/getActiveVisitors.ts +++ b/queries/analytics/stats/getActiveVisitors.ts @@ -33,7 +33,7 @@ async function clickhouseQuery(websiteId: string) { return rawQuery( `select count(distinct session_id) x - from event + from website_event where website_id = {websiteId:UUID} and created_at >= {startAt:DateTime('UTC')}`, params, diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index 39a89cbc..d61630dd 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -72,7 +72,7 @@ async function clickhouseQuery( count(*) c, min(created_at) min_time, max(created_at) max_time - from event + from website_event where event_type = ${EVENT_TYPE.pageView} and website_id = {websiteId:UUID} and created_at >= ${getDateFormat(resetDate)}