From f518066d932fc689a10ae32c142698468d667144 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 10 Jul 2024 00:54:46 -0700 Subject: [PATCH] refactor views and queries for dashboard --- src/lib/constants.ts | 4 +-- src/pages/api/websites/[websiteId]/stats.ts | 10 ++++-- src/queries/analytics/getWebsiteStats.ts | 33 ++++++++----------- .../analytics/pageviews/getPageviewMetrics.ts | 22 ++++--------- .../analytics/pageviews/getPageviewStats.ts | 4 +-- .../analytics/sessions/getSessionMetrics.ts | 2 +- .../analytics/sessions/getSessionStats.ts | 2 +- 7 files changed, 34 insertions(+), 43 deletions(-) diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 35917802..8a23f930 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -58,8 +58,8 @@ export const SESSION_COLUMNS = [ export const FILTER_COLUMNS = { url: 'url_path', - entry: 'url_path', - exit: 'url_path', + entry: 'entry_url', + exit: 'exit_url', referrer: 'referrer_domain', host: 'hostname', title: 'page_title', diff --git a/src/pages/api/websites/[websiteId]/stats.ts b/src/pages/api/websites/[websiteId]/stats.ts index 0189627a..1c684dbe 100644 --- a/src/pages/api/websites/[websiteId]/stats.ts +++ b/src/pages/api/websites/[websiteId]/stats.ts @@ -56,7 +56,7 @@ export default async ( await useAuth(req, res); await useValidate(schema, req, res); - const { websiteId, compare } = req.query; + const { websiteId, compare, unit } = req.query; if (req.method === 'GET') { if (!(await canViewWebsite(req.auth, websiteId))) { @@ -72,9 +72,13 @@ export default async ( const filters = getRequestFilters(req); - const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate }); + const metrics = await getWebsiteStats(websiteId, unit as string, { + ...filters, + startDate, + endDate, + }); - const prevPeriod = await getWebsiteStats(websiteId, { + const prevPeriod = await getWebsiteStats(websiteId, unit as string, { ...filters, startDate: compareStartDate, endDate: compareEndDate, diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 2f3c82e8..13eadbbb 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */ import clickhouse from 'lib/clickhouse'; import { EVENT_TYPE } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; @@ -5,7 +6,7 @@ import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; export async function getWebsiteStats( - ...args: [websiteId: string, filters: QueryFilters] + ...args: [websiteId: string, unit: string, filters: QueryFilters] ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { @@ -17,6 +18,7 @@ export async function getWebsiteStats( async function relationalQuery( websiteId: string, + unit: string, filters: QueryFilters, ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] @@ -57,6 +59,7 @@ async function relationalQuery( async function clickhouseQuery( websiteId: string, + unit: string, filters: QueryFilters, ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] @@ -66,29 +69,21 @@ async function clickhouseQuery( ...filters, eventType: EVENT_TYPE.pageView, }); + const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` select - sum(t.c) as "pageviews", - uniq(t.session_id) as "visitors", - uniq(t.visit_id) as "visits", - sum(if(t.c = 1, 1, 0)) as "bounces", + sum(views) as "pageviews", + uniq(session_id) as "visitors", + uniq(visit_id) as "visits", + sumIf(1, views = 1) as "bounces", sum(max_time-min_time) as "totaltime" - from ( - select - session_id, - visit_id, - count(*) c, - min(created_at) min_time, - max(created_at) max_time - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - ${filterQuery} - group by session_id, visit_id - ) as t; + from ${table} "website_event" + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery}; `, params, ).then(result => { diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts index 25c264b7..34d26d9d 100644 --- a/src/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts @@ -97,34 +97,26 @@ async function clickhouseQuery( eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, }); - let entryExitQuery = ''; + let columnAgg = column; let excludeDomain = ''; if (column === 'referrer_domain') { excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`; } - if (type === 'entry' || type === 'exit') { - const aggregrate = type === 'entry' ? 'min' : 'max'; + if (type === 'entry') { + columnAgg = `argMinMerge(${column})`; + } - entryExitQuery = ` - JOIN (select visit_id, - ${aggregrate}(created_at) target_created_at - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - group by visit_id) x - ON x.visit_id = website_event.visit_id - and x.target_created_at = website_event.created_at`; + if (type === 'exit') { + columnAgg = `argMaxMerge(${column})`; } const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily'; return rawQuery( ` - select ${column} x, countMerge(views) y + select ${column} x, sum(views) y from ${table} website_event - ${entryExitQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts index f6942e6a..57292f65 100644 --- a/src/queries/analytics/pageviews/getPageviewStats.ts +++ b/src/queries/analytics/pageviews/getPageviewStats.ts @@ -46,7 +46,7 @@ async function clickhouseQuery( ...filters, eventType: EVENT_TYPE.pageView, }); - const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily'; + const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` @@ -56,7 +56,7 @@ async function clickhouseQuery( from ( select ${getDateQuery('created_at', unit, timezone)} as t, - countMerge(views) as y + sum(views) as y from ${table} website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts index 58b5198c..93e36a55 100644 --- a/src/queries/analytics/sessions/getSessionMetrics.ts +++ b/src/queries/analytics/sessions/getSessionMetrics.ts @@ -80,7 +80,7 @@ async function clickhouseQuery( eventType: EVENT_TYPE.pageView, }); const includeCountry = column === 'city' || column === 'subdivision1'; - const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily'; + const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts index dc6d635d..6db03c46 100644 --- a/src/queries/analytics/sessions/getSessionStats.ts +++ b/src/queries/analytics/sessions/getSessionStats.ts @@ -46,7 +46,7 @@ async function clickhouseQuery( ...filters, eventType: EVENT_TYPE.pageView, }); - const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily'; + const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( `