From 9882ff24f6fecd317159705d0f6f32244ac382cd Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 1 Aug 2024 15:32:49 -0700 Subject: [PATCH] route dashboard queries based on filters selected --- src/lib/clickhouse.ts | 42 +++++++- src/lib/constants.ts | 15 +-- .../analytics/events/getEventMetrics.ts | 34 +++++-- src/queries/analytics/getWebsiteStats.ts | 59 ++++++------ .../analytics/pageviews/getPageviewMetrics.ts | 95 +++++++++++-------- .../analytics/pageviews/getPageviewStats.ts | 55 +++++------ .../analytics/sessions/getSessionMetrics.ts | 4 +- .../analytics/sessions/getSessionStats.ts | 4 +- 8 files changed, 191 insertions(+), 117 deletions(-) diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 2bda9bfa..4c544a71 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -3,7 +3,7 @@ import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; import { PageParams, QueryFilters, QueryOptions } from './types'; -import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants'; +import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants'; import { fetchWebsite } from './load'; import { maxDate } from './date'; import { filtersToArray } from './params'; @@ -100,6 +100,26 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) return query.join('\n'); } +function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) { + const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => { + if (column) { + if (EVENT_COLUMNS.includes(name)) { + arr.push(`and has(${column}, {${name}:String})`); + + if (name === 'referrer') { + arr.push('and not has(referrer_domain, {websiteDomain:String})'); + } + } else { + arr.push(`and ${mapFilter(column, operator, name)}`); + } + } + + return arr; + }, []); + + return query.join('\n'); +} + function getDateQuery(filters: QueryFilters = {}) { const { startDate, endDate } = filters; @@ -139,6 +159,25 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio }; } +async function parseSessionFilters( + websiteId: string, + filters: QueryFilters = {}, + options?: QueryOptions, +) { + const website = await fetchWebsite(websiteId); + + return { + filterQuery: getSessionFilterQuery(filters, options), + dateQuery: getDateQuery(filters), + params: { + ...getFilterParams(filters), + websiteId, + startDate: maxDate(filters.startDate, new Date(website?.resetAt)), + websiteDomain: website.domain, + }, + }; +} + async function pagedQuery( query: string, queryParams: { [key: string]: any }, @@ -221,6 +260,7 @@ export default { getDateFormat, getFilterQuery, parseFilters, + parseSessionFilters, pagedQuery, findUnique, findFirst, diff --git a/src/lib/constants.ts b/src/lib/constants.ts index aa1b3c0f..9d696f11 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -33,16 +33,7 @@ export const FILTER_REFERRERS = 'filter-referrers'; export const FILTER_PAGES = 'filter-pages'; export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute']; -export const EVENT_COLUMNS = [ - 'url', - 'entry', - 'exit', - 'referrer', - 'title', - 'query', - 'event', - 'host', -]; +export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event']; export const SESSION_COLUMNS = [ 'browser', @@ -58,8 +49,8 @@ export const SESSION_COLUMNS = [ export const FILTER_COLUMNS = { url: 'url_path', - entry: 'entry_url', - exit: 'exit_url', + entry: 'url_path', + exit: 'url_path', referrer: 'referrer_domain', host: 'hostname', title: 'page_title', diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts index 36232135..504cea11 100644 --- a/src/queries/analytics/events/getEventMetrics.ts +++ b/src/queries/analytics/events/getEventMetrics.ts @@ -1,8 +1,8 @@ -import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; -import { WebsiteEventMetric, QueryFilters } from 'lib/types'; import { EVENT_TYPE } from 'lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; +import { QueryFilters, WebsiteEventMetric } from 'lib/types'; export async function getEventMetrics( ...args: [websiteId: string, filters: QueryFilters] @@ -51,8 +51,24 @@ async function clickhouseQuery( eventType: EVENT_TYPE.customEvent, }); - return rawQuery( - ` + let sql = ''; + + if (filterQuery) { + sql = ` + select + event_name x, + ${getDateSQL('created_at', unit, timezone)} t, + count(*) y + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by x, t + order by t + `; + } else { + sql = ` select event_name x, ${getDateSQL('created_at', unit, timezone)} t, @@ -64,13 +80,13 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} - ${filterQuery} ) as g group by x, t order by t - `, - params, - ).then(a => { + `; + } + + return rawQuery(sql, params).then(a => { return Object.values(a).map(a => { return { x: a.x, t: a.t, y: Number(a.y) }; }); diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 7d8a76f6..09eebb91 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -3,6 +3,7 @@ import { EVENT_TYPE } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; +import { EVENT_COLUMNS } from 'lib/constants'; export async function getWebsiteStats( ...args: [websiteId: string, filters: QueryFilters] @@ -67,30 +68,33 @@ async function clickhouseQuery( eventType: EVENT_TYPE.pageView, }); - return rawQuery( - // ` - // select - // sum(t.c) as "pageviews", - // count(distinct t.session_id) as "visitors", - // count(distinct t.visit_id) as "visits", - // sum(if(t.c = 1, 1, 0)) as "bounces", - // sum(max_time-min_time) as "totaltime" - // from ( - // select - // session_id, - // visit_id, - // count(*) c, - // min(created_at) min_time, - // max(created_at) max_time - // from website_event - // where website_id = {websiteId:UUID} - // and created_at between {startDate:DateTime64} and {endDate:DateTime64} - // and event_type = {eventType:UInt32} - // ${filterQuery} - // group by session_id, visit_id - // ) as t; - // `, - ` + let sql = ''; + + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { + sql = ` + select + sum(t.c) as "pageviews", + count(distinct t.session_id) as "visitors", + count(distinct t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by session_id, visit_id + ) as t; + `; + } else { + sql = ` select sum(views) as "pageviews", uniq(session_id) as "visitors", @@ -102,9 +106,10 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery}; - `, - params, - ).then(result => { + `; + } + + return rawQuery(sql, params).then(result => { return Object.values(result).map((a: any) => { return { pageviews: Number(a.pageviews), diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts index 4ac101b2..f734b1dd 100644 --- a/src/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts @@ -1,5 +1,5 @@ import clickhouse from 'lib/clickhouse'; -import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; +import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; @@ -91,46 +91,66 @@ async function clickhouseQuery( }); let excludeDomain = ''; - let groupByQuery = ''; + let sql = ''; - if (column === 'referrer_domain') { - excludeDomain = `and t != {websiteDomain:String} and t != ''`; - } + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { + let entryExitQuery = ''; - let columnQuery = `arrayJoin(${column})`; + if (column === 'referrer_domain') { + excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`; + } - if (type === 'entry') { - columnQuery = `visit_id x, argMinMerge(${column})`; - } + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; - if (type === 'exit') { - columnQuery = `visit_id x, argMaxMerge(${column})`; - } + entryExitQuery = ` + JOIN (select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + group by visit_id) x + ON x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at`; + } - if (type === 'entry' || type === 'exit') { - groupByQuery = 'group by x'; - } + sql = ` + select ${column} x, count(*) y + from website_event + ${entryExitQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${excludeDomain} + ${filterQuery} + group by x + order by y desc + limit ${limit} + offset ${offset} + `; + } else { + let groupByQuery = ''; - // let excludeDomain = ''; - // if (column === 'referrer_domain') { - // excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`; - // } + if (column === 'referrer_domain') { + excludeDomain = `and t != {websiteDomain:String} and t != ''`; + } - return rawQuery( - // ` - // select ${column} x, count(*) y - // from website_event - // where website_id = {websiteId:UUID} - // and created_at between {startDate:DateTime64} and {endDate:DateTime64} - // and event_type = {eventType:UInt32} - // ${excludeDomain} - // ${filterQuery} - // group by x - // order by y desc - // limit ${limit} - // offset ${offset} - // `, - ` + let columnQuery = `arrayJoin(${column})`; + + if (type === 'entry') { + columnQuery = `visit_id x, argMinMerge(entry_url)`; + } + + if (type === 'exit') { + columnQuery = `visit_id x, argMaxMerge(exit_url)`; + } + + if (type === 'entry' || type === 'exit') { + groupByQuery = 'group by x'; + } + + sql = ` select g.t as x, count(*) as y from ( @@ -146,9 +166,10 @@ async function clickhouseQuery( order by y desc limit ${limit} offset ${offset} - `, - params, - ).then((result: any) => { + `; + } + + return rawQuery(sql, params).then((result: any) => { return Object.values(result).map((a: any) => { return { x: a.x, y: Number(a.y) }; }); diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts index c378d48a..1d027e79 100644 --- a/src/queries/analytics/pageviews/getPageviewStats.ts +++ b/src/queries/analytics/pageviews/getPageviewStats.ts @@ -1,7 +1,7 @@ import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; -import { EVENT_TYPE } from 'lib/constants'; +import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants'; import { QueryFilters } from 'lib/types'; export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) { @@ -47,36 +47,18 @@ async function clickhouseQuery( eventType: EVENT_TYPE.pageView, }); - const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly'; - const columnQuery = unit === 'minute' ? 'count(*)' : 'sum(views)'; + let sql = ''; - return rawQuery( - // ` - // select - // ${getDateStringSQL('g.t', unit)} as x, - // g.y as y - // from ( - // select - // ${getDateSQL('created_at', unit, timezone)} as t, - // count(*) as y - // from website_event - // where website_id = {websiteId:UUID} - // and created_at between {startDate:DateTime64} and {endDate:DateTime64} - // and event_type = {eventType:UInt32} - // ${filterQuery} - // group by t - // ) as g - // order by t - // `, - ` + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') { + sql = ` select ${getDateStringSQL('g.t', unit)} as x, g.y as y from ( select ${getDateSQL('created_at', unit, timezone)} as t, - ${columnQuery} as y - from ${table} website_event + count(*) as y + from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} @@ -84,9 +66,28 @@ async function clickhouseQuery( group by t ) as g order by t - `, - params, - ).then(result => { + `; + } else { + sql = ` + select + ${getDateStringSQL('g.t', unit)} as x, + g.y as y + from ( + select + ${getDateSQL('created_at', unit, timezone)} as t, + sum(views)as y + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `; + } + + return rawQuery(sql, params).then(result => { return Object.values(result).map((a: any) => { return { x: a.x, y: Number(a.y) }; }); diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts index 9baf2a5c..3e6f53c0 100644 --- a/src/queries/analytics/sessions/getSessionMetrics.ts +++ b/src/queries/analytics/sessions/getSessionMetrics.ts @@ -64,8 +64,8 @@ async function clickhouseQuery( offset: number = 0, ): Promise<{ x: string; y: number }[]> { const column = FILTER_COLUMNS[type] || type; - const { parseFilters, rawQuery } = clickhouse; - const { filterQuery, params } = await parseFilters(websiteId, { + const { parseSessionFilters, rawQuery } = clickhouse; + const { filterQuery, params } = await parseSessionFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, }); diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts index 7bba14df..fa0ed6a9 100644 --- a/src/queries/analytics/sessions/getSessionStats.ts +++ b/src/queries/analytics/sessions/getSessionStats.ts @@ -41,8 +41,8 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; - const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse; - const { filterQuery, params } = await parseFilters(websiteId, { + const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse; + const { filterQuery, params } = await parseSessionFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, });