diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 4c544a71..c25eff4e 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -2,11 +2,11 @@ import { ClickHouseClient, createClient } from '@clickhouse/client'; import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; -import { PageParams, QueryFilters, QueryOptions } from './types'; -import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants'; -import { fetchWebsite } from './load'; +import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants'; import { maxDate } from './date'; +import { fetchWebsite } from './load'; import { filtersToArray } from './params'; +import { PageParams, QueryFilters, QueryOptions } from './types'; export const CLICKHOUSE_DATE_FORMATS = { second: '%Y-%m-%dT%H:%i:%S', @@ -100,26 +100,6 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) return query.join('\n'); } -function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) { - const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => { - if (column) { - if (EVENT_COLUMNS.includes(name)) { - arr.push(`and has(${column}, {${name}:String})`); - - if (name === 'referrer') { - arr.push('and not has(referrer_domain, {websiteDomain:String})'); - } - } else { - arr.push(`and ${mapFilter(column, operator, name)}`); - } - } - - return arr; - }, []); - - return query.join('\n'); -} - function getDateQuery(filters: QueryFilters = {}) { const { startDate, endDate } = filters; @@ -159,25 +139,6 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio }; } -async function parseSessionFilters( - websiteId: string, - filters: QueryFilters = {}, - options?: QueryOptions, -) { - const website = await fetchWebsite(websiteId); - - return { - filterQuery: getSessionFilterQuery(filters, options), - dateQuery: getDateQuery(filters), - params: { - ...getFilterParams(filters), - websiteId, - startDate: maxDate(filters.startDate, new Date(website?.resetAt)), - websiteDomain: website.domain, - }, - }; -} - async function pagedQuery( query: string, queryParams: { [key: string]: any }, @@ -260,7 +221,6 @@ export default { getDateFormat, getFilterQuery, parseFilters, - parseSessionFilters, pagedQuery, findUnique, findFirst, diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 09eebb91..c5141d3b 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -74,8 +74,8 @@ async function clickhouseQuery( sql = ` select sum(t.c) as "pageviews", - count(distinct t.session_id) as "visitors", - count(distinct t.visit_id) as "visits", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", sum(if(t.c = 1, 1, 0)) as "bounces", sum(max_time-min_time) as "totaltime" from ( @@ -96,16 +96,24 @@ async function clickhouseQuery( } else { sql = ` select - sum(views) as "pageviews", + sum(t.c) as "pageviews", uniq(session_id) as "visitors", uniq(visit_id) as "visits", - sumIf(1, views = 1) as "bounces", + sumIf(1, t.c = 1) as "bounces", sum(max_time-min_time) as "totaltime" - from website_event_stats_hourly "website_event" + from (select + session_id, + visit_id, + sum(views) c, + min(min_time) min_time, + max(max_time) max_time + from umami.website_event_stats_hourly "website_event" where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} - ${filterQuery}; + ${filterQuery} + group by session_id, visit_id + ) as t; `; } diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts index 3e6f53c0..bb8bc4c5 100644 --- a/src/queries/analytics/sessions/getSessionMetrics.ts +++ b/src/queries/analytics/sessions/getSessionMetrics.ts @@ -1,5 +1,5 @@ import clickhouse from 'lib/clickhouse'; -import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; +import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; @@ -64,15 +64,34 @@ async function clickhouseQuery( offset: number = 0, ): Promise<{ x: string; y: number }[]> { const column = FILTER_COLUMNS[type] || type; - const { parseSessionFilters, rawQuery } = clickhouse; - const { filterQuery, params } = await parseSessionFilters(websiteId, { + const { parseFilters, rawQuery } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, }); const includeCountry = column === 'city' || column === 'subdivision1'; - return rawQuery( - ` + let sql = ''; + + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { + sql = ` + select + ${column} x, + count(distinct session_id) y + ${includeCountry ? ', country' : ''} + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by x + ${includeCountry ? ', country' : ''} + order by y desc + limit ${limit} + offset ${offset} + `; + } else { + sql = ` select ${column} x, uniq(session_id) y @@ -87,9 +106,10 @@ async function clickhouseQuery( order by y desc limit ${limit} offset ${offset} - `, - params, - ).then(a => { + `; + } + + return rawQuery(sql, params).then(a => { return Object.values(a).map(a => { return { x: a.x, y: Number(a.y), country: a.country }; }); diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts index fa0ed6a9..fa748333 100644 --- a/src/queries/analytics/sessions/getSessionStats.ts +++ b/src/queries/analytics/sessions/getSessionStats.ts @@ -1,7 +1,7 @@ import clickhouse from 'lib/clickhouse'; +import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; -import { EVENT_TYPE } from 'lib/constants'; import { QueryFilters } from 'lib/types'; export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) { @@ -41,25 +41,24 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; - const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse; - const { filterQuery, params } = await parseSessionFilters(websiteId, { + const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, }); - const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly'; - const columnQuery = unit === 'minute' ? 'count(distinct session_id)' : 'uniq(session_id)'; + let sql = ''; - return rawQuery( - ` + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') { + sql = ` select ${getDateStringSQL('g.t', unit)} as x, g.y as y from ( select ${getDateSQL('created_at', unit, timezone)} as t, - ${columnQuery} as y - from ${table} website_event + count(distinct session_id) as y + from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} @@ -67,9 +66,28 @@ async function clickhouseQuery( group by t ) as g order by t - `, - params, - ).then(result => { + `; + } else { + sql = ` + select + ${getDateStringSQL('g.t', unit)} as x, + g.y as y + from ( + select + ${getDateSQL('created_at', unit, timezone)} as t, + uniq(session_id) as y + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `; + } + + return rawQuery(sql, params).then(result => { return Object.values(result).map((a: any) => { return { x: a.x, y: Number(a.y) }; });