m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2025-02-14 21:10:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactored query parameter handling.

Browse Source
This commit is contained in:
Mike Cao 2023-08-04 13:18:30 -07:00
parent 157862834d
commit 7148f66d1a
17 changed files with 260 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/clickhouse.ts
View File

@ -2,8 +2,10 @@ import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat'; import dateFormat from 'dateformat';
import debug from 'debug'; import debug from 'debug';
import { CLICKHOUSE } from 'lib/db'; import { CLICKHOUSE } from 'lib/db';
import { WebsiteMetricFilter } from './types'; import { QueryFilters } from './types';
import { FILTER_COLUMNS } from './constants'; import { FILTER_COLUMNS, IGNORED_FILTERS } from './constants';
import { loadWebsite } from './load';
import { maxDate } from './date';
export const CLICKHOUSE_DATE_FORMATS = { export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00', minute: '%Y-%m-%d %H:%M:00',
@ -65,13 +67,13 @@ function getFilterQuery(filters = {}) {
const query = Object.keys(filters).reduce((arr, key) => { const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key]; const filter = filters[key];
if (filter !== undefined) { if (filter !== undefined && !IGNORED_FILTERS.includes(key)) {
const column = FILTER_COLUMNS[key] || key; const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column} = {${key}:String}`); arr.push(`and ${column} = {${key}:String}`);
} }
if (key === 'referrer') { if (key === 'referrer') {
arr.push('and referrer_domain != {domain:String}'); arr.push('and referrer_domain != {websiteDomain:String}');
} }
return arr; return arr;
@ -80,9 +82,20 @@ function getFilterQuery(filters = {}) {
return query.join('\n'); return query.join('\n');
} }
function parseFilters(filters: WebsiteMetricFilter = {}) { async function parseFilters(
websiteId: string,
filters: QueryFilters & { [key: string]: any } = {},
) {
const website = await loadWebsite(websiteId);
return { return {
filterQuery: getFilterQuery(filters), filterQuery: getFilterQuery(filters),
params: {
...filters,
websiteId,
startDate: maxDate(filters.startDate, website.resetAt),
websiteDomain: website.domain,
},
}; };
} }

3
lib/constants.ts
View File

@ -50,8 +50,11 @@ export const FILTER_COLUMNS = {
query: 'url_query', query: 'url_query',
event: 'event_name', event: 'event_name',
region: 'subdivision1', region: 'subdivision1',
type: 'event_type',
}; };
export const IGNORED_FILTERS = ['startDate', 'endDate', 'timezone', 'unit', 'eventType'];
export const COLLECTION_TYPE = { export const COLLECTION_TYPE = {
event: 'event', event: 'event',
identify: 'identify', identify: 'identify',

19
lib/prisma.ts
View File

@ -1,7 +1,10 @@
import prisma from '@umami/prisma-client'; import prisma from '@umami/prisma-client';
import moment from 'moment-timezone'; import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants'; import { FILTER_COLUMNS, IGNORED_FILTERS, SESSION_COLUMNS } from './constants';
import { loadWebsite } from './load';
import { maxDate } from './date';
import { QueryFilters } from './types';
const MYSQL_DATE_FORMATS = { const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00', minute: '%Y-%m-%d %H:%i:00',
@ -68,14 +71,14 @@ function getFilterQuery(filters = {}): string {
const query = Object.keys(filters).reduce((arr, key) => { const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key]; const filter = filters[key];
if (filter !== undefined) { if (filter !== undefined && !IGNORED_FILTERS.includes(key)) {
const column = FILTER_COLUMNS[key] || key; const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column}={{${key}}}`); arr.push(`and ${column}={{${key}}}`);
} }
if (key === 'referrer') { if (key === 'referrer') {
arr.push( arr.push(
'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)', 'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',
); );
} }
@ -85,12 +88,20 @@ function getFilterQuery(filters = {}): string {
return query.join('\n'); return query.join('\n');
} }
function parseFilters(filters: { [key: string]: any } = {}) { async function parseFilters(websiteId, filters: QueryFilters & { [key: string]: any } = {}) {
const website = await loadWebsite(websiteId);
return { return {
joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key]) joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key])
? `inner join session on website_event.session_id = session.session_id` ? `inner join session on website_event.session_id = session.session_id`
: '', : '',
filterQuery: getFilterQuery(filters), filterQuery: getFilterQuery(filters),
params: {
...filters,
websiteId,
startDate: maxDate(filters.startDate, website.resetAt),
websiteDomain: website.domain,
},
}; };
} }

36
lib/types.ts
View File

@ -73,21 +73,6 @@ export interface WebsiteMetric {
y: number; y: number;
} }
export interface WebsiteMetricFilter {
domain?: string;
url?: string;
referrer?: string;
title?: string;
query?: string;
event?: string;
os?: string;
browser?: string;
device?: string;
country?: string;
region?: string;
city?: string;
}
export interface WebsiteEventMetric { export interface WebsiteEventMetric {
x: string; x: string;
t: string; t: string;
@ -144,3 +129,24 @@ export interface DateRange {
unit: string; unit: string;
value: string; value: string;
} }
export interface QueryFilters {
startDate?: Date;
endDate?: Date;
timezone?: string;
unit?: string;
domain?: string;
eventType?: number;
url?: string;
referrer?: string;
title?: string;
query?: string;
event?: string;
os?: string;
browser?: string;
device?: string;
country?: string;
region?: string;
city?: string;
language?: string;
}

17
pages/api/websites/[id]/metrics.ts
View File

@ -23,6 +23,7 @@ export interface WebsiteMetricsRequestQuery {
country: string; country: string;
region: string; region: string;
city: string; city: string;
language: string;
} }
export default async ( export default async (
@ -57,6 +58,8 @@ export default async (
const { startDate, endDate } = await parseDateRangeQuery(req); const { startDate, endDate } = await parseDateRangeQuery(req);
const filters = { const filters = {
startDate,
endDate,
url, url,
referrer, referrer,
title, title,
@ -76,12 +79,7 @@ export default async (
const column = FILTER_COLUMNS[type] || type; const column = FILTER_COLUMNS[type] || type;
if (SESSION_COLUMNS.includes(type)) { if (SESSION_COLUMNS.includes(type)) {
const data = await getSessionMetrics(websiteId, { const data = await getSessionMetrics(websiteId, column, filters);
startDate,
endDate,
column,
filters,
});
if (type === 'language') { if (type === 'language') {
const combined = {}; const combined = {};
@ -103,12 +101,7 @@ export default async (
} }
if (EVENT_COLUMNS.includes(type)) { if (EVENT_COLUMNS.includes(type)) {
const data = await getPageviewMetrics(websiteId, { const data = await getPageviewMetrics(websiteId, column, filters);
startDate,
endDate,
column,
filters,
});
return ok(res, data); return ok(res, data);
} }

52
pages/api/websites/[id]/pageviews.ts
View File

@ -57,41 +57,25 @@ export default async (
return badRequest(res); return badRequest(res);
} }
const filters = {
startDate,
endDate,
timezone,
unit,
url,
referrer,
title,
os,
browser,
device,
country,
region,
city,
};
const [pageviews, sessions] = await Promise.all([ const [pageviews, sessions] = await Promise.all([
getPageviewStats(websiteId, { getPageviewStats(websiteId, filters),
startDate, getSessionStats(websiteId, filters),
endDate,
timezone,
unit,
filters: {
url,
referrer,
title,
os,
browser,
device,
country,
region,
city,
},
}),
getSessionStats(websiteId, {
startDate,
endDate,
timezone,
unit,
filters: {
url,
referrer,
title,
os,
browser,
device,
country,
region,
city,
},
}),
]); ]);
return ok(res, { pageviews, sessions }); return ok(res, { pageviews, sessions });

46
pages/api/websites/[id]/stats.ts
View File

@ -56,40 +56,26 @@ export default async (
const prevStartDate = subMinutes(startDate, diff); const prevStartDate = subMinutes(startDate, diff);
const prevEndDate = subMinutes(endDate, diff); const prevEndDate = subMinutes(endDate, diff);
const metrics = await getWebsiteStats(websiteId, { const filters = {
startDate, url,
endDate, referrer,
filters: { title,
url, query,
referrer, event,
title, os,
query, browser,
event, device,
os, country,
browser, region,
device, city,
country, };
region,
city, const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate });
},
});
const prevPeriod = await getWebsiteStats(websiteId, { const prevPeriod = await getWebsiteStats(websiteId, {
...filters,
startDate: prevStartDate, startDate: prevStartDate,
endDate: prevEndDate, endDate: prevEndDate,
filters: {
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
region,
city,
},
}); });
const stats = Object.keys(metrics[0]).reduce((obj, key) => { const stats = Object.keys(metrics[0]).reduce((obj, key) => {

77
queries/analytics/eventData/getEventDataEvents.ts
View File

@ -1,17 +1,10 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataFields } from 'lib/types'; import { QueryFilters, WebsiteEventDataFields } from 'lib/types';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getEventDataEvents( export async function getEventDataEvents(
...args: [ ...args: [websiteId: string, filters: QueryFilters & { field?: string; event?: string }]
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
]
): Promise<WebsiteEventDataFields[]> { ): Promise<WebsiteEventDataFields[]> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -21,64 +14,60 @@ export async function getEventDataEvents(
async function relationalQuery( async function relationalQuery(
websiteId: string, websiteId: string,
startDate: Date, filters: QueryFilters & { field?: string; event?: string },
endDate: Date,
filters: { field?: string; event?: string },
) { ) {
const { rawQuery } = prisma; const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const { params } = await parseFilters(websiteId, filters);
const { event } = filters;
if (event) { if (event) {
return rawQuery( return rawQuery(
` `
select select
we.event_name as event, website_event.event_name as event,
ed.event_key as field, event_data.event_key as field,
ed.data_type as type, event_data.data_type as type,
ed.string_value as value, event_data.string_value as value,
count(*) as total count(*) as total
from event_data as ed from event_data
inner join website_event as we inner join website_event
on we.event_id = ed.website_event_id on website_event.event_id = event_data.website_event_id
where ed.website_id = {{websiteId::uuid}} where event_data.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}} and event_data.created_at between {{startDate}} and {{endDate}}
and we.event_name = {{event}} and websit_event.event_name = {{event}}
group by we.event_name, ed.event_key, ed.data_type, ed.string_value group by website_event.event_name, event_data.event_key, event_data.data_type, event_data.string_value
order by 1 asc, 2 asc, 3 asc, 4 desc order by 1 asc, 2 asc, 3 asc, 4 desc
`, `,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate, ...filters }, params,
); );
} }
return rawQuery( return rawQuery(
` `
select select
we.event_name as event, website_event.event_name as event,
ed.event_key as field, event_data.event_key as field,
ed.data_type as type, event_data.data_type as type,
count(*) as total count(*) as total
from event_data as ed from event_data
inner join website_event as we inner join website_event
on we.event_id = ed.website_event_id on website_event.event_id = event_data.website_event_id
where ed.website_id = {{websiteId::uuid}} where event_data.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}} and event_data.created_at between {{startDate}} and {{endDate}}
group by we.event_name, ed.event_key, ed.data_type group by website_event.event_name, event_data.event_key, event_data.data_type
order by 1 asc, 2 asc order by 1 asc, 2 asc
limit 100 limit 100
`, `,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }
async function clickhouseQuery( async function clickhouseQuery(
websiteId: string, websiteId: string,
startDate: Date, filters: QueryFilters & { field?: string; event?: string },
endDate: Date,
filters: { field?: string; event?: string },
) { ) {
const { rawQuery } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const { event } = filters; const { event } = filters;
const { params } = await parseFilters(websiteId, filters);
if (event) { if (event) {
return rawQuery( return rawQuery(
@ -97,7 +86,7 @@ async function clickhouseQuery(
order by 1 asc, 2 asc, 3 asc, 4 desc order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100 limit 100
`, `,
{ ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }
@ -115,6 +104,6 @@ async function clickhouseQuery(
order by 1 asc, 2 asc order by 1 asc, 2 asc
limit 100 limit 100
`, `,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }

28
queries/analytics/eventData/getEventDataFields.ts
View File

@ -1,12 +1,10 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataFields } from 'lib/types'; import { QueryFilters, WebsiteEventDataFields } from 'lib/types';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getEventDataFields( export async function getEventDataFields(
...args: [websiteId: string, startDate: Date, endDate: Date, field?: string] ...args: [websiteId: string, filters: QueryFilters & { field?: string }]
): Promise<WebsiteEventDataFields[]> { ): Promise<WebsiteEventDataFields[]> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -14,9 +12,10 @@ export async function getEventDataFields(
}); });
} }
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) { async function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) {
const { rawQuery } = prisma; const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const { field } = filters;
const { params } = await parseFilters(websiteId, filters);
if (field) { if (field) {
return rawQuery( return rawQuery(
@ -33,7 +32,7 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 desc, 1 asc order by 3 desc, 2 desc, 1 asc
limit 100 limit 100
`, `,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }
@ -50,13 +49,14 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 asc, 1 asc order by 3 desc, 2 asc, 1 asc
limit 100 limit 100
`, `,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) { async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) {
const { rawQuery } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const { field } = filters;
const { params } = await parseFilters(websiteId, filters);
if (field) { if (field) {
return rawQuery( return rawQuery(
@ -73,7 +73,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 desc, 1 asc order by 3 desc, 2 desc, 1 asc
limit 100 limit 100
`, `,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }
@ -90,6 +90,6 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 asc, 1 asc order by 3 desc, 2 asc, 1 asc
limit 100 limit 100
`, `,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, params,
); );
} }

59
queries/analytics/events/getEventMetrics.ts
View File

@ -1,24 +1,11 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { WebsiteEventMetric } from 'lib/types'; import { WebsiteEventMetric, QueryFilters } from 'lib/types';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export interface GetEventMetricsCriteria {
startDate: Date;
endDate: Date;
timezone: string;
unit: string;
filters: {
url: string;
eventName: string;
};
}
export async function getEventMetrics( export async function getEventMetrics(
...args: [websiteId: string, criteria: GetEventMetricsCriteria] ...args: [websiteId: string, criteria: QueryFilters]
): Promise<WebsiteEventMetric[]> { ): Promise<WebsiteEventMetric[]> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -26,11 +13,13 @@ export async function getEventMetrics(
}); });
} }
async function relationalQuery(websiteId: string, criteria: GetEventMetricsCriteria) { async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters } = criteria; const { timezone = 'utc', unit = 'day' } = filters;
const { rawQuery, getDateQuery, getFilterQuery } = prisma; const { rawQuery, getDateQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
const filterQuery = getFilterQuery(filters); ...filters,
eventType: EVENT_TYPE.customEvent,
});
return rawQuery( return rawQuery(
` `
@ -46,22 +35,17 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite
group by 1, 2 group by 1, 2
order by 2 order by 2
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.customEvent,
domain: website.domain,
},
); );
} }
async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCriteria) { async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters } = criteria; const { timezone = 'utc', unit = 'day' } = filters;
const { rawQuery, getDateQuery, getFilterQuery } = clickhouse; const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
const filterQuery = getFilterQuery(filters); ...filters,
eventType: EVENT_TYPE.customEvent,
});
return rawQuery( return rawQuery(
` `
@ -77,13 +61,6 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite
group by x, t group by x, t
order by t order by t
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.customEvent,
domain: website.domain,
},
); );
} }

67
queries/analytics/pageviews/getPageviewMetrics.ts
View File

@ -2,19 +2,10 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { QueryFilters } from 'lib/types';
import { maxDate } from 'lib/date';
export async function getPageviewMetrics( export async function getPageviewMetrics(
...args: [ ...args: [websiteId: string, columns: string, filters: QueryFilters]
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
column: string;
filters: object;
},
]
) { ) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -22,20 +13,12 @@ export async function getPageviewMetrics(
}); });
} }
async function relationalQuery( async function relationalQuery(websiteId: string, column: string, filters: QueryFilters) {
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
column: string;
filters: object;
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters } = prisma; const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
const { filterQuery, joinSession } = parseFilters(filters); eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -50,31 +33,16 @@ async function relationalQuery(
order by 2 desc order by 2 desc
limit 100 limit 100
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }
async function clickhouseQuery( async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
column: string;
filters: object;
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
const { filterQuery } = parseFilters(filters); eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -88,13 +56,6 @@ async function clickhouseQuery(
order by y desc order by y desc
limit 100 limit 100
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }

65
queries/analytics/pageviews/getPageviewStats.ts
View File

@ -2,43 +2,22 @@ import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { QueryFilters } from 'lib/types';
import { maxDate } from 'lib/date';
export interface PageviewStatsCriteria { export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
filters: {
url?: string;
referrer?: string;
title?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
region?: string;
city?: string;
};
}
export async function getPageviewStats(
...args: [websiteId: string, criteria: PageviewStatsCriteria]
) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) { async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria; const { timezone = 'utc', unit = 'day' } = filters;
const { getDateQuery, parseFilters, rawQuery } = prisma; const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
const { filterQuery, joinSession } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -53,22 +32,17 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
${filterQuery} ${filterQuery}
group by 1 group by 1
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }
async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) { async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria; const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
const { filterQuery } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -88,13 +62,6 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
) as g ) as g
order by t order by t
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }

54
queries/analytics/reports/getInsights.ts
View File

@ -1,19 +1,10 @@
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { maxDate } from 'lib/date';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { QueryFilters } from 'lib/types';
export interface GetInsightsCriteria { export async function getInsights(...args: [websiteId: string, filters: QueryFilters]) {
startDate: Date;
endDate: Date;
fields: { name: string; type: string; value: string }[];
filters: string[];
groups: string[];
}
export async function getInsights(...args: [websiteId: string, criteria: GetInsightsCriteria]) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
@ -22,18 +13,18 @@ export async function getInsights(...args: [websiteId: string, criteria: GetInsi
async function relationalQuery( async function relationalQuery(
websiteId: string, websiteId: string,
criteria: GetInsightsCriteria, filters: QueryFilters,
): Promise< ): Promise<
{ {
x: string; x: string;
y: number; y: number;
}[] }[]
> { > {
const { startDate, endDate, filters = [] } = criteria;
const { parseFilters, rawQuery } = prisma; const { parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
const params = {}; ...filters,
const { filterQuery, joinSession } = parseFilters(params); eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -48,37 +39,30 @@ async function relationalQuery(
${filterQuery} ${filterQuery}
group by 1 group by 1
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
); );
} }
async function clickhouseQuery( async function clickhouseQuery(
websiteId: string, websiteId: string,
criteria: GetInsightsCriteria, filters: QueryFilters,
): Promise< ): Promise<
{ {
x: string; x: string;
y: number; y: number;
}[] }[]
> { > {
const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria;
const { parseFilters, rawQuery } = clickhouse; const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId); const { fields } = filters;
const params = {}; const { filterQuery, params } = await parseFilters(websiteId, {
const { filterQuery } = parseFilters(params); ...filters,
eventType: EVENT_TYPE.pageView,
const fieldsQuery = parseFields(fields); });
return rawQuery( return rawQuery(
` `
select select
${fieldsQuery} ${parseFields(fields)}
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime}
@ -88,13 +72,7 @@ async function clickhouseQuery(
order by total desc order by total desc
limit 500 limit 500
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
); );
} }

48
queries/analytics/sessions/getSessionMetrics.ts
View File

@ -2,14 +2,10 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { QueryFilters } from 'lib/types';
import { maxDate } from 'lib/date';
export async function getSessionMetrics( export async function getSessionMetrics(
...args: [ ...args: [websiteId: string, column: string, filters: QueryFilters]
websiteId: string,
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
]
) { ) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -17,14 +13,12 @@ export async function getSessionMetrics(
}); });
} }
async function relationalQuery( async function relationalQuery(websiteId: string, column: string, filters: QueryFilters) {
websiteId: string,
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const website = await loadWebsite(websiteId);
const { startDate, endDate, column, filters = {} } = criteria;
const { parseFilters, rawQuery } = prisma; const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession } = parseFilters(filters); const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
`select ${column} x, count(*) y `select ${column} x, count(*) y
@ -32,28 +26,22 @@ async function relationalQuery(
${joinSession} ${joinSession}
where website_event.website_id = {{websiteId::uuid}} where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}} and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = {{eventType}}
${filterQuery} ${filterQuery}
) as t ) as t
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100`, limit 100`,
{ params,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
...filters,
},
); );
} }
async function clickhouseQuery( async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
websiteId: string,
data: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const { startDate, endDate, column, filters = {} } = data;
const { parseFilters, rawQuery } = clickhouse; const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
const { filterQuery } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -68,12 +56,6 @@ async function clickhouseQuery(
order by y desc order by y desc
limit 100 limit 100
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
); );
} }

65
queries/analytics/sessions/getSessionStats.ts
View File

@ -2,43 +2,22 @@ import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { QueryFilters } from 'lib/types';
import { maxDate } from 'lib/date';
export interface SessionStatsCriteria { export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
filters: {
url?: string;
referrer?: string;
title?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
region?: string;
city?: string;
};
}
export async function getSessionStats(
...args: [websiteId: string, criteria: SessionStatsCriteria]
) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria) { async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria; const { timezone = 'utc', unit = 'day' } = filters;
const { getDateQuery, parseFilters, rawQuery } = prisma; const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
const { filterQuery, joinSession } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -53,22 +32,17 @@ async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria
${filterQuery} ${filterQuery}
group by 1 group by 1
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }
async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria) { async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria; const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
const { filterQuery } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -88,13 +62,6 @@ async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria
) as g ) as g
order by t order by t
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }

14
queries/analytics/stats/getWebsiteDateRange.ts
View File

@ -1,9 +1,7 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { loadWebsite } from 'lib/load';
import { DEFAULT_RESET_DATE } from 'lib/constants'; import { DEFAULT_RESET_DATE } from 'lib/constants';
import { maxDate } from 'lib/date';
export async function getWebsiteDateRange(...args: [websiteId: string]) { export async function getWebsiteDateRange(...args: [websiteId: string]) {
return runQuery({ return runQuery({
@ -13,8 +11,8 @@ export async function getWebsiteDateRange(...args: [websiteId: string]) {
} }
async function relationalQuery(websiteId: string) { async function relationalQuery(websiteId: string) {
const { rawQuery } = prisma; const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });
const result = await rawQuery( const result = await rawQuery(
` `
@ -25,15 +23,15 @@ async function relationalQuery(websiteId: string) {
where website_id = {{websiteId::uuid}} where website_id = {{websiteId::uuid}}
and created_at >= {{startDate}} and created_at >= {{startDate}}
`, `,
{ websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) }, params,
); );
return result[0] ?? null; return result[0] ?? null;
} }
async function clickhouseQuery(websiteId: string) { async function clickhouseQuery(websiteId: string) {
const { rawQuery } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });
const result = await rawQuery( const result = await rawQuery(
` `
@ -44,7 +42,7 @@ async function clickhouseQuery(websiteId: string) {
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime} and created_at >= {startDate:DateTime}
`, `,
{ websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) }, params,
); );
return result[0] ?? null; return result[0] ?? null;

56
queries/analytics/stats/getWebsiteStats.ts
View File

@ -2,29 +2,21 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { QueryFilters } from 'lib/types';
import { maxDate } from 'lib/date';
export async function getWebsiteStats( export async function getWebsiteStats(...args: [websiteId: string, filters: QueryFilters]) {
...args: [
websiteId: string,
data: { startDate: Date; endDate: Date; type?: string; filters: object },
]
) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
async function relationalQuery( async function relationalQuery(websiteId: string, filters: QueryFilters) {
websiteId: string,
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = criteria;
const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma; const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
const { filterQuery, joinSession } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -43,32 +35,23 @@ async function relationalQuery(
join website join website
on website_event.website_id = website.website_id on website_event.website_id = website.website_id
${joinSession} ${joinSession}
where event_type = {{eventType}} where website.website_id = {{websiteId::uuid}}
and website.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}} and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery} ${filterQuery}
group by 1, 2 group by 1, 2
) as t ) as t
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }
async function clickhouseQuery( async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
websiteId: string,
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = criteria;
const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const { filterQuery, params } = await parseFilters(websiteId, {
const { filterQuery } = parseFilters(filters); ...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery( return rawQuery(
` `
@ -92,13 +75,6 @@ async function clickhouseQuery(
group by session_id, time_series group by session_id, time_series
) as t; ) as t;
`, `,
{ params,
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }