m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2025-02-14 21:10:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactored queries.

Browse Source
This commit is contained in:
Mike Cao 2023-07-24 23:06:16 -07:00
parent e4bd314bd6
commit 4bd3ef8e12
19 changed files with 330 additions and 408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
lib/clickhouse.ts
View File

@ -2,7 +2,6 @@ import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat'; import dateFormat from 'dateformat';
import debug from 'debug'; import debug from 'debug';
import { CLICKHOUSE } from 'lib/db'; import { CLICKHOUSE } from 'lib/db';
import { getDynamicDataType } from './dynamicData';
import { WebsiteMetricFilter } from './types'; import { WebsiteMetricFilter } from './types';
import { FILTER_COLUMNS } from './constants'; import { FILTER_COLUMNS } from './constants';
@ -62,45 +61,6 @@ function getDateFormat(date) {
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
} }
function getEventDataFilterQuery(
filters: {
eventKey?: string;
eventValue?: string | number | boolean | Date;
}[] = [],
params: any,
) {
const query = filters.reduce((ac, cv, i) => {
const type = getDynamicDataType(cv.eventValue);
let value = cv.eventValue;
ac.push(`and (event_key = {eventKey${i}:String}`);
switch (type) {
case 'number':
ac.push(`and number_value = {eventValue${i}:UInt64})`);
break;
case 'string':
ac.push(`and string_value = {eventValue${i}:String})`);
break;
case 'boolean':
ac.push(`and string_value = {eventValue${i}:String})`);
value = cv ? 'true' : 'false';
break;
case 'date':
ac.push(`and date_value = {eventValue${i}:DateTime('UTC')})`);
break;
}
params[`eventKey${i}`] = cv.eventKey;
params[`eventValue${i}`] = value;
return ac;
}, []);
return query.join('\n');
}
function getFilterQuery(filters = {}, params = {}) { function getFilterQuery(filters = {}, params = {}) {
const query = Object.keys(filters).reduce((arr, key) => { const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key]; const filter = filters[key];
@ -146,22 +106,7 @@ function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
}; };
} }
function formatField(field, type, value) { async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
switch (type) {
case 'date':
return getDateFormat(value);
default:
return field;
}
}
async function rawQuery<T>(sql, params = {}): Promise<T> {
const query = sql.replaceAll(/\{\{\w+:\w+}}/g, token => {
const [, field, type] = token.match(/\{\{(\w+):(\w+)}}/);
return formatField(field, type, params[field]);
});
if (process.env.LOG_QUERY) { if (process.env.LOG_QUERY) {
log('QUERY:\n', query); log('QUERY:\n', query);
log('PARAMETERS:\n', params); log('PARAMETERS:\n', params);
@ -202,7 +147,6 @@ export default {
getDateFormat, getDateFormat,
getFilterQuery, getFilterQuery,
getFunnelQuery, getFunnelQuery,
getEventDataFilterQuery,
parseFilters, parseFilters,
findUnique, findUnique,
findFirst, findFirst,

4
lib/db.js
View File

@ -35,3 +35,7 @@ export async function runQuery(queries) {
return queries[CLICKHOUSE](); return queries[CLICKHOUSE]();
} }
} }
export function notImplemented() {
throw new Error('Not implemented.');
}

2
lib/kafka.ts
View File

@ -61,7 +61,7 @@ async function getProducer(): Promise<Producer> {
return producer; return producer;
} }
function getDateFormat(date, format?): string { function getDateFormat(date: Date, format?: string): string {
return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss'); return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');
} }

83
lib/prisma.ts
View File

@ -1,7 +1,6 @@
import prisma from '@umami/prisma-client'; import prisma from '@umami/prisma-client';
import moment from 'moment-timezone'; import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { getDynamicDataType } from './dynamicData';
import { FILTER_COLUMNS } from './constants'; import { FILTER_COLUMNS } from './constants';
const MYSQL_DATE_FORMATS = { const MYSQL_DATE_FORMATS = {
@ -20,20 +19,8 @@ const POSTGRESQL_DATE_FORMATS = {
year: 'YYYY-01-01', year: 'YYYY-01-01',
}; };
function toUuid(): string {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
return '::uuid';
}
if (db === MYSQL) {
return '';
}
}
function getAddMinutesQuery(field: string, minutes: number) { function getAddMinutesQuery(field: string, minutes: number) {
const db = getDatabaseType(process.env.DATABASE_URL); const db = getDatabaseType();
if (db === POSTGRESQL) { if (db === POSTGRESQL) {
return `${field} + interval '${minutes} minute'`; return `${field} + interval '${minutes} minute'`;
@ -45,7 +32,7 @@ function getAddMinutesQuery(field: string, minutes: number) {
} }
function getDateQuery(field: string, unit: string, timezone?: string): string { function getDateQuery(field: string, unit: string, timezone?: string): string {
const db = getDatabaseType(process.env.DATABASE_URL); const db = getDatabaseType();
if (db === POSTGRESQL) { if (db === POSTGRESQL) {
if (timezone) { if (timezone) {
@ -65,8 +52,8 @@ function getDateQuery(field: string, unit: string, timezone?: string): string {
} }
} }
function getTimestampInterval(field: string): string { function getTimestampIntervalQuery(field: string): string {
const db = getDatabaseType(process.env.DATABASE_URL); const db = getDatabaseType();
if (db === POSTGRESQL) { if (db === POSTGRESQL) {
return `floor(extract(epoch from max(${field}) - min(${field})))`; return `floor(extract(epoch from max(${field}) - min(${field})))`;
@ -77,47 +64,6 @@ function getTimestampInterval(field: string): string {
} }
} }
function getEventDataFilterQuery(
filters: {
eventKey?: string;
eventValue?: string | number | boolean | Date;
}[],
params: any[],
) {
const query = filters.reduce((ac, cv) => {
const type = getDynamicDataType(cv.eventValue);
let value = cv.eventValue;
ac.push(`and (event_key = $${params.length + 1}`);
params.push(cv.eventKey);
switch (type) {
case 'number':
ac.push(`and number_value = $${params.length + 1})`);
params.push(value);
break;
case 'string':
ac.push(`and string_value = $${params.length + 1})`);
params.push(decodeURIComponent(cv.eventValue as string));
break;
case 'boolean':
ac.push(`and string_value = $${params.length + 1})`);
params.push(decodeURIComponent(cv.eventValue as string));
value = cv ? 'true' : 'false';
break;
case 'date':
ac.push(`and date_value = $${params.length + 1})`);
params.push(cv.eventValue);
break;
}
return ac;
}, []);
return query.join('\n');
}
function getFilterQuery(filters = {}, params = []): string { function getFilterQuery(filters = {}, params = []): string {
const query = Object.keys(filters).reduce((arr, key) => { const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key]; const filter = filters[key];
@ -163,7 +109,7 @@ function getFunnelQuery(
and l0.referrer_path = $${i + initParamLength} and l0.referrer_path = $${i + initParamLength}
and l0.url_path = $${levelNumber + initParamLength} and l0.url_path = $${levelNumber + initParamLength}
and created_at between $2 and $3 and created_at between $2 and $3
and website_id = $1${toUuid()} and website_id = $1
)`; )`;
} }
@ -197,27 +143,32 @@ function parseFilters(
}; };
} }
async function rawQuery(query: string, params: never[] = []): Promise<any> { async function rawQuery(sql: string, data: object): Promise<any> {
const db = getDatabaseType(process.env.DATABASE_URL); const db = getDatabaseType();
const params = [];
if (db !== POSTGRESQL && db !== MYSQL) { if (db !== POSTGRESQL && db !== MYSQL) {
return Promise.reject(new Error('Unknown database.')); return Promise.reject(new Error('Unknown database.'));
} }
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query; const query = sql?.replaceAll(/\{\{(\w+)(::\w+)?}}/g, (...args) => {
const [, name, type] = args;
return prisma.rawQuery(sql, params); params.push(data[name]);
return db === MYSQL ? '?' : `$${params.length}${type ?? ''}`;
});
return prisma.rawQuery(query, params);
} }
export default { export default {
...prisma, ...prisma,
getAddMinutesQuery, getAddMinutesQuery,
getDateQuery, getDateQuery,
getTimestampInterval, getTimestampIntervalQuery,
getFilterQuery, getFilterQuery,
getFunnelQuery, getFunnelQuery,
getEventDataFilterQuery,
toUuid,
parseFilters, parseFilters,
rawQuery, rawQuery,
}; };

9
lib/sql.ts
View File

@ -1,9 +0,0 @@
export function buildSql(query: string, parameters: object) {
const params = { ...parameters };
const sql = query.replaceAll(/\$[\w_]+/g, name => {
return name;
});
return { sql, params };
}

115
queries/analytics/eventData/getEventDataEvents.ts
View File

@ -1,4 +1,3 @@
import { buildSql } from 'lib/sql';
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
@ -26,47 +25,52 @@ async function relationalQuery(
endDate: Date, endDate: Date,
filters: { field?: string; event?: string }, filters: { field?: string; event?: string },
) { ) {
const { toUuid, rawQuery } = prisma; const { rawQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { field, event } = filters; const { field, event } = filters;
if (field) { if (event) {
if (event) {
return rawQuery(
`select ed.event_key as field,
ed.string_value as value,
count(ed.*) as total
from event_data as ed
inner join website_event as we
on we.event_id = ed.website_event_id
where ed.website_id = $1${toUuid()}
and ed.event_key = $2
and ed.created_at >= $3
and ed.created_at between $4 and $5
and we.event_name = $6
group by ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
[websiteId, field, resetDate, startDate, endDate, event] as any,
);
}
return rawQuery( return rawQuery(
`select event_key as field, `
string_value as value, select
count(*) as total we.event_name as event,
from event_data ed.event_key as field,
where website_id = $1${toUuid()} ed.string_value as value,
and event_key = $2 count(ed.*) as total
and created_at >= $3 from event_data as ed
and created_at between $4 and $5 inner join website_event as we
group by event_key, string_value on we.event_id = ed.website_event_id
order by 3 desc, 2 desc, 1 asc where ed.website_id = {{websiteId:uuid}}
and ed.event_key = {{field}}
and ed.created_at >= {{resetDate}}
and ed.created_at between {{startDate}} and {{endDate}}
and we.event_name = {{event}}
group by ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`, `,
[websiteId, field, resetDate, startDate, endDate] as any, { ...filters, websiteId, resetDate, startDate, endDate },
); );
} }
return rawQuery(
`
select
we.event_name as event,
ed.event_key as field,
ed.string_value as value,
count(ed.*) as total
from event_data as ed
inner join website_event as we
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.event_key = {{field}}
and ed.created_at >= {{resetDate}}
and ed.created_at between {{startDate}} and {{endDate}}
group by we.event_name, ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
{ websiteId, field, resetDate, startDate, endDate },
);
} }
async function clickhouseQuery( async function clickhouseQuery(
@ -82,12 +86,13 @@ async function clickhouseQuery(
if (event) { if (event) {
return rawQuery( return rawQuery(
`select `
event_name as event, select
event_key as field, event_name as event,
data_type as type, event_key as field,
string_value as value, data_type as type,
count(*) as total string_value as value,
count(*) as total
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime} and created_at >= {resetDate:DateTime}
@ -95,24 +100,26 @@ async function clickhouseQuery(
and event_name = {event:String} and event_name = {event:String}
group by event_key, data_type, string_value, event_name group by event_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 4 desc order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100`, limit 100
{ websiteId, resetDate, startDate, endDate, event }, `,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }
return rawQuery( return rawQuery(
`select `
event_name as event, select
event_key as field, event_name as event,
data_type as type, event_key as field,
count(*) as total data_type as type,
from event_data count(*) as total
where website_id = {websiteId:UUID} from event_data
and created_at >= {resetDate:DateTime} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at >= {resetDate:DateTime}
group by event_key, data_type, event_name and created_at between {startDate:DateTime} and {endDate:DateTime}
order by 1 asc, 2 asc group by event_key, data_type, event_name
limit 100 order by 1 asc, 2 asc
limit 100
`, `,
{ websiteId, resetDate, startDate, endDate }, { websiteId, resetDate, startDate, endDate },
); );

103
queries/analytics/eventData/getEventDataFields.ts
View File

@ -15,82 +15,87 @@ export async function getEventDataFields(
} }
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) { async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { toUuid, rawQuery } = prisma; const { rawQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
if (field) { if (field) {
return rawQuery( return rawQuery(
`select event_key as field, `
string_value as value, select
count(*) as total event_key as field,
from event_data string_value as value,
where website_id = $1${toUuid()} count(*) as total
and event_key = $2 from event_data
and created_at >= $3 where website_id = {{websiteId::uuid}}
and created_at between $4 and $5 and event_key = {{field}}
group by event_key, string_value and created_at >= {{resetDate}}
order by 3 desc, 2 desc, 1 asc and created_at between {{startDate}} and {{endDate}}
limit 100 group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`, `,
[websiteId, field, resetDate, startDate, endDate] as any, { websiteId, field, resetDate, startDate, endDate },
); );
} }
return rawQuery( return rawQuery(
`select `
event_key as field, select
data_type as type, event_key as field,
count(*) as total data_type as type,
from event_data count(*) as total
where website_id = $1${toUuid()} from event_data
and created_at >= $2 where website_id = {{websiteId::uuid}}
and created_at between $3 and $4 and created_at >= {{resetDate}}
group by event_key, data_type and created_at between {{startDate}} and {{endDate}}
order by 3 desc, 2 asc, 1 asc group by event_key, data_type
limit 100 order by 3 desc, 2 asc, 1 asc
limit 100
`, `,
[websiteId, resetDate, startDate, endDate] as any, { websiteId, resetDate, startDate, endDate },
); );
} }
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) { async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery, getDateFormat } = clickhouse; const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
if (field) { if (field) {
return rawQuery( return rawQuery(
`select `
select
event_key as field, event_key as field,
string_value as value, string_value as value,
count(*) as total count(*) as total
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and event_key = {field:String} and event_key = {field:String}
and created_at >= ${getDateFormat(resetDate)} and created_at >= {resetDate:DateTime}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)} and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, string_value group by event_key, string_value
order by 3 desc, 2 desc, 1 asc order by 3 desc, 2 desc, 1 asc
limit 100 limit 100
`, `,
{ websiteId, field }, { websiteId, field, resetDate, startDate, endDate },
); );
} }
return rawQuery( return rawQuery(
`select `
event_key as field, select
data_type as type, event_key as field,
count(*) as total data_type as type,
from event_data count(*) as total
where website_id = {websiteId:UUID} from event_data
and created_at >= ${getDateFormat(resetDate)} where website_id = {websiteId:UUID}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)} and created_at >= {resetDate:DateTime}
group by event_key, data_type and created_at between {startDate:DateTime} and {endDate:DateTime}
order by 3 desc, 2 asc, 1 asc group by event_key, data_type
limit 100 order by 3 desc, 2 asc, 1 asc
limit 100
`, `,
{ websiteId }, { websiteId, resetDate, startDate, endDate },
); );
} }

20
queries/analytics/eventData/getEventDataUsage.ts
View File

@ -1,28 +1,26 @@
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';
export function getEventDataUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) { export function getEventDataUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: notImplemented,
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
function relationalQuery(websiteIds: string[], startDate: Date, endDate: Date) {
throw new Error('Not implemented.');
}
function clickhouseQuery(websiteIds: string[], startDate: Date, endDate: Date) { function clickhouseQuery(websiteIds: string[], startDate: Date, endDate: Date) {
const { rawQuery } = clickhouse; const { rawQuery } = clickhouse;
return rawQuery( return rawQuery(
`select `
website_id as websiteId, select
count(*) as count website_id as websiteId,
count(*) as count
from event_data from event_data
where created_at between {startDate:DateTime64} and {endDate:DateTime64} where created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_id in {websiteIds:Array(UUID)} and website_id in {websiteIds:Array(UUID)}
group by website_id`, group by website_id
`,
{ {
websiteIds, websiteIds,
startDate, startDate,

2
queries/analytics/eventData/saveEventData.ts
View File

@ -31,7 +31,7 @@ async function relationalQuery(data: {
const jsonKeys = flattenJSON(eventData); const jsonKeys = flattenJSON(eventData);
//id, websiteEventId, eventStringValue // id, websiteEventId, eventStringValue
const flattendData = jsonKeys.map(a => ({ const flattendData = jsonKeys.map(a => ({
id: uuid(), id: uuid(),
websiteEventId: eventId, websiteEventId: eventId,

37
queries/analytics/events/getEventMetrics.ts
View File

@ -45,26 +45,27 @@ async function relationalQuery(
}; };
}, },
) { ) {
const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma; const { rawQuery, getDateQuery, getFilterQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params: any = [websiteId, resetDate, startDate, endDate]; const filterQuery = getFilterQuery(filters);
const filterQuery = getFilterQuery(filters, params);
return rawQuery( return rawQuery(
`select `
select
event_name x, event_name x,
${getDateQuery('created_at', unit, timezone)} t, ${getDateQuery('created_at', unit, timezone)} t,
count(*) y count(*) y
from website_event from website_event
where website_id = $1${toUuid()} where website_id = {{websiteId::uuid}}
and created_at >= $2 and created_at >= {{resetDate}}
and created_at between $3 and $4 and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent} and event_type = ${EVENT_TYPE.customEvent}
${filterQuery} ${filterQuery}
group by 1, 2 group by 1, 2
order by 2`, order by 2
params, `,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }
@ -87,24 +88,26 @@ async function clickhouseQuery(
}; };
}, },
) { ) {
const { rawQuery, getDateQuery, getDateFormat, getFilterQuery } = clickhouse; const { rawQuery, getDateQuery, getFilterQuery } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params = { websiteId }; const filterQuery = getFilterQuery(filters);
return rawQuery( return rawQuery(
`select `
select
event_name x, event_name x,
${getDateQuery('created_at', unit, timezone)} t, ${getDateQuery('created_at', unit, timezone)} t,
count(*) y count(*) y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and event_type = ${EVENT_TYPE.customEvent} and event_type = ${EVENT_TYPE.customEvent}
and created_at >= ${getDateFormat(resetDate)} and created_at >= {resetDate:DateTIme}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)} and created_at between {startDate:DateTime} and {endDate:DateTime}
${getFilterQuery(filters, params)} ${filterQuery}
group by x, t group by x, t
order by t`, order by t
params, `,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }

22
queries/analytics/events/getEventUsage.ts
View File

@ -1,28 +1,26 @@
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';
export function getEventUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) { export function getEventUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: notImplemented,
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
function relationalQuery(websiteIds: string[], startDate: Date, endDate: Date) {
throw new Error('Not implemented.');
}
function clickhouseQuery(websiteIds: string[], startDate: Date, endDate: Date) { function clickhouseQuery(websiteIds: string[], startDate: Date, endDate: Date) {
const { rawQuery } = clickhouse; const { rawQuery } = clickhouse;
return rawQuery( return rawQuery(
`select `
website_id as websiteId, select
count(*) as count website_id as websiteId,
count(*) as count
from website_event from website_event
where created_at between {startDate:DateTime64} and {endDate:DateTime64} where website_id in {websiteIds:Array(UUID)}
and website_id in {websiteIds:Array(UUID)} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by website_id`, group by website_id
`,
{ {
websiteIds, websiteIds,
startDate, startDate,

6
queries/analytics/events/getEvents.ts
View File

@ -25,7 +25,8 @@ function clickhouseQuery(websiteId: string, startAt: Date, eventType: number) {
const { rawQuery } = clickhouse; const { rawQuery } = clickhouse;
return rawQuery( return rawQuery(
`select `
select
event_id as id, event_id as id,
website_id as websiteId, website_id as websiteId,
session_id as sessionId, session_id as sessionId,
@ -37,7 +38,8 @@ function clickhouseQuery(websiteId: string, startAt: Date, eventType: number) {
from website_event from website_event
where event_type = {eventType:UInt32} where event_type = {eventType:UInt32}
and website_id = {websiteId:UUID} and website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime('UTC')}`, and created_at >= {startAt:DateTime('UTC')}
`,
{ {
websiteId, websiteId,
startAt, startAt,

44
queries/analytics/pageviews/getPageviewMetrics.ts
View File

@ -31,40 +31,43 @@ async function relationalQuery(
}, },
) { ) {
const { startDate, endDate, filters = {}, column } = criteria; const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters, toUuid } = prisma; const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params: any = [ const params: any = {
websiteId, websiteId,
resetDate, resetDate,
startDate, startDate,
endDate, endDate,
column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
]; };
let excludeDomain = ''; let excludeDomain = '';
if (column === 'referrer_domain') { if (column === 'referrer_domain') {
excludeDomain = excludeDomain =
'and (website_event.referrer_domain != $6 or website_event.referrer_domain is null)'; 'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)';
params.push(website.domain);
params.domain = website.domain;
} }
const { filterQuery, joinSession } = parseFilters(filters, params); const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery( return rawQuery(
`select ${column} x, count(*) y `
select ${column} x, count(*) y
from website_event from website_event
${joinSession} ${joinSession}
where website_event.website_id = $1${toUuid()} where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at >= $2 and website_event.created_at >= {{resetDate}}
and website_event.created_at between $3 and $4 and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = $5 and event_type = {{eventType}}
${excludeDomain} ${excludeDomain}
${filterQuery} ${filterQuery}
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100`, limit 100
`,
params, params,
); );
} }
@ -79,11 +82,14 @@ async function clickhouseQuery(
}, },
) { ) {
const { startDate, endDate, filters = {}, column } = criteria; const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, getDateFormat, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params = { const params = {
websiteId, websiteId,
resetDate,
startDate,
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: undefined, domain: undefined,
}; };
@ -98,17 +104,19 @@ async function clickhouseQuery(
const { filterQuery } = parseFilters(filters, params); const { filterQuery } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select ${column} x, count(*) y `
select ${column} x, count(*) y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
and created_at >= ${getDateFormat(resetDate)}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)}
${excludeDomain} ${excludeDomain}
${filterQuery} ${filterQuery}
group by x group by x
order by y desc order by y desc
limit 100`, limit 100
`,
params, params,
); );
} }

58
queries/analytics/pageviews/getPageviewStats.ts
View File

@ -45,24 +45,26 @@ async function relationalQuery(
filters = {}, filters = {},
sessionKey = 'session_id', sessionKey = 'session_id',
} = criteria; } = criteria;
const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma; const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select ${getDateQuery('website_event.created_at', unit, timezone)} x, `
count(${count !== '*' ? `${count}${sessionKey}` : count}) y select
from website_event ${getDateQuery('website_event.created_at', unit, timezone)} x,
${joinSession} count(${count !== '*' ? `${count}${sessionKey}` : count}) y
where website_event.website_id = $1${toUuid()} from website_event
and website_event.created_at >= $2 ${joinSession}
and website_event.created_at between $3 and $4 where website_event.website_id = {{websiteId::uuid}}
and event_type = ${EVENT_TYPE.pageView} and website_event.created_at >= {{resetDate}}
${filterQuery} and website_event.created_at between {{startDate}} and {{endDate}}
group by 1`, and event_type = ${EVENT_TYPE.pageView}
params, ${filterQuery}
group by 1
`,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }
@ -86,28 +88,30 @@ async function clickhouseQuery(
count = '*', count = '*',
filters = {}, filters = {},
} = criteria; } = criteria;
const { parseFilters, getDateFormat, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params = { websiteId }; const { filterQuery } = parseFilters(filters);
const { filterQuery } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select `
select
${getDateStringQuery('g.t', unit)} as x, ${getDateStringQuery('g.t', unit)} as x,
g.y as y g.y as y
from from (
(select select
${getDateQuery('created_at', unit, timezone)} t, ${getDateQuery('created_at', unit, timezone)} as t,
count(${count !== '*' ? 'distinct session_id' : count}) y count(${count !== '*' ? 'distinct session_id' : count}) as y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = ${EVENT_TYPE.pageView} and event_type = ${EVENT_TYPE.pageView}
and created_at >= ${getDateFormat(resetDate)}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)}
${filterQuery} ${filterQuery}
group by t) g group by t
order by t`, ) as g
params, order by t
`,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }

10
queries/analytics/reports/getFunnel.ts
View File

@ -34,18 +34,16 @@ async function relationalQuery(
}[] }[]
> { > {
const { windowMinutes, startDate, endDate, urls } = criteria; const { windowMinutes, startDate, endDate, urls } = criteria;
const { rawQuery, getFunnelQuery, toUuid } = prisma; const { rawQuery, getFunnelQuery } = prisma;
const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes); const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes);
const params: any = [websiteId, startDate, endDate, ...urls];
return rawQuery( return rawQuery(
`WITH level0 AS ( `WITH level0 AS (
select distinct session_id, url_path, referrer_path, created_at select distinct session_id, url_path, referrer_path, created_at
from website_event from website_event
where url_path in (${urlFilterQuery}) where url_path in (${urlFilterQuery})
and website_id = $1${toUuid()} and website_id = {{websiteId::uuid}}
and created_at between $2 and $3 and created_at between {{startDate}} and {{endDate}}
),level1 AS ( ),level1 AS (
select distinct session_id, url_path as level_1_url, created_at as level_1_created_at select distinct session_id, url_path as level_1_url, created_at as level_1_created_at
from level0 from level0
@ -55,7 +53,7 @@ async function relationalQuery(
SELECT ${sumQuery} SELECT ${sumQuery}
from level${urls.length}; from level${urls.length};
`, `,
params, { websiteId, startDate, endDate, ...urls },
).then((a: { [key: string]: number }) => { ).then((a: { [key: string]: number }) => {
return urls.map((b, i) => ({ x: b, y: a[0][`level${i + 1}`] || 0 })); return urls.map((b, i) => ({ x: b, y: a[0][`level${i + 1}`] || 0 }));
}); });

33
queries/analytics/sessions/getSessionMetrics.ts
View File

@ -23,9 +23,8 @@ async function relationalQuery(
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { startDate, endDate, column, filters = {} } = criteria; const { startDate, endDate, column, filters = {} } = criteria;
const { toUuid, parseFilters, rawQuery } = prisma; const { parseFilters, rawQuery } = prisma;
const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select ${column} x, count(*) y `select ${column} x, count(*) y
@ -36,15 +35,15 @@ async function relationalQuery(
join website join website
on website_event.website_id = website.website_id on website_event.website_id = website.website_id
${joinSession} ${joinSession}
where website.website_id = $1${toUuid()} where website.website_id = {{websiteId::uuid}}
and website_event.created_at >= $2 and website_event.created_at >= {{resetDate}}
and website_event.created_at between $3 and $4 and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery} ${filterQuery}
) )
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100`, limit 100`,
params, { ...filters, websiteId, resetDate, startDate, endDate },
); );
} }
@ -53,23 +52,25 @@ async function clickhouseQuery(
data: { startDate: Date; endDate: Date; column: string; filters: object }, data: { startDate: Date; endDate: Date; column: string; filters: object },
) { ) {
const { startDate, endDate, column, filters = {} } = data; const { startDate, endDate, column, filters = {} } = data;
const { getDateFormat, parseFilters, rawQuery } = clickhouse; const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params = { websiteId }; const { filterQuery } = parseFilters(filters);
const { filterQuery } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select ${column} x, count(distinct session_id) y `
select
${column} x, count(distinct session_id) y
from website_event as x from website_event as x
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and event_type = ${EVENT_TYPE.pageView} and created_at >= {resetDate:DateTime}
and created_at >= ${getDateFormat(resetDate)} and created_at between {startDate:DateTime} and {endDate:DateTime}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)} and event_type = ${EVENT_TYPE.pageView}
${filterQuery} ${filterQuery}
group by x group by x
order by y desc order by y desc
limit 100`, limit 100
params, `,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }

6
queries/analytics/sessions/getSessions.ts
View File

@ -24,7 +24,8 @@ async function clickhouseQuery(websiteId: string, startAt: Date) {
const { rawQuery } = clickhouse; const { rawQuery } = clickhouse;
return rawQuery( return rawQuery(
`select distinct `
select distinct
session_id as id, session_id as id,
website_id as websiteId, website_id as websiteId,
created_at as createdAt, created_at as createdAt,
@ -41,7 +42,8 @@ async function clickhouseQuery(websiteId: string, startAt: Date) {
city city
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime('UTC')}`, and created_at >= {startAt:DateTime}
`,
{ {
websiteId, websiteId,
startAt, startAt,

27
queries/analytics/stats/getActiveVisitors.ts
View File

@ -11,31 +11,32 @@ export async function getActiveVisitors(...args: [websiteId: string]) {
} }
async function relationalQuery(websiteId: string) { async function relationalQuery(websiteId: string) {
const { toUuid, rawQuery } = prisma; const { rawQuery } = prisma;
const date = subMinutes(new Date(), 5);
const params: any = [websiteId, date];
return rawQuery( return rawQuery(
`select count(distinct session_id) x `
select count(distinct session_id) x
from website_event from website_event
join website join website
on website_event.website_id = website.website_id on website_event.website_id = website.website_id
where website.website_id = $1${toUuid()} where website.website_id = {{websiteId::uuid}}
and website_event.created_at >= $2`, and website_event.created_at >= {{startAt}}
params, `,
{ websiteId, startAt: subMinutes(new Date(), 5) },
); );
} }
async function clickhouseQuery(websiteId: string) { async function clickhouseQuery(websiteId: string) {
const { rawQuery } = clickhouse; const { rawQuery } = clickhouse;
const params = { websiteId, startAt: subMinutes(new Date(), 5) };
return rawQuery( return rawQuery(
`select count(distinct session_id) x `
select
count(distinct session_id) x
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime('UTC')}`, and created_at >= {startAt:DateTime}
params, `,
{ websiteId, startAt: subMinutes(new Date(), 5) },
); );
} }

99
queries/analytics/stats/getWebsiteStats.ts
View File

@ -21,34 +21,37 @@ async function relationalQuery(
criteria: { startDate: Date; endDate: Date; filters: object }, criteria: { startDate: Date; endDate: Date; filters: object },
) { ) {
const { startDate, endDate, filters = {} } = criteria; const { startDate, endDate, filters = {} } = criteria;
const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select sum(t.c) as "pageviews", `
count(distinct t.session_id) as "uniques", select
sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(t.c) as "pageviews",
sum(t.time) as "totaltime" count(distinct t.session_id) as "uniques",
from ( sum(case when t.c = 1 then 1 else 0 end) as "bounces",
select website_event.session_id, sum(t.time) as "totaltime"
${getDateQuery('website_event.created_at', 'hour')}, from (
count(*) c, select
${getTimestampInterval('website_event.created_at')} as "time" website_event.session_id,
from website_event ${getDateQuery('website_event.created_at', 'hour')},
join website count(*) as c,
on website_event.website_id = website.website_id ${getTimestampIntervalQuery('website_event.created_at')} as "time"
${joinSession} from website_event
where event_type = ${EVENT_TYPE.pageView} join website
and website.website_id = $1${toUuid()} on website_event.website_id = website.website_id
and website_event.created_at >= $2 ${joinSession}
and website_event.created_at between $3 and $4 where event_type = ${EVENT_TYPE.pageView}
${filterQuery} and website.website_id = {{websiteId::uuid}}
group by 1, 2 and website_event.created_at >= {{resetDate}}
) t`, and website_event.created_at between {{startDate}} and {{endDate}}
params, ${filterQuery}
group by 1, 2
) as t
`,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }
@ -57,32 +60,34 @@ async function clickhouseQuery(
criteria: { startDate: Date; endDate: Date; filters: object }, criteria: { startDate: Date; endDate: Date; filters: object },
) { ) {
const { startDate, endDate, filters = {} } = criteria; const { startDate, endDate, filters = {} } = criteria;
const { rawQuery, getDateFormat, getDateQuery, parseFilters } = clickhouse; const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE); const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params = { websiteId }; const { filterQuery } = parseFilters(filters);
const { filterQuery } = parseFilters(filters, params);
return rawQuery( return rawQuery(
`select `
sum(t.c) as "pageviews", select
count(distinct t.session_id) as "uniques", sum(t.c) as "pageviews",
sum(if(t.c = 1, 1, 0)) as "bounces", count(distinct t.session_id) as "uniques",
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" sum(if(t.c = 1, 1, 0)) as "bounces",
from ( sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
select session_id, from (
${getDateQuery('created_at', 'day')} time_series, select
count(*) c, session_id,
min(created_at) min_time, ${getDateQuery('created_at', 'day')} time_series,
max(created_at) max_time count(*) c,
from website_event min(created_at) min_time,
where event_type = ${EVENT_TYPE.pageView} max(created_at) max_time
and website_id = {websiteId:UUID} from website_event
and created_at >= ${getDateFormat(resetDate)} where website_id = {websiteId:UUID}
and created_at between ${getDateFormat(startDate)} and ${getDateFormat(endDate)} and created_at >= {resetDate:DateTime}
${filterQuery} and created_at between {startDate:DateTime} and {endDate:DateTime}
group by session_id, time_series and event_type = ${EVENT_TYPE.pageView}
) t;`, ${filterQuery}
params, group by session_id, time_series
) as t;
`,
{ ...filters, websiteId, resetDate, startDate, endDate },
); );
} }