m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2024-11-16 02:05:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/umami-software/umami into feat/um-376-retention-report

Browse Source
This commit is contained in:
Francis Cao 2023-08-04 13:18:58 -07:00
commit 74b5b140ae
17 changed files with 260 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/clickhouse.ts
View File

@ -2,8 +2,10 @@ import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import debug from 'debug';
import { CLICKHOUSE } from 'lib/db';
import { WebsiteMetricFilter } from './types';
import { FILTER_COLUMNS } from './constants';
import { QueryFilters } from './types';
import { FILTER_COLUMNS, IGNORED_FILTERS } from './constants';
import { loadWebsite } from './load';
import { maxDate } from './date';
export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00',
@ -65,13 +67,13 @@ function getFilterQuery(filters = {}) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter !== undefined) {
if (filter !== undefined && !IGNORED_FILTERS.includes(key)) {
const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column} = {${key}:String}`);
}
if (key === 'referrer') {
arr.push('and referrer_domain != {domain:String}');
arr.push('and referrer_domain != {websiteDomain:String}');
}
return arr;
@ -80,9 +82,20 @@ function getFilterQuery(filters = {}) {
return query.join('\n');
}
function parseFilters(filters: WebsiteMetricFilter = {}) {
async function parseFilters(
websiteId: string,
filters: QueryFilters & { [key: string]: any } = {},
) {
const website = await loadWebsite(websiteId);
return {
filterQuery: getFilterQuery(filters),
params: {
...filters,
websiteId,
startDate: maxDate(filters.startDate, website.resetAt),
websiteDomain: website.domain,
},
};
}

3
lib/constants.ts
View File

@ -50,8 +50,11 @@ export const FILTER_COLUMNS = {
query: 'url_query',
event: 'event_name',
region: 'subdivision1',
type: 'event_type',
};
export const IGNORED_FILTERS = ['startDate', 'endDate', 'timezone', 'unit', 'eventType'];
export const COLLECTION_TYPE = {
event: 'event',
identify: 'identify',

19
lib/prisma.ts
View File

@ -1,7 +1,10 @@
import prisma from '@umami/prisma-client';
import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants';
import { FILTER_COLUMNS, IGNORED_FILTERS, SESSION_COLUMNS } from './constants';
import { loadWebsite } from './load';
import { maxDate } from './date';
import { QueryFilters } from './types';
const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',
@ -68,14 +71,14 @@ function getFilterQuery(filters = {}): string {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter !== undefined) {
if (filter !== undefined && !IGNORED_FILTERS.includes(key)) {
const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column}={{${key}}}`);
}
if (key === 'referrer') {
arr.push(
'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)',
'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',
);
}
@ -85,12 +88,20 @@ function getFilterQuery(filters = {}): string {
return query.join('\n');
}
function parseFilters(filters: { [key: string]: any } = {}) {
async function parseFilters(websiteId, filters: QueryFilters & { [key: string]: any } = {}) {
const website = await loadWebsite(websiteId);
return {
joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key])
? `inner join session on website_event.session_id = session.session_id`
: '',
filterQuery: getFilterQuery(filters),
params: {
...filters,
websiteId,
startDate: maxDate(filters.startDate, website.resetAt),
websiteDomain: website.domain,
},
};
}

36
lib/types.ts
View File

@ -73,21 +73,6 @@ export interface WebsiteMetric {
y: number;
}
export interface WebsiteMetricFilter {
domain?: string;
url?: string;
referrer?: string;
title?: string;
query?: string;
event?: string;
os?: string;
browser?: string;
device?: string;
country?: string;
region?: string;
city?: string;
}
export interface WebsiteEventMetric {
x: string;
t: string;
@ -144,3 +129,24 @@ export interface DateRange {
unit: string;
value: string;
}
export interface QueryFilters {
startDate?: Date;
endDate?: Date;
timezone?: string;
unit?: string;
domain?: string;
eventType?: number;
url?: string;
referrer?: string;
title?: string;
query?: string;
event?: string;
os?: string;
browser?: string;
device?: string;
country?: string;
region?: string;
city?: string;
language?: string;
}

17
pages/api/websites/[id]/metrics.ts
View File

@ -23,6 +23,7 @@ export interface WebsiteMetricsRequestQuery {
country: string;
region: string;
city: string;
language: string;
}
export default async (
@ -57,6 +58,8 @@ export default async (
const { startDate, endDate } = await parseDateRangeQuery(req);
const filters = {
startDate,
endDate,
url,
referrer,
title,
@ -76,12 +79,7 @@ export default async (
const column = FILTER_COLUMNS[type] || type;
if (SESSION_COLUMNS.includes(type)) {
const data = await getSessionMetrics(websiteId, {
startDate,
endDate,
column,
filters,
});
const data = await getSessionMetrics(websiteId, column, filters);
if (type === 'language') {
const combined = {};
@ -103,12 +101,7 @@ export default async (
}
if (EVENT_COLUMNS.includes(type)) {
const data = await getPageviewMetrics(websiteId, {
startDate,
endDate,
column,
filters,
});
const data = await getPageviewMetrics(websiteId, column, filters);
return ok(res, data);
}

52
pages/api/websites/[id]/pageviews.ts
View File

@ -57,41 +57,25 @@ export default async (
return badRequest(res);
}
const filters = {
startDate,
endDate,
timezone,
unit,
url,
referrer,
title,
os,
browser,
device,
country,
region,
city,
};
const [pageviews, sessions] = await Promise.all([
getPageviewStats(websiteId, {
startDate,
endDate,
timezone,
unit,
filters: {
url,
referrer,
title,
os,
browser,
device,
country,
region,
city,
},
}),
getSessionStats(websiteId, {
startDate,
endDate,
timezone,
unit,
filters: {
url,
referrer,
title,
os,
browser,
device,
country,
region,
city,
},
}),
getPageviewStats(websiteId, filters),
getSessionStats(websiteId, filters),
]);
return ok(res, { pageviews, sessions });

46
pages/api/websites/[id]/stats.ts
View File

@ -56,40 +56,26 @@ export default async (
const prevStartDate = subMinutes(startDate, diff);
const prevEndDate = subMinutes(endDate, diff);
const metrics = await getWebsiteStats(websiteId, {
startDate,
endDate,
filters: {
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
region,
city,
},
});
const filters = {
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
region,
city,
};
const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate });
const prevPeriod = await getWebsiteStats(websiteId, {
...filters,
startDate: prevStartDate,
endDate: prevEndDate,
filters: {
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
region,
city,
},
});
const stats = Object.keys(metrics[0]).reduce((obj, key) => {

77
queries/analytics/eventData/getEventDataEvents.ts
View File

@ -1,17 +1,10 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataFields } from 'lib/types';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters, WebsiteEventDataFields } from 'lib/types';
export async function getEventDataEvents(
...args: [
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
]
...args: [websiteId: string, filters: QueryFilters & { field?: string; event?: string }]
): Promise<WebsiteEventDataFields[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -21,64 +14,60 @@ export async function getEventDataEvents(
async function relationalQuery(
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
filters: QueryFilters & { field?: string; event?: string },
) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { event } = filters;
const { rawQuery, parseFilters } = prisma;
const { params } = await parseFilters(websiteId, filters);
if (event) {
return rawQuery(
`
select
we.event_name as event,
ed.event_key as field,
ed.data_type as type,
ed.string_value as value,
website_event.event_name as event,
event_data.event_key as field,
event_data.data_type as type,
event_data.string_value as value,
count(*) as total
from event_data as ed
inner join website_event as we
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and we.event_name = {{event}}
group by we.event_name, ed.event_key, ed.data_type, ed.string_value
from event_data
inner join website_event
on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
and websit_event.event_name = {{event}}
group by website_event.event_name, event_data.event_key, event_data.data_type, event_data.string_value
order by 1 asc, 2 asc, 3 asc, 4 desc
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate, ...filters },
params,
);
}
return rawQuery(
`
select
we.event_name as event,
ed.event_key as field,
ed.data_type as type,
website_event.event_name as event,
event_data.event_key as field,
event_data.data_type as type,
count(*) as total
from event_data as ed
inner join website_event as we
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
group by we.event_name, ed.event_key, ed.data_type
from event_data
inner join website_event
on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
group by website_event.event_name, event_data.event_key, event_data.data_type
order by 1 asc, 2 asc
limit 100
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}
async function clickhouseQuery(
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
filters: QueryFilters & { field?: string; event?: string },
) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { rawQuery, parseFilters } = clickhouse;
const { event } = filters;
const { params } = await parseFilters(websiteId, filters);
if (event) {
return rawQuery(
@ -97,7 +86,7 @@ async function clickhouseQuery(
order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100
`,
{ ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}
@ -115,6 +104,6 @@ async function clickhouseQuery(
order by 1 asc, 2 asc
limit 100
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}

28
queries/analytics/eventData/getEventDataFields.ts
View File

@ -1,12 +1,10 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataFields } from 'lib/types';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters, WebsiteEventDataFields } from 'lib/types';
export async function getEventDataFields(
...args: [websiteId: string, startDate: Date, endDate: Date, field?: string]
...args: [websiteId: string, filters: QueryFilters & { field?: string }]
): Promise<WebsiteEventDataFields[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -14,9 +12,10 @@ export async function getEventDataFields(
});
}
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
async function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) {
const { rawQuery, parseFilters } = prisma;
const { field } = filters;
const { params } = await parseFilters(websiteId, filters);
if (field) {
return rawQuery(
@ -33,7 +32,7 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}
@ -50,13 +49,14 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) {
const { rawQuery, parseFilters } = clickhouse;
const { field } = filters;
const { params } = await parseFilters(websiteId, filters);
if (field) {
return rawQuery(
@ -73,7 +73,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}
@ -90,6 +90,6 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
params,
);
}

59
queries/analytics/events/getEventMetrics.ts
View File

@ -1,24 +1,11 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { WebsiteEventMetric } from 'lib/types';
import { WebsiteEventMetric, QueryFilters } from 'lib/types';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export interface GetEventMetricsCriteria {
startDate: Date;
endDate: Date;
timezone: string;
unit: string;
filters: {
url: string;
eventName: string;
};
}
export async function getEventMetrics(
...args: [websiteId: string, criteria: GetEventMetricsCriteria]
...args: [websiteId: string, criteria: QueryFilters]
): Promise<WebsiteEventMetric[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -26,11 +13,13 @@ export async function getEventMetrics(
});
}
async function relationalQuery(websiteId: string, criteria: GetEventMetricsCriteria) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters } = criteria;
const { rawQuery, getDateQuery, getFilterQuery } = prisma;
const website = await loadWebsite(websiteId);
const filterQuery = getFilterQuery(filters);
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { rawQuery, getDateQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.customEvent,
});
return rawQuery(
`
@ -46,22 +35,17 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite
group by 1, 2
order by 2
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.customEvent,
domain: website.domain,
},
params,
);
}
async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCriteria) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters } = criteria;
const { rawQuery, getDateQuery, getFilterQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const filterQuery = getFilterQuery(filters);
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.customEvent,
});
return rawQuery(
`
@ -77,13 +61,6 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite
group by x, t
order by t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.customEvent,
domain: website.domain,
},
params,
);
}

67
queries/analytics/pageviews/getPageviewMetrics.ts
View File

@ -2,19 +2,10 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters } from 'lib/types';
export async function getPageviewMetrics(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
column: string;
filters: object;
},
]
...args: [websiteId: string, columns: string, filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -22,20 +13,12 @@ export async function getPageviewMetrics(
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
column: string;
filters: object;
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
async function relationalQuery(websiteId: string, column: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -50,31 +33,16 @@ async function relationalQuery(
order by 2 desc
limit 100
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
column: string;
filters: object;
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -88,13 +56,6 @@ async function clickhouseQuery(
order by y desc
limit 100
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}

65
queries/analytics/pageviews/getPageviewStats.ts
View File

@ -2,43 +2,22 @@ import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters } from 'lib/types';
export interface PageviewStatsCriteria {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
filters: {
url?: string;
referrer?: string;
title?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
region?: string;
city?: string;
};
}
export async function getPageviewStats(
...args: [websiteId: string, criteria: PageviewStatsCriteria]
) {
export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria;
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -53,22 +32,17 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
${filterQuery}
group by 1
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}
async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) {
const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria;
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -88,13 +62,6 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
) as g
order by t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}

54
queries/analytics/reports/getInsights.ts
View File

@ -1,19 +1,10 @@
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { maxDate } from 'lib/date';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { QueryFilters } from 'lib/types';
export interface GetInsightsCriteria {
startDate: Date;
endDate: Date;
fields: { name: string; type: string; value: string }[];
filters: string[];
groups: string[];
}
export async function getInsights(...args: [websiteId: string, criteria: GetInsightsCriteria]) {
export async function getInsights(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
@ -22,18 +13,18 @@ export async function getInsights(...args: [websiteId: string, criteria: GetInsi
async function relationalQuery(
websiteId: string,
criteria: GetInsightsCriteria,
filters: QueryFilters,
): Promise<
{
x: string;
y: number;
}[]
> {
const { startDate, endDate, filters = [] } = criteria;
const { parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const params = {};
const { filterQuery, joinSession } = parseFilters(params);
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -48,37 +39,30 @@ async function relationalQuery(
${filterQuery}
group by 1
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
params,
);
}
async function clickhouseQuery(
websiteId: string,
criteria: GetInsightsCriteria,
filters: QueryFilters,
): Promise<
{
x: string;
y: number;
}[]
> {
const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria;
const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const params = {};
const { filterQuery } = parseFilters(params);
const fieldsQuery = parseFields(fields);
const { fields } = filters;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
${fieldsQuery}
${parseFields(fields)}
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
@ -88,13 +72,7 @@ async function clickhouseQuery(
order by total desc
limit 500
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
params,
);
}

48
queries/analytics/sessions/getSessionMetrics.ts
View File

@ -2,14 +2,10 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters } from 'lib/types';
export async function getSessionMetrics(
...args: [
websiteId: string,
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
]
...args: [websiteId: string, column: string, filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -17,14 +13,12 @@ export async function getSessionMetrics(
});
}
async function relationalQuery(
websiteId: string,
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const website = await loadWebsite(websiteId);
const { startDate, endDate, column, filters = {} } = criteria;
async function relationalQuery(websiteId: string, column: string, filters: QueryFilters) {
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`select ${column} x, count(*) y
@ -32,28 +26,22 @@ async function relationalQuery(
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = {{eventType}}
${filterQuery}
) as t
group by 1
order by 2 desc
limit 100`,
{
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
...filters,
},
params,
);
}
async function clickhouseQuery(
websiteId: string,
data: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const { startDate, endDate, column, filters = {} } = data;
async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -68,12 +56,6 @@ async function clickhouseQuery(
order by y desc
limit 100
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
params,
);
}

65
queries/analytics/sessions/getSessionStats.ts
View File

@ -2,43 +2,22 @@ import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters } from 'lib/types';
export interface SessionStatsCriteria {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
filters: {
url?: string;
referrer?: string;
title?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
region?: string;
city?: string;
};
}
export async function getSessionStats(
...args: [websiteId: string, criteria: SessionStatsCriteria]
) {
export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria;
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -53,22 +32,17 @@ async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria
${filterQuery}
group by 1
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}
async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria) {
const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria;
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -88,13 +62,6 @@ async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria
) as g
order by t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}

14
queries/analytics/stats/getWebsiteDateRange.ts
View File

@ -1,9 +1,7 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { loadWebsite } from 'lib/load';
import { DEFAULT_RESET_DATE } from 'lib/constants';
import { maxDate } from 'lib/date';
export async function getWebsiteDateRange(...args: [websiteId: string]) {
return runQuery({
@ -13,8 +11,8 @@ export async function getWebsiteDateRange(...args: [websiteId: string]) {
}
async function relationalQuery(websiteId: string) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { rawQuery, parseFilters } = prisma;
const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });
const result = await rawQuery(
`
@ -25,15 +23,15 @@ async function relationalQuery(websiteId: string) {
where website_id = {{websiteId::uuid}}
and created_at >= {{startDate}}
`,
{ websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) },
params,
);
return result[0] ?? null;
}
async function clickhouseQuery(websiteId: string) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { rawQuery, parseFilters } = clickhouse;
const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });
const result = await rawQuery(
`
@ -44,7 +42,7 @@ async function clickhouseQuery(websiteId: string) {
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime}
`,
{ websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) },
params,
);
return result[0] ?? null;

56
queries/analytics/stats/getWebsiteStats.ts
View File

@ -2,29 +2,21 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
import { QueryFilters } from 'lib/types';
export async function getWebsiteStats(
...args: [
websiteId: string,
data: { startDate: Date; endDate: Date; type?: string; filters: object },
]
) {
export async function getWebsiteStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = criteria;
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters);
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -43,32 +35,23 @@ async function relationalQuery(
join website
on website_event.website_id = website.website_id
${joinSession}
where event_type = {{eventType}}
and website.website_id = {{websiteId::uuid}}
where website.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1, 2
) as t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}
async function clickhouseQuery(
websiteId: string,
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = criteria;
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
@ -92,13 +75,6 @@ async function clickhouseQuery(
group by session_id, time_series
) as t;
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
params,
);
}