m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2025-02-06 01:15:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

route dashboard queries based on filters selected

Browse Source
This commit is contained in:
Francis Cao 2024-08-01 15:32:49 -07:00
parent cb4368e12c
commit 9882ff24f6
8 changed files with 191 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/lib/clickhouse.ts
View File

@ -3,7 +3,7 @@ import dateFormat from 'dateformat';
import debug from 'debug';
import { CLICKHOUSE } from 'lib/db';
import { PageParams, QueryFilters, QueryOptions } from './types';
import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
import { fetchWebsite } from './load';
import { maxDate } from './date';
import { filtersToArray } from './params';
@ -100,6 +100,26 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
return query.join('\n');
}
function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
if (column) {
if (EVENT_COLUMNS.includes(name)) {
arr.push(`and has(${column}, {${name}:String})`);
if (name === 'referrer') {
arr.push('and not has(referrer_domain, {websiteDomain:String})');
}
} else {
arr.push(`and ${mapFilter(column, operator, name)}`);
}
}
return arr;
}, []);
return query.join('\n');
}
function getDateQuery(filters: QueryFilters = {}) {
const { startDate, endDate } = filters;
@ -139,6 +159,25 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
};
}
async function parseSessionFilters(
websiteId: string,
filters: QueryFilters = {},
options?: QueryOptions,
) {
const website = await fetchWebsite(websiteId);
return {
filterQuery: getSessionFilterQuery(filters, options),
dateQuery: getDateQuery(filters),
params: {
...getFilterParams(filters),
websiteId,
startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
websiteDomain: website.domain,
},
};
}
async function pagedQuery(
query: string,
queryParams: { [key: string]: any },
@ -221,6 +260,7 @@ export default {
getDateFormat,
getFilterQuery,
parseFilters,
parseSessionFilters,
pagedQuery,
findUnique,
findFirst,

15
src/lib/constants.ts
View File

@ -33,16 +33,7 @@ export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
export const EVENT_COLUMNS = [
'url',
'entry',
'exit',
'referrer',
'title',
'query',
'event',
'host',
];
export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event'];
export const SESSION_COLUMNS = [
'browser',
@ -58,8 +49,8 @@ export const SESSION_COLUMNS = [
export const FILTER_COLUMNS = {
url: 'url_path',
entry: 'entry_url',
exit: 'exit_url',
entry: 'url_path',
exit: 'url_path',
referrer: 'referrer_domain',
host: 'hostname',
title: 'page_title',

34
src/queries/analytics/events/getEventMetrics.ts
View File

@ -1,8 +1,8 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { WebsiteEventMetric, QueryFilters } from 'lib/types';
import { EVENT_TYPE } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters, WebsiteEventMetric } from 'lib/types';
export async function getEventMetrics(
...args: [websiteId: string, filters: QueryFilters]
@ -51,8 +51,24 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.customEvent,
});
return rawQuery(
`
let sql = '';
if (filterQuery) {
sql = `
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
count(*) y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x, t
order by t
`;
} else {
sql = `
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
@ -64,13 +80,13 @@ async function clickhouseQuery(
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
) as g
group by x, t
order by t
`,
params,
).then(a => {
`;
}
return rawQuery(sql, params).then(a => {
return Object.values(a).map(a => {
return { x: a.x, t: a.t, y: Number(a.y) };
});

59
src/queries/analytics/getWebsiteStats.ts
View File

@ -3,6 +3,7 @@ import { EVENT_TYPE } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
import { EVENT_COLUMNS } from 'lib/constants';
export async function getWebsiteStats(
...args: [websiteId: string, filters: QueryFilters]
@ -67,30 +68,33 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
// `
// select
// sum(t.c) as "pageviews",
// count(distinct t.session_id) as "visitors",
// count(distinct t.visit_id) as "visits",
// sum(if(t.c = 1, 1, 0)) as "bounces",
// sum(max_time-min_time) as "totaltime"
// from (
// select
// session_id,
// visit_id,
// count(*) c,
// min(created_at) min_time,
// max(created_at) max_time
// from website_event
// where website_id = {websiteId:UUID}
// and created_at between {startDate:DateTime64} and {endDate:DateTime64}
// and event_type = {eventType:UInt32}
// ${filterQuery}
// group by session_id, visit_id
// ) as t;
// `,
`
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
sql = `
select
sum(t.c) as "pageviews",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(max_time-min_time) as "totaltime"
from (
select
session_id,
visit_id,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by session_id, visit_id
) as t;
`;
} else {
sql = `
select
sum(views) as "pageviews",
uniq(session_id) as "visitors",
@ -102,9 +106,10 @@ async function clickhouseQuery(
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery};
`,
params,
).then(result => {
`;
}
return rawQuery(sql, params).then(result => {
return Object.values(result).map((a: any) => {
return {
pageviews: Number(a.pageviews),

95
src/queries/analytics/pageviews/getPageviewMetrics.ts
View File

@ -1,5 +1,5 @@
import clickhouse from 'lib/clickhouse';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
@ -91,46 +91,66 @@ async function clickhouseQuery(
});
let excludeDomain = '';
let groupByQuery = '';
let sql = '';
if (column === 'referrer_domain') {
excludeDomain = `and t != {websiteDomain:String} and t != ''`;
}
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
let entryExitQuery = '';
let columnQuery = `arrayJoin(${column})`;
if (column === 'referrer_domain') {
excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
}
if (type === 'entry') {
columnQuery = `visit_id x, argMinMerge(${column})`;
}
if (type === 'entry' || type === 'exit') {
const aggregrate = type === 'entry' ? 'min' : 'max';
if (type === 'exit') {
columnQuery = `visit_id x, argMaxMerge(${column})`;
}
entryExitQuery = `
JOIN (select visit_id,
${aggregrate}(created_at) target_created_at
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
group by visit_id) x
ON x.visit_id = website_event.visit_id
and x.target_created_at = website_event.created_at`;
}
if (type === 'entry' || type === 'exit') {
groupByQuery = 'group by x';
}
sql = `
select ${column} x, count(*) y
from website_event
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery}
group by x
order by y desc
limit ${limit}
offset ${offset}
`;
} else {
let groupByQuery = '';
// let excludeDomain = '';
// if (column === 'referrer_domain') {
// excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
// }
if (column === 'referrer_domain') {
excludeDomain = `and t != {websiteDomain:String} and t != ''`;
}
return rawQuery(
// `
// select ${column} x, count(*) y
// from website_event
// where website_id = {websiteId:UUID}
// and created_at between {startDate:DateTime64} and {endDate:DateTime64}
// and event_type = {eventType:UInt32}
// ${excludeDomain}
// ${filterQuery}
// group by x
// order by y desc
// limit ${limit}
// offset ${offset}
// `,
`
let columnQuery = `arrayJoin(${column})`;
if (type === 'entry') {
columnQuery = `visit_id x, argMinMerge(entry_url)`;
}
if (type === 'exit') {
columnQuery = `visit_id x, argMaxMerge(exit_url)`;
}
if (type === 'entry' || type === 'exit') {
groupByQuery = 'group by x';
}
sql = `
select g.t as x,
count(*) as y
from (
@ -146,9 +166,10 @@ async function clickhouseQuery(
order by y desc
limit ${limit}
offset ${offset}
`,
params,
).then((result: any) => {
`;
}
return rawQuery(sql, params).then((result: any) => {
return Object.values(result).map((a: any) => {
return { x: a.x, y: Number(a.y) };
});

55
src/queries/analytics/pageviews/getPageviewStats.ts
View File

@ -1,7 +1,7 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants';
import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';
import { QueryFilters } from 'lib/types';
export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
@ -47,36 +47,18 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView,
});
const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';
const columnQuery = unit === 'minute' ? 'count(*)' : 'sum(views)';
let sql = '';
return rawQuery(
// `
// select
// ${getDateStringSQL('g.t', unit)} as x,
// g.y as y
// from (
// select
// ${getDateSQL('created_at', unit, timezone)} as t,
// count(*) as y
// from website_event
// where website_id = {websiteId:UUID}
// and created_at between {startDate:DateTime64} and {endDate:DateTime64}
// and event_type = {eventType:UInt32}
// ${filterQuery}
// group by t
// ) as g
// order by t
// `,
`
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
sql = `
select
${getDateStringSQL('g.t', unit)} as x,
g.y as y
from (
select
${getDateSQL('created_at', unit, timezone)} as t,
${columnQuery} as y
from ${table} website_event
count(*) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
@ -84,9 +66,28 @@ async function clickhouseQuery(
group by t
) as g
order by t
`,
params,
).then(result => {
`;
} else {
sql = `
select
${getDateStringSQL('g.t', unit)} as x,
g.y as y
from (
select
${getDateSQL('created_at', unit, timezone)} as t,
sum(views)as y
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
}
return rawQuery(sql, params).then(result => {
return Object.values(result).map((a: any) => {
return { x: a.x, y: Number(a.y) };
});

4
src/queries/analytics/sessions/getSessionMetrics.ts
View File

@ -64,8 +64,8 @@ async function clickhouseQuery(
offset: number = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
const { parseSessionFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseSessionFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});

4
src/queries/analytics/sessions/getSessionStats.ts
View File

@ -41,8 +41,8 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
const { filterQuery, params } = await parseSessionFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});