update CH rawquery and type

This commit is contained in:
Francis Cao 2023-09-29 11:00:06 -07:00
parent 35d45334df
commit 98b7624e8f
17 changed files with 169 additions and 52 deletions

View File

@ -1,4 +1,4 @@
import { ClickHouse } from 'clickhouse'; import { ClickHouseClient, createClient } from '@clickhouse/client';
import dateFormat from 'dateformat'; import dateFormat from 'dateformat';
import debug from 'debug'; import debug from 'debug';
import { CLICKHOUSE } from 'lib/db'; import { CLICKHOUSE } from 'lib/db';
@ -17,7 +17,7 @@ export const CLICKHOUSE_DATE_FORMATS = {
const log = debug('umami:clickhouse'); const log = debug('umami:clickhouse');
let clickhouse: ClickHouse; let clickhouse: ClickHouseClient;
const enabled = Boolean(process.env.CLICKHOUSE_URL); const enabled = Boolean(process.env.CLICKHOUSE_URL);
function getClient() { function getClient() {
@ -25,18 +25,19 @@ function getClient() {
hostname, hostname,
port, port,
pathname, pathname,
// protocol,
username = 'default', username = 'default',
password, password,
} = new URL(process.env.CLICKHOUSE_URL); } = new URL(process.env.CLICKHOUSE_URL);
const client = new ClickHouse({ // const formattedProtocol =
url: hostname, // protocol.toLowerCase() === 'clickhouse:' || protocol === 'https:' ? 'https:' : 'http:';
port: Number(port),
format: 'json', const client = createClient({
config: { host: `http://${hostname}:${port}`,
database: pathname.replace('/', ''), database: pathname.replace('/', ''),
}, username: username,
basicAuth: password ? { username, password } : null, password,
}); });
if (process.env.NODE_ENV !== 'production') { if (process.env.NODE_ENV !== 'production') {
@ -118,7 +119,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
}; };
} }
async function rawQuery<T>(query: string, params: object = {}): Promise<T> { async function rawQuery(query: string, params: Record<string, unknown> = {}): Promise<unknown> {
if (process.env.LOG_QUERY) { if (process.env.LOG_QUERY) {
log('QUERY:\n', query); log('QUERY:\n', query);
log('PARAMETERS:\n', params); log('PARAMETERS:\n', params);
@ -126,7 +127,15 @@ async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
await connect(); await connect();
return clickhouse.query(query, { params }).toPromise() as Promise<T>; const resultSet = await clickhouse.query({
query: query,
query_params: params,
format: 'JSONEachRow',
});
const data = await resultSet.json();
return data;
} }
async function findUnique(data) { async function findUnique(data) {

View File

@ -59,7 +59,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ eventName: string; fieldName: string; dataType: number; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const { event } = filters; const { event } = filters;
const { params } = await parseFilters(websiteId, filters); const { params } = await parseFilters(websiteId, filters);
@ -75,14 +78,23 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as total count(*) as total
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_name = {event:String} and event_name = {event:String}
group by event_key, data_type, string_value, event_name group by event_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 4 desc order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100 limit 100
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return {
eventName: a.eventName,
fieldName: a.fieldName,
dataType: Number(a.dataType),
total: Number(a.total),
};
});
});
} }
return rawQuery( return rawQuery(
@ -94,11 +106,20 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as total count(*) as total
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by event_key, data_type, event_name group by event_key, data_type, event_name
order by 1 asc, 2 asc order by 1 asc, 2 asc
limit 100 limit 100
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return {
eventName: a.eventName,
fieldName: a.fieldName,
dataType: Number(a.dataType),
total: Number(a.total),
};
});
});
} }

View File

@ -37,7 +37,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { field?: string },
): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters, { const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { field: 'event_key' }, columns: { field: 'event_key' },
@ -52,12 +55,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters & { fiel
count(*) as total count(*) as total
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery} ${filterQuery}
group by event_key, data_type, string_value group by event_key, data_type, string_value
order by 3 desc, 2 desc, 1 asc order by 3 desc, 2 desc, 1 asc
limit 100 limit 100
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return {
fieldName: a.fieldName,
dataType: Number(a.dataType),
fieldValue: a.fieldValue,
total: Number(a.total),
};
});
});
} }

View File

@ -42,7 +42,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ events: number; fields: number; records: number }> {
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters); const { filterQuery, params } = await parseFilters(websiteId, filters);
@ -59,11 +62,19 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as "total" count(*) as "total"
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery} ${filterQuery}
group by event_id, event_key group by event_id, event_key
) as t ) as t
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return {
events: Number(a.events),
fields: Number(a.fields),
records: Number(a.records),
};
});
});
} }

View File

@ -40,7 +40,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; t: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters; const { timezone = 'UTC', unit = 'day' } = filters;
const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
@ -56,12 +59,16 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) y count(*) y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
group by x, t group by x, t
order by t order by t
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return { x: a.x, t: a.t, y: Number(a.y) };
});
});
} }

View File

@ -37,7 +37,7 @@ function clickhouseQuery(websiteId: string, startDate: Date, eventType: number)
event_name as eventName event_name as eventName
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime} and created_at >= {startDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
`, `,
{ {

View File

@ -24,17 +24,23 @@ async function relationalQuery(websiteId: string) {
); );
} }
async function clickhouseQuery(websiteId: string) { async function clickhouseQuery(websiteId: string): Promise<{ x: number }> {
const { rawQuery } = clickhouse; const { rawQuery } = clickhouse;
return rawQuery( const result = rawQuery(
` `
select select
count(distinct session_id) x count(distinct session_id) x
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime} and created_at >= {startAt:DateTime64}
`, `,
{ websiteId, startAt: subMinutes(new Date(), 5) }, { websiteId, startAt: subMinutes(new Date(), 5) },
); ).then(a => {
return Object.values(a).map(a => {
return { x: Number(a.x) };
});
});
return result[0] ?? null;
} }

View File

@ -40,7 +40,7 @@ async function clickhouseQuery(websiteId: string) {
max(created_at) as maxdate max(created_at) as maxdate
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime} and created_at >= {startDate:DateTime64}
`, `,
params, params,
); );

View File

@ -46,7 +46,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> {
const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
...filters, ...filters,
@ -69,12 +72,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
max(created_at) max_time max(created_at) max_time
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
group by session_id, time_series group by session_id, time_series
) as t; ) as t;
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return {
pageviews: Number(a.pageviews),
uniques: Number(a.uniques),
bounces: Number(a.bounces),
totaltime: Number(a.totaltime),
};
});
});
} }

View File

@ -48,7 +48,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
); );
} }
async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
column: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
...filters, ...filters,
@ -65,7 +69,7 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
select ${column} x, count(*) y select ${column} x, count(*) y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${excludeDomain} ${excludeDomain}
${filterQuery} ${filterQuery}
@ -74,5 +78,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
limit 100 limit 100
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return { x: a.x, y: Number(a.y) };
});
});
} }

View File

@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters; const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as y count(*) as y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
group by t group by t
@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
order by t order by t
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return { x: a.x, y: Number(a.y) };
});
});
} }

View File

@ -172,7 +172,7 @@ async function clickhouseQuery(
); );
} }
return rawQuery<{ level: number; count: number }[]>( return rawQuery(
` `
WITH level0 AS ( WITH level0 AS (
select distinct session_id, url_path, referrer_path, created_at select distinct session_id, url_path, referrer_path, created_at
@ -201,7 +201,7 @@ async function clickhouseQuery(
).then(results => { ).then(results => {
return urls.map((a, i) => ({ return urls.map((a, i) => ({
x: a, x: a,
y: results[i]?.count || 0, y: Number(results[i]?.count) || 0,
z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
})); }));
}); });

View File

@ -75,7 +75,7 @@ async function clickhouseQuery(
${parseFields(fields)} ${parseFields(fields)}
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
${parseGroupBy(fields)} ${parseGroupBy(fields)}

View File

@ -172,5 +172,15 @@ async function clickhouseQuery(
startDate, startDate,
endDate, endDate,
}, },
); ).then(a => {
return Object.values(a).map(a => {
return {
date: a.date,
day: Number(a.day),
visitors: Number(a.visitors),
returnVisitors: Number(a.returnVisitors),
percentage: Number(a.percentage),
};
});
});
} }

View File

@ -47,7 +47,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
); );
} }
async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
column: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { parseFilters, rawQuery } = clickhouse; const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
...filters, ...filters,
@ -63,7 +67,6 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
${includeCountry ? ', country' : ''} ${includeCountry ? ', country' : ''}
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
group by x group by x
@ -72,5 +75,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
limit 100 limit 100
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return { x: a.x, y: Number(a.y) };
});
});
} }

View File

@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
); );
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters; const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(distinct session_id) as y count(distinct session_id) as y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
group by t group by t
@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
order by t order by t
`, `,
params, params,
); ).then(a => {
return Object.values(a).map(a => {
return { x: a.x, y: Number(a.y) };
});
});
} }

View File

@ -42,7 +42,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date) {
city city
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime} and created_at >= {startDate:DateTime64}
`, `,
{ {
websiteId, websiteId,