diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 186a085d..c371e7f9 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -1,4 +1,4 @@ -import { ClickHouse } from 'clickhouse'; +import { ClickHouseClient, createClient } from '@clickhouse/client'; import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; @@ -17,7 +17,7 @@ export const CLICKHOUSE_DATE_FORMATS = { const log = debug('umami:clickhouse'); -let clickhouse: ClickHouse; +let clickhouse: ClickHouseClient; const enabled = Boolean(process.env.CLICKHOUSE_URL); function getClient() { @@ -25,18 +25,19 @@ function getClient() { hostname, port, pathname, + // protocol, username = 'default', password, } = new URL(process.env.CLICKHOUSE_URL); - const client = new ClickHouse({ - url: hostname, - port: Number(port), - format: 'json', - config: { - database: pathname.replace('/', ''), - }, - basicAuth: password ? { username, password } : null, + // const formattedProtocol = + // protocol.toLowerCase() === 'clickhouse:' || protocol === 'https:' ? 'https:' : 'http:'; + + const client = createClient({ + host: `http://${hostname}:${port}`, + database: pathname.replace('/', ''), + username: username, + password, }); if (process.env.NODE_ENV !== 'production') { @@ -118,7 +119,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio }; } -async function rawQuery(query: string, params: object = {}): Promise { +async function rawQuery(query: string, params: Record = {}): Promise { if (process.env.LOG_QUERY) { log('QUERY:\n', query); log('PARAMETERS:\n', params); @@ -126,7 +127,15 @@ async function rawQuery(query: string, params: object = {}): Promise { await connect(); - return clickhouse.query(query, { params }).toPromise() as Promise; + const resultSet = await clickhouse.query({ + query: query, + query_params: params, + format: 'JSONEachRow', + }); + + const data = await resultSet.json(); + + return data; } async function findUnique(data) { diff --git a/src/queries/analytics/eventData/getEventDataEvents.ts b/src/queries/analytics/eventData/getEventDataEvents.ts index 2c8cb0e0..29158503 100644 --- a/src/queries/analytics/eventData/getEventDataEvents.ts +++ b/src/queries/analytics/eventData/getEventDataEvents.ts @@ -59,7 +59,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ eventName: string; fieldName: string; dataType: number; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { event } = filters; const { params } = await parseFilters(websiteId, filters); @@ -75,14 +78,23 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { count(*) as total from event_data where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_name = {event:String} group by event_key, data_type, string_value, event_name order by 1 asc, 2 asc, 3 asc, 4 desc limit 100 `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { + eventName: a.eventName, + fieldName: a.fieldName, + dataType: Number(a.dataType), + total: Number(a.total), + }; + }); + }); } return rawQuery( @@ -94,11 +106,20 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { count(*) as total from event_data where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} group by event_key, data_type, event_name order by 1 asc, 2 asc limit 100 `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { + eventName: a.eventName, + fieldName: a.fieldName, + dataType: Number(a.dataType), + total: Number(a.total), + }; + }); + }); } diff --git a/src/queries/analytics/eventData/getEventDataFields.ts b/src/queries/analytics/eventData/getEventDataFields.ts index ac32b188..df5a8874 100644 --- a/src/queries/analytics/eventData/getEventDataFields.ts +++ b/src/queries/analytics/eventData/getEventDataFields.ts @@ -37,7 +37,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters & { field?: string }, +): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, filters, { columns: { field: 'event_key' }, @@ -52,12 +55,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters & { fiel count(*) as total from event_data where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} group by event_key, data_type, string_value order by 3 desc, 2 desc, 1 asc limit 100 `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { + fieldName: a.fieldName, + dataType: Number(a.dataType), + fieldValue: a.fieldValue, + total: Number(a.total), + }; + }); + }); } diff --git a/src/queries/analytics/eventData/getEventDataStats.ts b/src/queries/analytics/eventData/getEventDataStats.ts index cf77ff89..b940e9c4 100644 --- a/src/queries/analytics/eventData/getEventDataStats.ts +++ b/src/queries/analytics/eventData/getEventDataStats.ts @@ -42,7 +42,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ events: number; fields: number; records: number }> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, filters); @@ -59,11 +62,19 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { count(*) as "total" from event_data where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} group by event_id, event_key ) as t `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { + events: Number(a.events), + fields: Number(a.fields), + records: Number(a.records), + }; + }); + }); } diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts index 778cfee1..53638fbd 100644 --- a/src/queries/analytics/events/getEventMetrics.ts +++ b/src/queries/analytics/events/getEventMetrics.ts @@ -40,7 +40,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ x: string; t: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { @@ -56,12 +59,16 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { count(*) y from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} group by x, t order by t `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { x: a.x, t: a.t, y: Number(a.y) }; + }); + }); } diff --git a/src/queries/analytics/events/getEvents.ts b/src/queries/analytics/events/getEvents.ts index 17528d66..fe074ec2 100644 --- a/src/queries/analytics/events/getEvents.ts +++ b/src/queries/analytics/events/getEvents.ts @@ -37,7 +37,7 @@ function clickhouseQuery(websiteId: string, startDate: Date, eventType: number) event_name as eventName from website_event where website_id = {websiteId:UUID} - and created_at >= {startDate:DateTime} + and created_at >= {startDate:DateTime64} and event_type = {eventType:UInt32} `, { diff --git a/src/queries/analytics/getActiveVisitors.ts b/src/queries/analytics/getActiveVisitors.ts index db3583eb..962bea35 100644 --- a/src/queries/analytics/getActiveVisitors.ts +++ b/src/queries/analytics/getActiveVisitors.ts @@ -24,17 +24,23 @@ async function relationalQuery(websiteId: string) { ); } -async function clickhouseQuery(websiteId: string) { +async function clickhouseQuery(websiteId: string): Promise<{ x: number }> { const { rawQuery } = clickhouse; - return rawQuery( + const result = rawQuery( ` select count(distinct session_id) x from website_event where website_id = {websiteId:UUID} - and created_at >= {startAt:DateTime} + and created_at >= {startAt:DateTime64} `, { websiteId, startAt: subMinutes(new Date(), 5) }, - ); + ).then(a => { + return Object.values(a).map(a => { + return { x: Number(a.x) }; + }); + }); + + return result[0] ?? null; } diff --git a/src/queries/analytics/getWebsiteDateRange.ts b/src/queries/analytics/getWebsiteDateRange.ts index 4fb24733..a4daaafc 100644 --- a/src/queries/analytics/getWebsiteDateRange.ts +++ b/src/queries/analytics/getWebsiteDateRange.ts @@ -40,7 +40,7 @@ async function clickhouseQuery(websiteId: string) { max(created_at) as maxdate from website_event where website_id = {websiteId:UUID} - and created_at >= {startDate:DateTime} + and created_at >= {startDate:DateTime64} `, params, ); diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 16519511..654a09a9 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -46,7 +46,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> { const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, @@ -69,12 +72,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { max(created_at) max_time from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} group by session_id, time_series ) as t; `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { + pageviews: Number(a.pageviews), + uniques: Number(a.uniques), + bounces: Number(a.bounces), + totaltime: Number(a.totaltime), + }; + }); + }); } diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts index 3cf6c122..4e7e9da2 100644 --- a/src/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts @@ -48,7 +48,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query ); } -async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + column: string, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, @@ -65,7 +69,7 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query select ${column} x, count(*) y from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} @@ -74,5 +78,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query limit 100 `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { x: a.x, y: Number(a.y) }; + }); + }); } diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts index d6a980ef..b221e010 100644 --- a/src/queries/analytics/pageviews/getPageviewStats.ts +++ b/src/queries/analytics/pageviews/getPageviewStats.ts @@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { @@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { count(*) as y from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} group by t @@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { order by t `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { x: a.x, y: Number(a.y) }; + }); + }); } diff --git a/src/queries/analytics/reports/getFunnel.ts b/src/queries/analytics/reports/getFunnel.ts index 1bbbc878..8dbd8d45 100644 --- a/src/queries/analytics/reports/getFunnel.ts +++ b/src/queries/analytics/reports/getFunnel.ts @@ -172,7 +172,7 @@ async function clickhouseQuery( ); } - return rawQuery<{ level: number; count: number }[]>( + return rawQuery( ` WITH level0 AS ( select distinct session_id, url_path, referrer_path, created_at @@ -201,7 +201,7 @@ async function clickhouseQuery( ).then(results => { return urls.map((a, i) => ({ x: a, - y: results[i]?.count || 0, + y: Number(results[i]?.count) || 0, z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off })); }); diff --git a/src/queries/analytics/reports/getInsights.ts b/src/queries/analytics/reports/getInsights.ts index fa54488b..c5fac48b 100644 --- a/src/queries/analytics/reports/getInsights.ts +++ b/src/queries/analytics/reports/getInsights.ts @@ -75,7 +75,7 @@ async function clickhouseQuery( ${parseFields(fields)} from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} ${parseGroupBy(fields)} diff --git a/src/queries/analytics/reports/getRetention.ts b/src/queries/analytics/reports/getRetention.ts index 3c384b6e..1ec65fcb 100644 --- a/src/queries/analytics/reports/getRetention.ts +++ b/src/queries/analytics/reports/getRetention.ts @@ -172,5 +172,15 @@ async function clickhouseQuery( startDate, endDate, }, - ); + ).then(a => { + return Object.values(a).map(a => { + return { + date: a.date, + day: Number(a.day), + visitors: Number(a.visitors), + returnVisitors: Number(a.returnVisitors), + percentage: Number(a.percentage), + }; + }); + }); } diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts index 43d9ef5a..e97501d7 100644 --- a/src/queries/analytics/sessions/getSessionMetrics.ts +++ b/src/queries/analytics/sessions/getSessionMetrics.ts @@ -47,7 +47,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query ); } -async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + column: string, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { const { parseFilters, rawQuery } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, @@ -63,7 +67,6 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query ${includeCountry ? ', country' : ''} from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} ${filterQuery} group by x @@ -72,5 +75,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query limit 100 `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { x: a.x, y: Number(a.y) }; + }); + }); } diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts index 9ed01a59..c977187d 100644 --- a/src/queries/analytics/sessions/getSessionStats.ts +++ b/src/queries/analytics/sessions/getSessionStats.ts @@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { @@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { count(distinct session_id) as y from website_event where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime} and {endDate:DateTime} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} group by t @@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { order by t `, params, - ); + ).then(a => { + return Object.values(a).map(a => { + return { x: a.x, y: Number(a.y) }; + }); + }); } diff --git a/src/queries/analytics/sessions/getSessions.ts b/src/queries/analytics/sessions/getSessions.ts index 6936f902..d67edd5e 100644 --- a/src/queries/analytics/sessions/getSessions.ts +++ b/src/queries/analytics/sessions/getSessions.ts @@ -42,7 +42,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date) { city from website_event where website_id = {websiteId:UUID} - and created_at >= {startDate:DateTime} + and created_at >= {startDate:DateTime64} `, { websiteId,