update CH rawquery and type

src/lib/clickhouse.ts

@@ -1,4 +1,4 @@
-import { ClickHouse } from 'clickhouse';
+import { ClickHouseClient, createClient } from '@clickhouse/client';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
@@ -17,7 +17,7 @@ export const CLICKHOUSE_DATE_FORMATS = {
 
 const log = debug('umami:clickhouse');
 
-let clickhouse: ClickHouse;
+let clickhouse: ClickHouseClient;
 const enabled = Boolean(process.env.CLICKHOUSE_URL);
 
 function getClient() {
@@ -25,18 +25,19 @@ function getClient() {
     hostname,
     port,
     pathname,
+    // protocol,
     username = 'default',
     password,
   } = new URL(process.env.CLICKHOUSE_URL);
 
-  const client = new ClickHouse({
-    url: hostname,
-    port: Number(port),
-    format: 'json',
-    config: {
-      database: pathname.replace('/', ''),
-    },
-    basicAuth: password ? { username, password } : null,
+  // const formattedProtocol =
+  //   protocol.toLowerCase() === 'clickhouse:' || protocol === 'https:' ? 'https:' : 'http:';
+
+  const client = createClient({
+    host: `http://${hostname}:${port}`,
+    database: pathname.replace('/', ''),
+    username: username,
+    password,
   });
 
   if (process.env.NODE_ENV !== 'production') {
@@ -118,7 +119,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
   };
 }
 
-async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
+async function rawQuery(query: string, params: Record<string, unknown> = {}): Promise<unknown> {
   if (process.env.LOG_QUERY) {
     log('QUERY:\n', query);
     log('PARAMETERS:\n', params);
@@ -126,7 +127,15 @@ async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
 
   await connect();
 
-  return clickhouse.query(query, { params }).toPromise() as Promise<T>;
+  const resultSet = await clickhouse.query({
+    query: query,
+    query_params: params,
+    format: 'JSONEachRow',
+  });
+
+  const data = await resultSet.json();
+
+  return data;
 }
 
 async function findUnique(data) {

src/queries/analytics/eventData/getEventDataEvents.ts

@@ -59,7 +59,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters,
+): Promise<{ eventName: string; fieldName: string; dataType: number; total: number }[]> {
   const { rawQuery, parseFilters } = clickhouse;
   const { event } = filters;
   const { params } = await parseFilters(websiteId, filters);
@@ -75,14 +78,23 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
         count(*) as total
       from event_data
       where website_id = {websiteId:UUID}
-        and created_at between {startDate:DateTime} and {endDate:DateTime}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_name = {event:String}
       group by event_key, data_type, string_value, event_name
       order by 1 asc, 2 asc, 3 asc, 4 desc
       limit 100
       `,
       params,
-    );
+    ).then(a => {
+      return Object.values(a).map(a => {
+        return {
+          eventName: a.eventName,
+          fieldName: a.fieldName,
+          dataType: Number(a.dataType),
+          total: Number(a.total),
+        };
+      });
+    });
   }
 
   return rawQuery(
@@ -94,11 +106,20 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
       count(*) as total
     from event_data
     where website_id = {websiteId:UUID}
-      and created_at between {startDate:DateTime} and {endDate:DateTime}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
     group by event_key, data_type, event_name
     order by 1 asc, 2 asc
     limit 100
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return {
+        eventName: a.eventName,
+        fieldName: a.fieldName,
+        dataType: Number(a.dataType),
+        total: Number(a.total),
+      };
+    });
+  });
 }

src/queries/analytics/eventData/getEventDataFields.ts

@@ -37,7 +37,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters & { field?: string },
+): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, filters, {
     columns: { field: 'event_key' },
@@ -52,12 +55,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters & { fiel
       count(*) as total
     from event_data
     where website_id = {websiteId:UUID}
-      and created_at between {startDate:DateTime} and {endDate:DateTime}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
     ${filterQuery}
     group by event_key, data_type, string_value
     order by 3 desc, 2 desc, 1 asc
     limit 100
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return {
+        fieldName: a.fieldName,
+        dataType: Number(a.dataType),
+        fieldValue: a.fieldValue,
+        total: Number(a.total),
+      };
+    });
+  });
 }

src/queries/analytics/eventData/getEventDataStats.ts

@@ -42,7 +42,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters,
+): Promise<{ events: number; fields: number; records: number }> {
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, filters);
 
@@ -59,11 +62,19 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
         count(*) as "total"
       from event_data
       where website_id = {websiteId:UUID}
-        and created_at between {startDate:DateTime} and {endDate:DateTime}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       ${filterQuery}
       group by event_id, event_key
       ) as t
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return {
+        events: Number(a.events),
+        fields: Number(a.fields),
+        records: Number(a.records),
+      };
+    });
+  });
 }

src/queries/analytics/events/getEventMetrics.ts

@@ -40,7 +40,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters,
+): Promise<{ x: string; t: string; y: number }[]> {
   const { timezone = 'UTC', unit = 'day' } = filters;
   const { rawQuery, getDateQuery, parseFilters } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, {
@@ -56,12 +59,16 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
       count(*) y
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at between {startDate:DateTime} and {endDate:DateTime}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}
       ${filterQuery}
     group by x, t
     order by t
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return { x: a.x, t: a.t, y: Number(a.y) };
+    });
+  });
 }

src/queries/analytics/events/getEvents.ts

@@ -37,7 +37,7 @@ function clickhouseQuery(websiteId: string, startDate: Date, eventType: number)
       event_name as eventName
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at >= {startDate:DateTime}
+      and created_at >= {startDate:DateTime64}
       and event_type = {eventType:UInt32}
     `,
     {

src/queries/analytics/getActiveVisitors.ts

@@ -24,17 +24,23 @@ async function relationalQuery(websiteId: string) {
   );
 }
 
-async function clickhouseQuery(websiteId: string) {
+async function clickhouseQuery(websiteId: string): Promise<{ x: number }> {
   const { rawQuery } = clickhouse;
 
-  return rawQuery(
+  const result = rawQuery(
     `
     select
       count(distinct session_id) x
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at >= {startAt:DateTime}
+      and created_at >= {startAt:DateTime64}
     `,
     { websiteId, startAt: subMinutes(new Date(), 5) },
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return { x: Number(a.x) };
+    });
+  });
+
+  return result[0] ?? null;
 }

src/queries/analytics/getWebsiteDateRange.ts

@@ -40,7 +40,7 @@ async function clickhouseQuery(websiteId: string) {
       max(created_at) as maxdate
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at >= {startDate:DateTime}
+      and created_at >= {startDate:DateTime64}
     `,
     params,
   );

src/queries/analytics/getWebsiteStats.ts

@@ -46,7 +46,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters,
+): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> {
   const { rawQuery, getDateQuery, parseFilters } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, {
     ...filters,
@@ -69,12 +72,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
         max(created_at) max_time
       from website_event
       where website_id = {websiteId:UUID}
-        and created_at between {startDate:DateTime} and {endDate:DateTime}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
         ${filterQuery}
       group by session_id, time_series
     ) as t;
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return {
+        pageviews: Number(a.pageviews),
+        uniques: Number(a.uniques),
+        bounces: Number(a.bounces),
+        totaltime: Number(a.totaltime),
+      };
+    });
+  });
 }

src/queries/analytics/pageviews/getPageviewMetrics.ts

@@ -48,7 +48,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
   );
 }
 
-async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  column: string,
+  filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, {
     ...filters,
@@ -65,7 +69,7 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
     select ${column} x, count(*) y
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at between {startDate:DateTime} and {endDate:DateTime}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}
       ${excludeDomain}
       ${filterQuery}
@@ -74,5 +78,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
     limit 100
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return { x: a.x, y: Number(a.y) };
+    });
+  });
 }

src/queries/analytics/pageviews/getPageviewStats.ts

@@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
   const { timezone = 'UTC', unit = 'day' } = filters;
   const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, {
@@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
         count(*) as y
       from website_event
       where website_id = {websiteId:UUID}
-        and created_at between {startDate:DateTime} and {endDate:DateTime}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
         ${filterQuery}
       group by t
@@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
     order by t
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return { x: a.x, y: Number(a.y) };
+    });
+  });
 }

src/queries/analytics/reports/getFunnel.ts

@@ -172,7 +172,7 @@ async function clickhouseQuery(
     );
   }
 
-  return rawQuery<{ level: number; count: number }[]>(
+  return rawQuery(
     `
     WITH level0 AS (
       select distinct session_id, url_path, referrer_path, created_at
@@ -201,7 +201,7 @@ async function clickhouseQuery(
   ).then(results => {
     return urls.map((a, i) => ({
       x: a,
-      y: results[i]?.count || 0,
+      y: Number(results[i]?.count) || 0,
       z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
     }));
   });

src/queries/analytics/reports/getInsights.ts

@@ -75,7 +75,7 @@ async function clickhouseQuery(
       ${parseFields(fields)}
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at between {startDate:DateTime} and {endDate:DateTime}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}
       ${filterQuery}
     ${parseGroupBy(fields)}

src/queries/analytics/reports/getRetention.ts

@@ -172,5 +172,15 @@ async function clickhouseQuery(
       startDate,
       endDate,
     },
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return {
+        date: a.date,
+        day: Number(a.day),
+        visitors: Number(a.visitors),
+        returnVisitors: Number(a.returnVisitors),
+        percentage: Number(a.percentage),
+      };
+    });
+  });
 }

src/queries/analytics/sessions/getSessionMetrics.ts

@@ -47,7 +47,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
   );
 }
 
-async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  column: string,
+  filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
   const { parseFilters, rawQuery } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, {
     ...filters,
@@ -63,7 +67,6 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
       ${includeCountry ? ', country' : ''}
     from website_event
     where website_id = {websiteId:UUID}
-      and created_at between {startDate:DateTime} and {endDate:DateTime}
       and event_type = {eventType:UInt32}
       ${filterQuery}
     group by x 
@@ -72,5 +75,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
     limit 100
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return { x: a.x, y: Number(a.y) };
+    });
+  });
 }

src/queries/analytics/sessions/getSessionStats.ts

@@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
   );
 }
 
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+  websiteId: string,
+  filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
   const { timezone = 'UTC', unit = 'day' } = filters;
   const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
   const { filterQuery, params } = await parseFilters(websiteId, {
@@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
         count(distinct session_id) as y
       from website_event
       where website_id = {websiteId:UUID}
-        and created_at between {startDate:DateTime} and {endDate:DateTime}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
         ${filterQuery}
       group by t
@@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
     order by t
     `,
     params,
-  );
+  ).then(a => {
+    return Object.values(a).map(a => {
+      return { x: a.x, y: Number(a.y) };
+    });
+  });
 }

src/queries/analytics/sessions/getSessions.ts

@@ -42,7 +42,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date) {
       city
     from website_event
     where website_id = {websiteId:UUID}
-    and created_at >= {startDate:DateTime}
+    and created_at >= {startDate:DateTime64}
     `,
     {
       websiteId,