remove parse session filters

src/lib/clickhouse.ts

@@ -2,11 +2,11 @@ import { ClickHouseClient, createClient } from '@clickhouse/client';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
-import { PageParams, QueryFilters, QueryOptions } from './types';
-import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
-import { fetchWebsite } from './load';
+import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
 import { maxDate } from './date';
+import { fetchWebsite } from './load';
 import { filtersToArray } from './params';
+import { PageParams, QueryFilters, QueryOptions } from './types';
 
 export const CLICKHOUSE_DATE_FORMATS = {
   second: '%Y-%m-%dT%H:%i:%S',
@@ -100,26 +100,6 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
   return query.join('\n');
 }
 
-function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
-  const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
-    if (column) {
-      if (EVENT_COLUMNS.includes(name)) {
-        arr.push(`and has(${column}, {${name}:String})`);
-
-        if (name === 'referrer') {
-          arr.push('and not has(referrer_domain, {websiteDomain:String})');
-        }
-      } else {
-        arr.push(`and ${mapFilter(column, operator, name)}`);
-      }
-    }
-
-    return arr;
-  }, []);
-
-  return query.join('\n');
-}
-
 function getDateQuery(filters: QueryFilters = {}) {
   const { startDate, endDate } = filters;
 
@@ -159,25 +139,6 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
   };
 }
 
-async function parseSessionFilters(
-  websiteId: string,
-  filters: QueryFilters = {},
-  options?: QueryOptions,
-) {
-  const website = await fetchWebsite(websiteId);
-
-  return {
-    filterQuery: getSessionFilterQuery(filters, options),
-    dateQuery: getDateQuery(filters),
-    params: {
-      ...getFilterParams(filters),
-      websiteId,
-      startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
-      websiteDomain: website.domain,
-    },
-  };
-}
-
 async function pagedQuery(
   query: string,
   queryParams: { [key: string]: any },
@@ -260,7 +221,6 @@ export default {
   getDateFormat,
   getFilterQuery,
   parseFilters,
-  parseSessionFilters,
   pagedQuery,
   findUnique,
   findFirst,

src/queries/analytics/getWebsiteStats.ts

@@ -74,8 +74,8 @@ async function clickhouseQuery(
     sql = `
     select
       sum(t.c) as "pageviews",
-      count(distinct t.session_id) as "visitors",
-      count(distinct t.visit_id) as "visits",
+      uniq(t.session_id) as "visitors",
+      uniq(t.visit_id) as "visits",
       sum(if(t.c = 1, 1, 0)) as "bounces",
       sum(max_time-min_time) as "totaltime"
     from (
@@ -96,16 +96,24 @@ async function clickhouseQuery(
   } else {
     sql = `
     select
-      sum(views) as "pageviews",
+      sum(t.c) as "pageviews",
       uniq(session_id) as "visitors",
       uniq(visit_id) as "visits",
-      sumIf(1, views = 1) as "bounces",
+      sumIf(1, t.c = 1) as "bounces",
       sum(max_time-min_time) as "totaltime"
-    from website_event_stats_hourly "website_event"
+    from (select
+            session_id,
+            visit_id,
+            sum(views) c,
+            min(min_time) min_time,
+            max(max_time) max_time
+        from umami.website_event_stats_hourly "website_event"
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}
-      ${filterQuery};
+      ${filterQuery}
+      group by session_id, visit_id
+    ) as t;
     `;
   }
 

src/queries/analytics/sessions/getSessionMetrics.ts

@@ -1,5 +1,5 @@
 import clickhouse from 'lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import { QueryFilters } from 'lib/types';
@@ -64,15 +64,34 @@ async function clickhouseQuery(
   offset: number = 0,
 ): Promise<{ x: string; y: number }[]> {
   const column = FILTER_COLUMNS[type] || type;
-  const { parseSessionFilters, rawQuery } = clickhouse;
-  const { filterQuery, params } = await parseSessionFilters(websiteId, {
+  const { parseFilters, rawQuery } = clickhouse;
+  const { filterQuery, params } = await parseFilters(websiteId, {
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });
   const includeCountry = column === 'city' || column === 'subdivision1';
 
-  return rawQuery(
-    `
+  let sql = '';
+
+  if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
+    sql = `
+    select
+      ${column} x,
+      count(distinct session_id) y
+      ${includeCountry ? ', country' : ''}
+    from website_event
+    where website_id = {websiteId:UUID}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_type = {eventType:UInt32}
+      ${filterQuery}
+    group by x 
+    ${includeCountry ? ', country' : ''}
+    order by y desc
+    limit ${limit}
+    offset ${offset}
+    `;
+  } else {
+    sql = `
     select
       ${column} x,
       uniq(session_id) y
@@ -87,9 +106,10 @@ async function clickhouseQuery(
     order by y desc
     limit ${limit}
     offset ${offset}
-    `,
-    params,
-  ).then(a => {
+    `;
+  }
+
+  return rawQuery(sql, params).then(a => {
     return Object.values(a).map(a => {
       return { x: a.x, y: Number(a.y), country: a.country };
     });

src/queries/analytics/sessions/getSessionStats.ts

@@ -1,7 +1,7 @@
 import clickhouse from 'lib/clickhouse';
+import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
-import { EVENT_TYPE } from 'lib/constants';
 import { QueryFilters } from 'lib/types';
 
 export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {
@@ -41,25 +41,24 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { timezone = 'UTC', unit = 'day' } = filters;
-  const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
-  const { filterQuery, params } = await parseSessionFilters(websiteId, {
+  const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
+  const { filterQuery, params } = await parseFilters(websiteId, {
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });
 
-  const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';
-  const columnQuery = unit === 'minute' ? 'count(distinct session_id)' : 'uniq(session_id)';
+  let sql = '';
 
-  return rawQuery(
-    `
+  if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
+    sql = `
     select
       ${getDateStringSQL('g.t', unit)} as x, 
       g.y as y
     from (
       select 
         ${getDateSQL('created_at', unit, timezone)} as t,
-        ${columnQuery} as y
-      from ${table} website_event
+        count(distinct session_id) as y
+      from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
@@ -67,9 +66,28 @@ async function clickhouseQuery(
       group by t
     ) as g
     order by t
-    `,
-    params,
-  ).then(result => {
+    `;
+  } else {
+    sql = `
+    select
+      ${getDateStringSQL('g.t', unit)} as x, 
+      g.y as y
+    from (
+      select 
+        ${getDateSQL('created_at', unit, timezone)} as t,
+        uniq(session_id) as y
+      from website_event_stats_hourly website_event
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and event_type = {eventType:UInt32}
+        ${filterQuery}
+      group by t
+    ) as g
+    order by t
+    `;
+  }
+
+  return rawQuery(sql, params).then(result => {
     return Object.values(result).map((a: any) => {
       return { x: a.x, y: Number(a.y) };
     });