route dashboard queries based on filters selected

src/lib/clickhouse.ts

@@ -3,7 +3,7 @@ import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
 import { PageParams, QueryFilters, QueryOptions } from './types';
-import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
+import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
 import { fetchWebsite } from './load';
 import { maxDate } from './date';
 import { filtersToArray } from './params';
@@ -100,6 +100,26 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
   return query.join('\n');
 }
 
+function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
+  const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
+    if (column) {
+      if (EVENT_COLUMNS.includes(name)) {
+        arr.push(`and has(${column}, {${name}:String})`);
+
+        if (name === 'referrer') {
+          arr.push('and not has(referrer_domain, {websiteDomain:String})');
+        }
+      } else {
+        arr.push(`and ${mapFilter(column, operator, name)}`);
+      }
+    }
+
+    return arr;
+  }, []);
+
+  return query.join('\n');
+}
+
 function getDateQuery(filters: QueryFilters = {}) {
   const { startDate, endDate } = filters;
 
@@ -139,6 +159,25 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
   };
 }
 
+async function parseSessionFilters(
+  websiteId: string,
+  filters: QueryFilters = {},
+  options?: QueryOptions,
+) {
+  const website = await fetchWebsite(websiteId);
+
+  return {
+    filterQuery: getSessionFilterQuery(filters, options),
+    dateQuery: getDateQuery(filters),
+    params: {
+      ...getFilterParams(filters),
+      websiteId,
+      startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
+      websiteDomain: website.domain,
+    },
+  };
+}
+
 async function pagedQuery(
   query: string,
   queryParams: { [key: string]: any },
@@ -221,6 +260,7 @@ export default {
   getDateFormat,
   getFilterQuery,
   parseFilters,
+  parseSessionFilters,
   pagedQuery,
   findUnique,
   findFirst,

src/lib/constants.ts

@@ -33,16 +33,7 @@ export const FILTER_REFERRERS = 'filter-referrers';
 export const FILTER_PAGES = 'filter-pages';
 
 export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
-export const EVENT_COLUMNS = [
+export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event'];
-  'url',
-  'entry',
-  'exit',
-  'referrer',
-  'title',
-  'query',
-  'event',
-  'host',
-];
 
 export const SESSION_COLUMNS = [
   'browser',
@@ -58,8 +49,8 @@ export const SESSION_COLUMNS = [
 
 export const FILTER_COLUMNS = {
   url: 'url_path',
-  entry: 'entry_url',
+  entry: 'url_path',
-  exit: 'exit_url',
+  exit: 'url_path',
   referrer: 'referrer_domain',
   host: 'hostname',
   title: 'page_title',

src/queries/analytics/events/getEventMetrics.ts

@@ -1,8 +1,8 @@
-import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
-import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
-import { WebsiteEventMetric, QueryFilters } from 'lib/types';
 import { EVENT_TYPE } from 'lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
+import { QueryFilters, WebsiteEventMetric } from 'lib/types';
 
 export async function getEventMetrics(
   ...args: [websiteId: string, filters: QueryFilters]
@@ -51,8 +51,24 @@ async function clickhouseQuery(
     eventType: EVENT_TYPE.customEvent,
   });
 
-  return rawQuery(
+  let sql = '';
-    `
+
+  if (filterQuery) {
+    sql = `
+    select
+      event_name x,
+      ${getDateSQL('created_at', unit, timezone)} t,
+      count(*) y
+    from website_event
+    where website_id = {websiteId:UUID}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_type = {eventType:UInt32}
+      ${filterQuery}
+    group by x, t
+    order by t
+    `;
+  } else {
+    sql = `
     select
       event_name x,
       ${getDateSQL('created_at', unit, timezone)} t,
@@ -64,13 +80,13 @@ async function clickhouseQuery(
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
-      ${filterQuery}
     ) as g
     group by x, t
     order by t
-    `,
+    `;
-    params,
+  }
-  ).then(a => {
+
+  return rawQuery(sql, params).then(a => {
     return Object.values(a).map(a => {
       return { x: a.x, t: a.t, y: Number(a.y) };
     });

src/queries/analytics/getWebsiteStats.ts

@@ -3,6 +3,7 @@ import { EVENT_TYPE } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import { QueryFilters } from 'lib/types';
+import { EVENT_COLUMNS } from 'lib/constants';
 
 export async function getWebsiteStats(
   ...args: [websiteId: string, filters: QueryFilters]
@@ -67,30 +68,33 @@ async function clickhouseQuery(
     eventType: EVENT_TYPE.pageView,
   });
 
-  return rawQuery(
+  let sql = '';
-    // `
+
-    // select
+  if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
-    //   sum(t.c) as "pageviews",
+    sql = `
-    //   count(distinct t.session_id) as "visitors",
+    select
-    //   count(distinct t.visit_id) as "visits",
+      sum(t.c) as "pageviews",
-    //   sum(if(t.c = 1, 1, 0)) as "bounces",
+      count(distinct t.session_id) as "visitors",
-    //   sum(max_time-min_time) as "totaltime"
+      count(distinct t.visit_id) as "visits",
-    // from (
+      sum(if(t.c = 1, 1, 0)) as "bounces",
-    //   select
+      sum(max_time-min_time) as "totaltime"
-    //     session_id,
+    from (
-    //     visit_id,
+      select
-    //     count(*) c,
+        session_id,
-    //     min(created_at) min_time,
+        visit_id,
-    //     max(created_at) max_time
+        count(*) c,
-    //   from website_event
+        min(created_at) min_time,
-    //   where website_id = {websiteId:UUID}
+        max(created_at) max_time
-    //     and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      from website_event
-    //     and event_type = {eventType:UInt32}
+      where website_id = {websiteId:UUID}
-    //     ${filterQuery}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-    //   group by session_id, visit_id
+        and event_type = {eventType:UInt32}
-    // ) as t;
+        ${filterQuery}
-    // `,
+      group by session_id, visit_id
-    `
+    ) as t;
+    `;
+  } else {
+    sql = `
     select
       sum(views) as "pageviews",
       uniq(session_id) as "visitors",
@@ -102,9 +106,10 @@ async function clickhouseQuery(
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}
       ${filterQuery};
-    `,
+    `;
-    params,
+  }
-  ).then(result => {
+
+  return rawQuery(sql, params).then(result => {
     return Object.values(result).map((a: any) => {
       return {
         pageviews: Number(a.pageviews),

src/queries/analytics/pageviews/getPageviewMetrics.ts

@@ -1,5 +1,5 @@
 import clickhouse from 'lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import { QueryFilters } from 'lib/types';
@@ -91,46 +91,66 @@ async function clickhouseQuery(
   });
 
   let excludeDomain = '';
-  let groupByQuery = '';
+  let sql = '';
 
-  if (column === 'referrer_domain') {
+  if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
-    excludeDomain = `and t != {websiteDomain:String} and t != ''`;
+    let entryExitQuery = '';
-  }
 
-  let columnQuery = `arrayJoin(${column})`;
+    if (column === 'referrer_domain') {
+      excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
+    }
 
-  if (type === 'entry') {
+    if (type === 'entry' || type === 'exit') {
-    columnQuery = `visit_id x, argMinMerge(${column})`;
+      const aggregrate = type === 'entry' ? 'min' : 'max';
-  }
 
-  if (type === 'exit') {
+      entryExitQuery = `
-    columnQuery = `visit_id x, argMaxMerge(${column})`;
+      JOIN (select visit_id,
-  }
+          ${aggregrate}(created_at) target_created_at
+      from website_event
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and event_type = {eventType:UInt32}
+      group by visit_id) x
+      ON x.visit_id = website_event.visit_id
+          and x.target_created_at = website_event.created_at`;
+    }
 
-  if (type === 'entry' || type === 'exit') {
+    sql = `
-    groupByQuery = 'group by x';
+    select ${column} x, count(*) y
-  }
+    from website_event
+    ${entryExitQuery}
+    where website_id = {websiteId:UUID}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_type = {eventType:UInt32}
+      ${excludeDomain}
+      ${filterQuery}
+    group by x
+    order by y desc
+    limit ${limit}
+    offset ${offset}
+    `;
+  } else {
+    let groupByQuery = '';
 
-  // let excludeDomain = '';
+    if (column === 'referrer_domain') {
-  // if (column === 'referrer_domain') {
+      excludeDomain = `and t != {websiteDomain:String} and t != ''`;
-  //   excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
+    }
-  // }
 
-  return rawQuery(
+    let columnQuery = `arrayJoin(${column})`;
-    // `
+
-    //   select ${column} x, count(*) y
+    if (type === 'entry') {
-    //   from website_event
+      columnQuery = `visit_id x, argMinMerge(entry_url)`;
-    //   where website_id = {websiteId:UUID}
+    }
-    //     and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+
-    //     and event_type = {eventType:UInt32}
+    if (type === 'exit') {
-    //     ${excludeDomain}
+      columnQuery = `visit_id x, argMaxMerge(exit_url)`;
-    //     ${filterQuery}
+    }
-    //   group by x
+
-    //   order by y desc
+    if (type === 'entry' || type === 'exit') {
-    //   limit ${limit}
+      groupByQuery = 'group by x';
-    //   offset ${offset}
+    }
-    //   `,
+
-    `
+    sql = `
     select g.t as x,
       count(*) as y
     from (
@@ -146,9 +166,10 @@ async function clickhouseQuery(
     order by y desc
     limit ${limit}
     offset ${offset}
-    `,
+    `;
-    params,
+  }
-  ).then((result: any) => {
+
+  return rawQuery(sql, params).then((result: any) => {
     return Object.values(result).map((a: any) => {
       return { x: a.x, y: Number(a.y) };
     });

src/queries/analytics/pageviews/getPageviewStats.ts

@@ -1,7 +1,7 @@
 import clickhouse from 'lib/clickhouse';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
-import { EVENT_TYPE } from 'lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';
 import { QueryFilters } from 'lib/types';
 
 export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
@@ -47,36 +47,18 @@ async function clickhouseQuery(
     eventType: EVENT_TYPE.pageView,
   });
 
-  const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';
+  let sql = '';
-  const columnQuery = unit === 'minute' ? 'count(*)' : 'sum(views)';
 
-  return rawQuery(
+  if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
-    // `
+    sql = `
-    // select
-    //   ${getDateStringSQL('g.t', unit)} as x,
-    //   g.y as y
-    // from (
-    //   select
-    //     ${getDateSQL('created_at', unit, timezone)} as t,
-    //     count(*) as y
-    //   from website_event
-    //   where website_id = {websiteId:UUID}
-    //     and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-    //     and event_type = {eventType:UInt32}
-    //     ${filterQuery}
-    //   group by t
-    // ) as g
-    // order by t
-    // `,
-    `
     select
       ${getDateStringSQL('g.t', unit)} as x,
       g.y as y
     from (
       select
         ${getDateSQL('created_at', unit, timezone)} as t,
-        ${columnQuery} as y
+        count(*) as y
-      from ${table} website_event
+      from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
@@ -84,9 +66,28 @@ async function clickhouseQuery(
       group by t
     ) as g
     order by t
-    `,
+    `;
-    params,
+  } else {
-  ).then(result => {
+    sql = `
+    select
+      ${getDateStringSQL('g.t', unit)} as x,
+      g.y as y
+    from (
+      select
+        ${getDateSQL('created_at', unit, timezone)} as t,
+        sum(views)as y
+      from website_event_stats_hourly website_event
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and event_type = {eventType:UInt32}
+        ${filterQuery}
+      group by t
+    ) as g
+    order by t
+    `;
+  }
+
+  return rawQuery(sql, params).then(result => {
     return Object.values(result).map((a: any) => {
       return { x: a.x, y: Number(a.y) };
     });

src/queries/analytics/sessions/getSessionMetrics.ts

@@ -64,8 +64,8 @@ async function clickhouseQuery(
   offset: number = 0,
 ): Promise<{ x: string; y: number }[]> {
   const column = FILTER_COLUMNS[type] || type;
-  const { parseFilters, rawQuery } = clickhouse;
+  const { parseSessionFilters, rawQuery } = clickhouse;
-  const { filterQuery, params } = await parseFilters(websiteId, {
+  const { filterQuery, params } = await parseSessionFilters(websiteId, {
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });

src/queries/analytics/sessions/getSessionStats.ts

@@ -41,8 +41,8 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { timezone = 'UTC', unit = 'day' } = filters;
-  const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
+  const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
-  const { filterQuery, params } = await parseFilters(websiteId, {
+  const { filterQuery, params } = await parseSessionFilters(websiteId, {
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });