update api to new CH columns

db/clickhouse/schema.sql

@@ -3,9 +3,9 @@ SET allow_experimental_object_type = 1;
 -- Create Event
 CREATE TABLE event
 (
-    website_id UInt32,
+    website_id UUID,
-    session_uuid UUID,
+    session_id UUID,
-    event_uuid Nullable(UUID),
+    event_id Nullable(UUID),
     --session
     hostname LowCardinality(String),
     browser LowCardinality(String),
@@ -27,9 +27,9 @@ CREATE TABLE event
         SETTINGS index_granularity = 8192;
 
 CREATE TABLE event_queue (
-    website_id UInt32,
+    website_id UUID,
-    session_uuid UUID,
+    session_id UUID,
-    event_uuid Nullable(UUID),
+    event_id Nullable(UUID),
     url String,
     referrer String,
     hostname LowCardinality(String),
@@ -52,9 +52,9 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro
        kafka_skip_broken_messages = 1;
 
 CREATE MATERIALIZED VIEW event_queue_mv TO event AS
-SELECT website_id,
+SELECT website_id UUID,
-    session_uuid,
+    session_id UUID,
-    event_uuid,
+    event_id,
     url,
     referrer,
     hostname,

lib/clickhouse.js

@@ -60,6 +60,10 @@ function getDateFormat(date) {
   return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
 }
 
+function getCommaSeparatedStringFormat(data, column) {
+  return data.map(obj => `'${obj[column]}'`).join(',');
+}
+
 function getBetweenDates(field, start_at, end_at) {
   return `${field} between ${getDateFormat(start_at)} 
     and ${getDateFormat(end_at)}`;
@@ -180,6 +184,7 @@ export default {
   getDateStringQuery,
   getDateQuery,
   getDateFormat,
+  getCommaSeparatedStringFormat,
   getBetweenDates,
   getFilterQuery,
   parseFilters,

lib/session.js

@@ -66,7 +66,7 @@ export async function getSession(req) {
 
     if (!sessionId) {
       try {
-        session = await createSession(websiteId, {
+        session = await createSession(websiteId, website_uuid, {
           session_uuid,
           hostname,
           browser,
@@ -98,6 +98,7 @@ export async function getSession(req) {
 
   return {
     website_id: websiteId,
+    website_uuid: website_uuid,
     session,
   };
 }

pages/api/collect.js

@@ -59,7 +59,7 @@ export default async (req, res) => {
   await useSession(req, res);
 
   const {
-    session: { website_id, session },
+    session: { website_id, website_uuid, session },
   } = req;
 
   const { type, payload } = getJsonBody(req);
@@ -73,9 +73,9 @@ export default async (req, res) => {
   const event_uuid = uuid();
 
   if (type === 'pageview') {
-    await savePageView(website_id, { session, url, referrer });
+    await savePageView(website_id, website_uuid, { session, url, referrer });
   } else if (type === 'event') {
-    await saveEvent(website_id, {
+    await saveEvent(website_id, website_uuid, {
       session,
       event_uuid,
       url,
@@ -87,7 +87,12 @@ export default async (req, res) => {
   }
 
   const token = createToken(
-    { website_id, session_id: session.session_id, session_uuid: session.session_uuid },
+    {
+      website_id,
+      website_uuid,
+      session_id: session.session_id,
+      session_uuid: session.session_uuid,
+    },
     secret(),
   );
 

pages/api/websites/[id]/active.js

@@ -14,8 +14,9 @@ export default async (req, res) => {
     const { id } = req.query;
 
     const websiteId = +id;
+    const website_uuid = id;
 
-    const result = await getActiveVisitors(websiteId);
+    const result = await getActiveVisitors(websiteId, website_uuid);
 
     return ok(res, result);
   }

pages/api/websites/[id]/events.js

@@ -21,10 +21,11 @@ export default async (req, res) => {
     }
 
     const websiteId = +id;
+    const website_uuid = id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 
-    const events = await getEventMetrics(websiteId, startDate, endDate, tz, unit, {
+    const events = await getEventMetrics(websiteId, website_uuid, startDate, endDate, tz, unit, {
       url,
       event_name,
     });

pages/api/websites/[id]/metrics.js

@@ -44,6 +44,7 @@ export default async (req, res) => {
     const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query;
 
     const websiteId = +id;
+    const website_uuid = id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 
@@ -106,7 +107,7 @@ export default async (req, res) => {
         query: type === 'query' && table !== 'event' ? true : undefined,
       };
 
-      const data = await getPageviewMetrics(websiteId, {
+      const data = await getPageviewMetrics(websiteId, website_uuid, {
         startDate,
         endDate,
         column,

pages/api/websites/[id]/pageviews.js

@@ -18,6 +18,7 @@ export default async (req, res) => {
       req.query;
 
     const websiteId = +id;
+    const website_uuid = id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 
@@ -26,7 +27,7 @@ export default async (req, res) => {
     }
 
     const [pageviews, sessions] = await Promise.all([
-      getPageviewStats(websiteId, {
+      getPageviewStats(websiteId, website_uuid, {
         start_at: startDate,
         end_at: endDate,
         timezone: tz,
@@ -41,7 +42,7 @@ export default async (req, res) => {
           country,
         },
       }),
-      getPageviewStats(websiteId, {
+      getPageviewStats(websiteId, website_uuid, {
         start_at: startDate,
         end_at: endDate,
         timezone: tz,

pages/api/websites/[id]/stats.js

@@ -14,6 +14,7 @@ export default async (req, res) => {
     const { id, start_at, end_at, url, referrer, os, browser, device, country } = req.query;
 
     const websiteId = +id;
+    const website_uuid = id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 
@@ -21,7 +22,7 @@ export default async (req, res) => {
     const prevStartDate = new Date(+start_at - distance);
     const prevEndDate = new Date(+end_at - distance);
 
-    const metrics = await getWebsiteStats(websiteId, {
+    const metrics = await getWebsiteStats(websiteId, website_uuid, {
       start_at: startDate,
       end_at: endDate,
       filters: {
@@ -33,7 +34,7 @@ export default async (req, res) => {
         country,
       },
     });
-    const prevPeriod = await getWebsiteStats(websiteId, {
+    const prevPeriod = await getWebsiteStats(websiteId, website_uuid, {
       start_at: prevStartDate,
       end_at: prevEndDate,
       filters: {

queries/analytics/event/getEventMetrics.js

@@ -36,7 +36,7 @@ async function relationalQuery(
 }
 
 async function clickhouseQuery(
-  website_id,
+  website_uuid,
   start_at,
   end_at,
   timezone = 'UTC',
@@ -44,7 +44,7 @@ async function clickhouseQuery(
   filters = {},
 ) {
   const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse;
-  const params = [website_id];
+  const params = [website_uuid];
 
   return rawQuery(
     `select

queries/analytics/event/getEvents.js

@@ -25,19 +25,23 @@ function relationalQuery(websites, start_at) {
 }
 
 function clickhouseQuery(websites, start_at) {
-  const { rawQuery, getDateFormat } = clickhouse;
+  const { rawQuery, getDateFormat, getCommaSeparatedStringFormat } = clickhouse;
 
   return rawQuery(
     `select
-      event_uuid,
+      event_id,
       website_id, 
-      session_uuid,
+      session_id,
       created_at,
       url,
       event_name
     from event
     where event_name != ''
-      and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'}
+      and ${
+        websites && websites.length > 0
+          ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})`
+          : '0 = 0'
+      }
       and created_at >= ${getDateFormat(start_at)}`,
   );
 }

queries/analytics/event/saveEvent.js

@@ -32,14 +32,14 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_
 }
 
 async function clickhouseQuery(
-  website_id,
+  website_uuid,
   { session: { country, ...sessionArgs }, event_uuid, url, event_name, event_data },
 ) {
   const { getDateFormat, sendMessage } = kafka;
 
   const params = {
     event_uuid,
-    website_id,
+    website_uuid,
     created_at: getDateFormat(new Date()),
     url: url?.substring(0, URL_LENGTH),
     event_name: event_name?.substring(0, EVENT_NAME_LENGTH),

queries/analytics/pageview/getPageviewMetrics.js

@@ -34,9 +34,9 @@ async function relationalQuery(website_id, { startDate, endDate, column, table,
   );
 }
 
-async function clickhouseQuery(website_id, { startDate, endDate, column, filters = {} }) {
+async function clickhouseQuery(website_uuid, { startDate, endDate, column, filters = {} }) {
   const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
-  const params = [website_id];
+  const params = [website_uuid];
   const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params);
 
   return rawQuery(

queries/analytics/pageview/getPageviewStats.js

@@ -45,11 +45,11 @@ async function relationalQuery(
 }
 
 async function clickhouseQuery(
-  website_id,
+  website_uuid,
   { start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} },
 ) {
   const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
-  const params = [website_id];
+  const params = [website_uuid];
   const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
 
   return rawQuery(
@@ -59,7 +59,7 @@ async function clickhouseQuery(
     from
       (select 
         ${getDateQuery('created_at', unit, timezone)} t,
-        count(${count !== '*' ? 'distinct session_uuid' : count}) y
+        count(${count !== '*' ? 'distinct session_id' : count}) y
       from event
       where event_name = ''
         and website_id= $1        

queries/analytics/pageview/getPageviews.js

@@ -25,15 +25,21 @@ async function relationalQuery(websites, start_at) {
 }
 
 async function clickhouseQuery(websites, start_at) {
+  const { getCommaSeparatedStringFormat } = clickhouse;
+
   return clickhouse.rawQuery(
     `select
         website_id,
-        session_uuid,
+        session_id,
         created_at,
         url
       from event
       where event_name = ''
-      and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'}
+      and ${
+        websites && websites.length > 0
+          ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})`
+          : '0 = 0'
+      }
       and created_at >= ${clickhouse.getDateFormat(start_at)}`,
   );
 }

queries/analytics/pageview/savePageView.js

@@ -22,12 +22,12 @@ async function relationalQuery(website_id, { session: { session_id }, url, refer
 }
 
 async function clickhouseQuery(
-  website_id,
+  website_uuid,
   { session: { country, ...sessionArgs }, url, referrer },
 ) {
   const { getDateFormat, sendMessage } = kafka;
   const params = {
-    website_id: website_id,
+    website_id: website_uuid,
     created_at: getDateFormat(new Date()),
     url: url?.substring(0, URL_LENGTH),
     referrer: referrer?.substring(0, URL_LENGTH),

queries/analytics/session/createSession.js

@@ -39,14 +39,14 @@ async function relationalQuery(website_id, data) {
 }
 
 async function clickhouseQuery(
-  website_id,
+  website_uuid,
   { session_uuid, hostname, browser, os, screen, language, country, device },
 ) {
   const { getDateFormat, sendMessage } = kafka;
 
   const params = {
     session_uuid,
-    website_id,
+    website_uuid,
     created_at: getDateFormat(new Date()),
     hostname,
     browser,

queries/analytics/stats/getActiveVisitors.js

@@ -23,12 +23,12 @@ async function relationalQuery(website_id) {
   );
 }
 
-async function clickhouseQuery(website_id) {
+async function clickhouseQuery(website_uuid) {
   const { rawQuery, getDateFormat } = clickhouse;
-  const params = [website_id];
+  const params = [website_uuid];
 
   return rawQuery(
-    `select count(distinct session_uuid) x
+    `select count(distinct session_id) x
     from event
     where website_id = $1
     and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`,

queries/analytics/stats/getWebsiteStats.js

@@ -41,19 +41,19 @@ async function relationalQuery(website_id, { start_at, end_at, filters = {} }) {
   );
 }
 
-async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) {
+async function clickhouseQuery(website_uuid, { start_at, end_at, filters = {} }) {
   const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
-  const params = [website_id];
+  const params = [website_uuid];
   const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
 
   return rawQuery(
     `select 
        sum(t.c) as "pageviews",
-       count(distinct t.session_uuid) as "uniques",
+       count(distinct t.session_id) as "uniques",
        sum(if(t.c = 1, 1, 0)) as "bounces",
        sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
      from (
-       select session_uuid,
+       select session_id,
          ${getDateQuery('created_at', 'day')} time_series,
          count(*) c,
          min(created_at) min_time,
@@ -64,7 +64,7 @@ async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) {
          and ${getBetweenDates('created_at', start_at, end_at)}
          ${pageviewQuery}
          ${sessionQuery}
-       group by session_uuid, time_series
+       group by session_id, time_series
      ) t;`,
     params,
   );