Merge pull request #1563 from umami-software/feat/um-66-update-clickhouse-schema-uuid

Feat/um 66 update clickhouse schema UUID

components/metrics/WebsiteHeader.js

@@ -17,8 +17,8 @@ export default function WebsiteHeader({ websiteId, title, domain, showLink = fal
       <Favicon domain={domain} />
       <Link
         className={styles.titleLink}
-        href="/website/[...id]"
+        href="/websites/[...id]"
-        as={`/website/${websiteId}/${title}`}
+        as={`/websites/${websiteId}/${title}`}
       >
         <OverflowText tooltipId={`${websiteId}-title`}>{title}</OverflowText>
       </Link>
@@ -41,8 +41,8 @@ export default function WebsiteHeader({ websiteId, title, domain, showLink = fal
           <RefreshButton websiteId={websiteId} />
           {showLink && (
             <Link
-              href="/website/[...id]"
+              href="/websites/[...id]"
-              as={`/website/${websiteId}/${title}`}
+              as={`/websites/${websiteId}/${title}`}
               className={styles.link}
               icon={<Arrow />}
               size="small"

components/pages/WebsiteList.js

@@ -27,7 +27,7 @@ export default function WebsiteList({ websites, showCharts, limit }) {
   const ordered = useMemo(
     () =>
       websites
-        .map(website => ({ ...website, order: websiteOrder.indexOf(website.id) || 0 }))
+        .map(website => ({ ...website, order: websiteOrder.indexOf(website.websiteUuid) || 0 }))
         .sort(firstBy('order')),
     [websites, websiteOrder],
   );
@@ -46,11 +46,11 @@ export default function WebsiteList({ websites, showCharts, limit }) {
 
   return (
     <div>
-      {ordered.map(({ id, name, domain }, index) =>
+      {ordered.map(({ websiteUuid, name, domain }, index) =>
         index < limit ? (
-          <div key={id} className={styles.website}>
+          <div key={websiteUuid} className={styles.website}>
             <WebsiteChart
-              websiteId={id}
+              websiteId={websiteUuid}
               title={name}
               domain={domain}
               showChart={showCharts}

components/settings/WebsiteSettings.js

@@ -46,7 +46,7 @@ export default function WebsiteSettings() {
           icon={<LinkIcon />}
           size="small"
           tooltip={<FormattedMessage id="message.get-share-url" defaultMessage="Get share URL" />}
-          tooltipId={`button-share-${row.id}`}
+          tooltipId={`button-share-${row.websiteUuid}`}
           onClick={() => setShowUrl(row)}
         />
       )}
@@ -56,42 +56,46 @@ export default function WebsiteSettings() {
         tooltip={
           <FormattedMessage id="message.get-tracking-code" defaultMessage="Get tracking code" />
         }
-        tooltipId={`button-code-${row.id}`}
+        tooltipId={`button-code-${row.websiteUuid}`}
         onClick={() => setShowCode(row)}
       />
       <Button
         icon={<Pen />}
         size="small"
         tooltip={<FormattedMessage id="label.edit" defaultMessage="Edit" />}
-        tooltipId={`button-edit-${row.id}`}
+        tooltipId={`button-edit-${row.websiteUuid}`}
         onClick={() => setEditWebsite(row)}
       />
       <Button
         icon={<Reset />}
         size="small"
         tooltip={<FormattedMessage id="label.reset" defaultMessage="Reset" />}
-        tooltipId={`button-reset-${row.id}`}
+        tooltipId={`button-reset-${row.websiteUuid}`}
         onClick={() => setResetWebsite(row)}
       />
       <Button
         icon={<Trash />}
         size="small"
         tooltip={<FormattedMessage id="label.delete" defaultMessage="Delete" />}
-        tooltipId={`button-delete-${row.id}`}
+        tooltipId={`button-delete-${row.websiteUuid}`}
         onClick={() => setDeleteWebsite(row)}
       />
     </ButtonLayout>
   );
 
-  const DetailsLink = ({ id, name, domain }) => (
+  const DetailsLink = ({ websiteUuid, name, domain }) => (
-    <Link className={styles.detailLink} href="/website/[...id]" as={`/website/${id}/${name}`}>
+    <Link
+      className={styles.detailLink}
+      href="/websites/[...id]"
+      as={`/websites/${websiteUuid}/${name}`}
+    >
       <Favicon domain={domain} />
-      <OverflowText tooltipId={`${id}-name`}>{name}</OverflowText>
+      <OverflowText tooltipId={`${websiteUuid}-name`}>{name}</OverflowText>
     </Link>
   );
 
-  const Domain = ({ domain, id }) => (
+  const Domain = ({ domain, websiteUuid }) => (
-    <OverflowText tooltipId={`${id}-domain`}>{domain}</OverflowText>
+    <OverflowText tooltipId={`${websiteUuid}-domain`}>{domain}</OverflowText>
   );
 
   const adminColumns = [
@@ -199,7 +203,7 @@ export default function WebsiteSettings() {
           title={<FormattedMessage id="label.reset-website" defaultMessage="Reset statistics" />}
         >
           <ResetForm
-            values={{ type: 'websites', id: resetWebsite.id, name: resetWebsite.name }}
+            values={{ type: 'websites', id: resetWebsite.websiteUuid, name: resetWebsite.name }}
             onSave={handleSave}
             onClose={handleClose}
           />
@@ -210,7 +214,7 @@ export default function WebsiteSettings() {
           title={<FormattedMessage id="label.delete-website" defaultMessage="Delete website" />}
         >
           <DeleteForm
-            values={{ type: 'websites', id: deleteWebsite.id, name: deleteWebsite.name }}
+            values={{ type: 'websites', id: deleteWebsite.websiteUuid, name: deleteWebsite.name }}
             onSave={handleSave}
             onClose={handleClose}
           />

db/clickhouse/schema.sql

@@ -3,9 +3,9 @@ SET allow_experimental_object_type = 1;
 -- Create Event
 CREATE TABLE event
 (
-    website_id UInt32,
+    website_id UUID,
-    session_uuid UUID,
+    session_id UUID,
-    event_uuid Nullable(UUID),
+    event_id Nullable(UUID),
     --session
     hostname LowCardinality(String),
     browser LowCardinality(String),
@@ -23,13 +23,13 @@ CREATE TABLE event
     created_at DateTime('UTC')
 )
     engine = MergeTree
-        ORDER BY (website_id, session_uuid, created_at)
+        ORDER BY (website_id, session_id, created_at)
         SETTINGS index_granularity = 8192;
 
 CREATE TABLE event_queue (
-    website_id UInt32,
+    website_id UUID,
-    session_uuid UUID,
+    session_id UUID,
-    event_uuid Nullable(UUID),
+    event_id Nullable(UUID),
     url String,
     referrer String,
     hostname LowCardinality(String),
@@ -44,7 +44,7 @@ CREATE TABLE event_queue (
     created_at DateTime('UTC')
 )
 ENGINE = Kafka
-SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list
+SETTINGS kafka_broker_list = 'dev-01.umami.dev:9092,dev-01.umami.dev:9093,dev-01.umami.dev:9094', -- input broker list
        kafka_topic_list = 'event',
        kafka_group_name = 'event_consumer_group',
        kafka_format = 'JSONEachRow',
@@ -53,8 +53,8 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro
 
 CREATE MATERIALIZED VIEW event_queue_mv TO event AS
 SELECT website_id,
-    session_uuid,
+    session_id,
-    event_uuid,
+    event_id,
     url,
     referrer,
     hostname,

lib/clickhouse.js

@@ -60,6 +60,10 @@ function getDateFormat(date) {
   return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
 }
 
+function getCommaSeparatedStringFormat(data) {
+  return data.map(a => `'${a}'`).join(',');
+}
+
 function getBetweenDates(field, start_at, end_at) {
   return `${field} between ${getDateFormat(start_at)} 
     and ${getDateFormat(end_at)}`;
@@ -180,6 +184,7 @@ export default {
   getDateStringQuery,
   getDateQuery,
   getDateFormat,
+  getCommaSeparatedStringFormat,
   getBetweenDates,
   getFilterQuery,
   parseFilters,

pages/api/websites/[id]/active.js

@@ -11,9 +11,7 @@ export default async (req, res) => {
       return unauthorized(res);
     }
 
-    const { id } = req.query;
+    const { id: websiteId } = req.query;
-
-    const websiteId = +id;
 
     const result = await getActiveVisitors(websiteId);
 

pages/api/websites/[id]/events.js

@@ -14,13 +14,11 @@ export default async (req, res) => {
       return unauthorized(res);
     }
 
-    const { id, start_at, end_at, unit, tz, url, event_name } = req.query;
+    const { id: websiteId, start_at, end_at, unit, tz, url, event_name } = req.query;
 
     if (!moment.tz.zone(tz) || !unitTypes.includes(unit)) {
       return badRequest(res);
     }
-
-    const websiteId = +id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 

pages/api/websites/[id]/index.js

@@ -1,14 +1,10 @@
-import { methodNotAllowed, ok, unauthorized, getRandomChars } from 'next-basics';
-import { deleteWebsite, getAccount, getWebsite, updateWebsite } from 'queries';
 import { allowQuery } from 'lib/auth';
 import { useAuth, useCors } from 'lib/middleware';
-import { validate } from 'uuid';
+import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics';
+import { deleteWebsite, getAccount, getWebsite, getWebsiteByUuid, updateWebsite } from 'queries';
 
 export default async (req, res) => {
-  const { id } = req.query;
+  const { id: websiteId } = req.query;
-
-  const websiteId = +id;
-  const where = validate(id) ? { websiteUuid: id } : { id: +id };
 
   if (req.method === 'GET') {
     await useCors(req, res);
@@ -17,7 +13,7 @@ export default async (req, res) => {
       return unauthorized(res);
     }
 
-    const website = await getWebsite(where);
+    const website = await getWebsiteByUuid(websiteId);
 
     return ok(res, website);
   }
@@ -33,7 +29,7 @@ export default async (req, res) => {
       account = await getAccount({ accountUuid });
     }
 
-    const website = await getWebsite(where);
+    const website = await getWebsite(websiteId);
 
     const shareId = enable_share_url ? website.shareId || getRandomChars(8) : null;
 
@@ -48,7 +44,7 @@ export default async (req, res) => {
         shareId: shareId,
         userId: account ? account.id : +owner,
       },
-      where,
+      { websiteUuid: websiteId },
     );
 
     return ok(res);

pages/api/websites/[id]/metrics.js

@@ -1,8 +1,8 @@
-import { getPageviewMetrics, getSessionMetrics, getWebsiteById } from 'queries';
-import { ok, methodNotAllowed, unauthorized, badRequest } from 'next-basics';
 import { allowQuery } from 'lib/auth';
-import { useCors } from 'lib/middleware';
 import { FILTER_IGNORED } from 'lib/constants';
+import { useCors } from 'lib/middleware';
+import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';
+import { getPageviewMetrics, getSessionMetrics, getWebsiteByUuid } from 'queries';
 
 const sessionColumns = ['browser', 'os', 'device', 'screen', 'country', 'language'];
 const pageviewColumns = ['url', 'referrer', 'query'];
@@ -41,9 +41,19 @@ export default async (req, res) => {
       return unauthorized(res);
     }
 
-    const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query;
+    const {
+      id: websiteId,
+      type,
+      start_at,
+      end_at,
+      url,
+      referrer,
+      os,
+      browser,
+      device,
+      country,
+    } = req.query;
 
-    const websiteId = +id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 
@@ -83,7 +93,7 @@ export default async (req, res) => {
       let domain;
 
       if (type === 'referrer') {
-        const website = await getWebsiteById(websiteId);
+        const website = await getWebsiteByUuid(websiteId);
 
         if (!website) {
           return badRequest(res);

pages/api/websites/[id]/pageviews.js

@@ -14,10 +14,20 @@ export default async (req, res) => {
       return unauthorized(res);
     }
 
-    const { id, start_at, end_at, unit, tz, url, referrer, os, browser, device, country } =
+    const {
-      req.query;
+      id: websiteId,
+      start_at,
+      end_at,
+      unit,
+      tz,
+      url,
+      referrer,
+      os,
+      browser,
+      device,
+      country,
+    } = req.query;
 
-    const websiteId = +id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 

pages/api/websites/[id]/reset.js

@@ -3,8 +3,7 @@ import { methodNotAllowed, ok, unauthorized } from 'next-basics';
 import { allowQuery } from 'lib/auth';
 
 export default async (req, res) => {
-  const { id } = req.query;
+  const { id: websiteId } = req.query;
-  const websiteId = +id;
 
   if (req.method === 'POST') {
     if (!(await allowQuery(req))) {

pages/api/websites/[id]/stats.js

@@ -11,9 +11,18 @@ export default async (req, res) => {
       return unauthorized(res);
     }
 
-    const { website_id, start_at, end_at, url, referrer, os, browser, device, country } = req.query;
+    const {
+      id: websiteId,
+      start_at,
+      end_at,
+      url,
+      referrer,
+      os,
+      browser,
+      device,
+      country,
+    } = req.query;
 
-    const websiteId = +website_id;
     const startDate = new Date(+start_at);
     const endDate = new Date(+end_at);
 

queries/admin/website/deleteWebsite.js

@@ -1,27 +1,27 @@
 import prisma from 'lib/prisma';
 import redis, { DELETED } from 'lib/redis';
-import { getWebsiteById } from 'queries';
+import { getWebsiteByUuid } from 'queries';
 
 export async function deleteWebsite(websiteId) {
   const { client, transaction } = prisma;
 
-  const { websiteUuid } = await getWebsiteById(websiteId);
+  const { websiteUuid } = await getWebsiteByUuid(websiteId);
 
   return transaction([
     client.pageview.deleteMany({
-      where: { session: { website: { id: websiteId } } },
+      where: { session: { website: { websiteUuid: websiteId } } },
     }),
     client.eventData.deleteMany({
-      where: { event: { session: { website: { id: websiteId } } } },
+      where: { event: { session: { website: { websiteUuid: websiteId } } } },
     }),
     client.event.deleteMany({
-      where: { session: { website: { id: websiteId } } },
+      where: { session: { website: { websiteUuid: websiteId } } },
     }),
     client.session.deleteMany({
-      where: { website: { id: websiteId } },
+      where: { website: { websiteUuid: websiteId } },
     }),
     client.website.delete({
-      where: { id: websiteId },
+      where: { websiteUuid: websiteId },
     }),
   ]).then(async res => {
     if (redis.client) {

queries/admin/website/resetWebsite.js

@@ -5,16 +5,16 @@ export async function resetWebsite(websiteId) {
 
   return transaction([
     client.pageview.deleteMany({
-      where: { session: { website: { id: websiteId } } },
+      where: { session: { website: { websiteUuid: websiteId } } },
     }),
     client.eventData.deleteMany({
-      where: { event: { session: { website: { id: websiteId } } } },
+      where: { event: { session: { website: { websiteUuid: websiteId } } } },
     }),
     client.event.deleteMany({
-      where: { session: { website: { id: websiteId } } },
+      where: { session: { website: { websiteUuid: websiteId } } },
     }),
     client.session.deleteMany({
-      where: { website: { id: websiteId } },
+      where: { website: { websiteUuid: websiteId } },
     }),
   ]);
 }

queries/analytics/event/getEventMetrics.js

@@ -23,11 +23,13 @@ async function relationalQuery(
   return rawQuery(
     `select
       event_name x,
-      ${getDateQuery('created_at', unit, timezone)} t,
+      ${getDateQuery('event.created_at', unit, timezone)} t,
       count(*) y
     from event
-    where website_id=$1
+      join website 
-    and created_at between $2 and $3
+        on event.website_id = website.website_id
+    where website_uuid='${websiteId}'
+      and event.created_at between $2 and $3
     ${getFilterQuery('event', filters, params)}
     group by 1, 2
     order by 2`,

queries/analytics/event/getEvents.js

@@ -25,19 +25,23 @@ function relationalQuery(websites, start_at) {
 }
 
 function clickhouseQuery(websites, start_at) {
-  const { rawQuery, getDateFormat } = clickhouse;
+  const { rawQuery, getDateFormat, getCommaSeparatedStringFormat } = clickhouse;
 
   return rawQuery(
     `select
-      event_uuid,
+      event_id,
       website_id, 
-      session_uuid,
+      session_id,
       created_at,
       url,
       event_name
     from event
     where event_name != ''
-      and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'}
+      and ${
+        websites && websites.length > 0
+          ? `website_id in (${getCommaSeparatedStringFormat(websites)})`
+          : '0 = 0'
+      }
       and created_at >= ${getDateFormat(start_at)}`,
   );
 }

queries/analytics/event/saveEvent.js

@@ -38,8 +38,8 @@ async function clickhouseQuery(
   const { getDateFormat, sendMessage } = kafka;
 
   const params = {
-    session_uuid: sessionUuid,
+    session_id: sessionUuid,
-    event_uuid: eventUuid,
+    event_id: eventUuid,
     website_id: websiteId,
     created_at: getDateFormat(new Date()),
     url: url?.substring(0, URL_LENGTH),

queries/analytics/pageview/getPageviewMetrics.js

@@ -22,8 +22,9 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, f
   return rawQuery(
     `select ${column} x, count(*) y
     from ${table}
+      ${` join website on ${table}.website_id = website.website_id`}
       ${joinSession}
-    where ${table}.website_id=$1
+    where website.website_uuid='${websiteId}'
       and ${table}.created_at between $2 and $3
       ${pageviewQuery}
       ${joinSession && sessionQuery}

queries/analytics/pageview/getPageviewParams.js

@@ -22,8 +22,9 @@ async function relationalQuery(websiteId, start_at, end_at, column, table, filte
     `select url x,
       count(*) y
     from ${table}
+      ${` join website on ${table}.website_id = website.website_id`}
       ${joinSession}
-    where ${table}.website_id=$1
+    where website.website_uuid='${websiteId}'
       and ${table}.created_at between $2 and $3
       and ${table}.url like '%?%'
       ${pageviewQuery}

queries/analytics/pageview/getPageviewStats.js

@@ -34,8 +34,10 @@ async function relationalQuery(
     `select ${getDateQuery('pageview.created_at', unit, timezone)} t,
         count(${count !== '*' ? `${count}${sessionKey}` : count}) y
       from pageview
+        join website
+            on pageview.website_id = website.website_id
         ${joinSession}
-      where pageview.website_id=$1
+      where website.website_uuid='${websiteId}'
         and pageview.created_at between $2 and $3
         ${pageviewQuery}
         ${sessionQuery}
@@ -59,7 +61,7 @@ async function clickhouseQuery(
     from
       (select 
         ${getDateQuery('created_at', unit, timezone)} t,
-        count(${count !== '*' ? 'distinct session_uuid' : count}) y
+        count(${count !== '*' ? 'distinct session_id' : count}) y
       from event
       where event_name = ''
         and website_id= $1        

queries/analytics/pageview/getPageviews.js

@@ -25,15 +25,21 @@ async function relationalQuery(websites, start_at) {
 }
 
 async function clickhouseQuery(websites, start_at) {
+  const { getCommaSeparatedStringFormat } = clickhouse;
+
   return clickhouse.rawQuery(
     `select
         website_id,
-        session_uuid,
+        session_id,
         created_at,
         url
       from event
       where event_name = ''
-      and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'}
+      and ${
+        websites && websites.length > 0
+          ? `website_id in (${getCommaSeparatedStringFormat(websites)})`
+          : '0 = 0'
+      }
       and created_at >= ${clickhouse.getDateFormat(start_at)}`,
   );
 }

queries/analytics/session/getSessionMetrics.js

@@ -20,8 +20,10 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters =
     where x.session_id in (
       select pageview.session_id
       from pageview
+        join website 
+          on pageview.website_id = website.website_id
         ${joinSession}
-      where pageview.website_id=$1
+      where website.website_uuid='${websiteId}'
       and pageview.created_at between $2 and $3
       ${pageviewQuery}
       ${sessionQuery}

queries/analytics/stats/getActiveVisitors.js

@@ -17,8 +17,10 @@ async function relationalQuery(websiteId) {
   return prisma.rawQuery(
     `select count(distinct session_id) x
     from pageview
-    where website_id = $1
+      join website 
-    and created_at >= $2`,
+        on pageview.website_id = website.website_id
+    where website.website_uuid = '${websiteId}'
+    and pageview.created_at >= $2`,
     params,
   );
 }
@@ -28,7 +30,7 @@ async function clickhouseQuery(websiteId) {
   const params = [websiteId];
 
   return rawQuery(
-    `select count(distinct session_uuid) x
+    `select count(distinct session_id) x
     from event
     where website_id = $1
     and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`,

queries/analytics/stats/getWebsiteStats.js

@@ -30,11 +30,13 @@ async function relationalQuery(websiteId, { start_at, end_at, filters = {} }) {
           count(*) c,
           ${getTimestampInterval('pageview.created_at')} as "time"
         from pageview
+          join website 
+            on pageview.website_id = website.website_id
           ${joinSession}
-        where pageview.website_id=$1
+        where website.website_uuid='${websiteId}'
-        and pageview.created_at between $2 and $3
+          and pageview.created_at between $2 and $3
-        ${pageviewQuery}
+          ${pageviewQuery}
-        ${sessionQuery}
+          ${sessionQuery}
         group by 1, 2
      ) t`,
     params,
@@ -49,11 +51,11 @@ async function clickhouseQuery(websiteId, { start_at, end_at, filters = {} }) {
   return rawQuery(
     `select 
        sum(t.c) as "pageviews",
-       count(distinct t.session_uuid) as "uniques",
+       count(distinct t.session_id) as "uniques",
        sum(if(t.c = 1, 1, 0)) as "bounces",
        sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
      from (
-       select session_uuid,
+       select session_id,
          ${getDateQuery('created_at', 'day')} time_series,
          count(*) c,
          min(created_at) min_time,
@@ -64,7 +66,7 @@ async function clickhouseQuery(websiteId, { start_at, end_at, filters = {} }) {
          and ${getBetweenDates('created_at', start_at, end_at)}
          ${pageviewQuery}
          ${sessionQuery}
-       group by session_uuid, time_series
+       group by session_id, time_series
      ) t;`,
     params,
   );