only use hourly table, remove daily table logic, fix updatechart undefined

db/clickhouse/schema.sql

@@ -78,6 +78,8 @@ CREATE TABLE umami.website_event_stats_hourly
     browser LowCardinality(String),
     os LowCardinality(String),
     device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
     country LowCardinality(String),
     subdivision1 LowCardinality(String),
     city String,
@@ -103,8 +105,7 @@ ORDER BY (
     visit_id,
     event_type
 )
-SAMPLE BY cityHash64(visit_id)
+SAMPLE BY cityHash64(visit_id);
-TTL created_at + INTERVAL 10 DAY;
 
 CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
 TO umami.website_event_stats_hourly
@@ -117,6 +118,8 @@ SELECT
     browser,
     os,
     device,
+    screen,
+    language,
     country,
     subdivision1,
     city,
@@ -140,6 +143,8 @@ FROM (SELECT
     browser,
     os,
     device,
+    screen,
+    language,
     country,
     subdivision1,
     city,
@@ -163,106 +168,8 @@ GROUP BY website_id,
     browser,
     os,
     device,
-    country,
+    screen,
-    subdivision1,
+    language,
-    city,
-    event_type,
-    timestamp);
-
--- stats daily
-CREATE TABLE umami.website_event_stats_daily
-(
-    website_id UUID,
-    session_id UUID,
-    visit_id UUID,
-    hostname LowCardinality(String),
-    browser LowCardinality(String),
-    os LowCardinality(String),
-    device LowCardinality(String),
-    country LowCardinality(String),
-    subdivision1 LowCardinality(String),
-    city String,
-    entry_url AggregateFunction(argMin, String, DateTime('UTC')),
-    exit_url AggregateFunction(argMax, String, DateTime('UTC')),
-    url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
-    url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
-    referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
-    page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
-    event_type UInt32,
-    event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
-    views SimpleAggregateFunction(sum, UInt64),
-    min_time SimpleAggregateFunction(min, DateTime('UTC')),
-    max_time SimpleAggregateFunction(max, DateTime('UTC')),
-    created_at Datetime('UTC')
-)
-ENGINE = AggregatingMergeTree
-PARTITION BY toYYYYMM(created_at)
-ORDER BY (
-    website_id,
-    toStartOfDay(created_at),
-    cityHash64(visit_id),
-    visit_id,
-    event_type
-)
-SAMPLE BY cityHash64(visit_id);
-
-CREATE MATERIALIZED VIEW umami.website_event_stats_daily_mv
-TO umami.website_event_stats_daily
-AS
-SELECT
-    website_id,
-    session_id,
-    visit_id,
-    hostname,
-    browser,
-    os,
-    device,
-    country,
-    subdivision1,
-    city,
-    entry_url,
-    exit_url,
-    url_paths as url_path,
-    url_query,
-    referrer_domain,
-    page_title,
-    event_type,
-    event_name,
-    views,
-    min_time,
-    max_time,
-    timestamp as created_at
-FROM (SELECT
-    website_id,
-    session_id,
-    visit_id,
-    hostname,
-    browser,
-    os,
-    device,
-    country,
-    subdivision1,
-    city,
-    argMinState(url_path, created_at) entry_url,
-    argMaxState(url_path, created_at) exit_url,
-    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
-    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
-    arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
-    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
-    event_type,
-    if(event_type = 2, groupArray(event_name), []) event_name,
-    sumIf(1, event_type = 1) views,
-    min(created_at) min_time,
-    max(created_at) max_time,
-    toStartOfDay(created_at) timestamp
-FROM umami.website_event
-GROUP BY website_id,
-    session_id,
-    visit_id,
-    hostname,
-    browser,
-    os,
-    device,
     country,
     subdivision1,
     city,

src/components/charts/Chart.tsx

@@ -79,7 +79,8 @@ export function Chart({
   };
 
   const updateChart = (data: any) => {
-    if ((data?.datasets?.length || 0) === chart.current.data.datasets.length) {
+    if (data.datasets) {
+      if (data.datasets.length === chart.current.data.datasets.length) {
         chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => {
           if (data?.datasets[index]) {
             dataset.data = data?.datasets[index]?.data;
@@ -92,6 +93,7 @@ export function Chart({
       } else {
         chart.current.data.datasets = data.datasets;
       }
+    }
 
     chart.current.options = options;
 

src/pages/api/websites/[websiteId]/metrics.ts

@@ -64,7 +64,7 @@ export default async (
   await useAuth(req, res);
   await useValidate(schema, req, res);
 
-  const { websiteId, type, limit, offset, search, unit } = req.query;
+  const { websiteId, type, limit, offset, search } = req.query;
 
   if (req.method === 'GET') {
     if (!(await canViewWebsite(req.auth, websiteId))) {
@@ -89,7 +89,7 @@ export default async (
     }
 
     if (SESSION_COLUMNS.includes(type)) {
-      const data = await getSessionMetrics(websiteId, type, filters, limit, offset, unit as string);
+      const data = await getSessionMetrics(websiteId, type, filters, limit, offset);
 
       if (type === 'language') {
         const combined = {};
@@ -111,14 +111,7 @@ export default async (
     }
 
     if (EVENT_COLUMNS.includes(type)) {
-      const data = await getPageviewMetrics(
+      const data = await getPageviewMetrics(websiteId, type, filters, limit, offset);
-        websiteId,
-        type,
-        filters,
-        limit,
-        offset,
-        unit as string,
-      );
 
       return ok(res, data);
     }

src/pages/api/websites/[websiteId]/stats.ts

@@ -56,7 +56,7 @@ export default async (
   await useAuth(req, res);
   await useValidate(schema, req, res);
 
-  const { websiteId, compare, unit } = req.query;
+  const { websiteId, compare } = req.query;
 
   if (req.method === 'GET') {
     if (!(await canViewWebsite(req.auth, websiteId))) {
@@ -72,13 +72,13 @@ export default async (
 
     const filters = getRequestFilters(req);
 
-    const metrics = await getWebsiteStats(websiteId, unit as string, {
+    const metrics = await getWebsiteStats(websiteId, {
       ...filters,
       startDate,
       endDate,
     });
 
-    const prevPeriod = await getWebsiteStats(websiteId, unit as string, {
+    const prevPeriod = await getWebsiteStats(websiteId, {
       ...filters,
       startDate: compareStartDate,
       endDate: compareEndDate,

src/queries/analytics/events/getEventMetrics.ts

@@ -51,8 +51,6 @@ async function clickhouseQuery(
     eventType: EVENT_TYPE.customEvent,
   });
 
-  const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
-
   return rawQuery(
     `
     select
@@ -62,7 +60,7 @@ async function clickhouseQuery(
     from (
       select arrayJoin(event_name) as event_name,
         created_at
-      from ${table} website_event
+      from website_event_stats_hourly website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}

src/queries/analytics/getWebsiteStats.ts

@@ -6,7 +6,7 @@ import prisma from 'lib/prisma';
 import { QueryFilters } from 'lib/types';
 
 export async function getWebsiteStats(
-  ...args: [websiteId: string, unit: string, filters: QueryFilters]
+  ...args: [websiteId: string, filters: QueryFilters]
 ): Promise<
   { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
 > {
@@ -18,7 +18,6 @@ export async function getWebsiteStats(
 
 async function relationalQuery(
   websiteId: string,
-  unit: string,
   filters: QueryFilters,
 ): Promise<
   { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
@@ -59,7 +58,6 @@ async function relationalQuery(
 
 async function clickhouseQuery(
   websiteId: string,
-  unit: string,
   filters: QueryFilters,
 ): Promise<
   { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
@@ -69,7 +67,6 @@ async function clickhouseQuery(
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });
-  const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
 
   return rawQuery(
     `
@@ -79,7 +76,7 @@ async function clickhouseQuery(
       uniq(visit_id) as "visits",
       sumIf(1, views = 1) as "bounces",
       sum(max_time-min_time) as "totaltime"
-    from ${table} "website_event"
+    from website_event_stats_hourly "website_event"
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}

src/queries/analytics/pageviews/getPageviewMetrics.ts

@@ -1,4 +1,3 @@
-/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
 import clickhouse from 'lib/clickhouse';
 import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
@@ -6,14 +5,7 @@ import prisma from 'lib/prisma';
 import { QueryFilters } from 'lib/types';
 
 export async function getPageviewMetrics(
-  ...args: [
+  ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]
-    websiteId: string,
-    type: string,
-    filters: QueryFilters,
-    limit?: number,
-    offset?: number,
-    unit?: string,
-  ]
 ) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -27,7 +19,6 @@ async function relationalQuery(
   filters: QueryFilters,
   limit: number = 500,
   offset: number = 0,
-  unit: string,
 ) {
   const column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = prisma;
@@ -91,7 +82,6 @@ async function clickhouseQuery(
   filters: QueryFilters,
   limit: number = 500,
   offset: number = 0,
-  unit: string,
 ): Promise<{ x: string; y: number }[]> {
   const column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = clickhouse;
@@ -121,15 +111,13 @@ async function clickhouseQuery(
     groupByQuery = 'group by x';
   }
 
-  const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
-
   return rawQuery(
     `
     select g.t as x,
       count(*) as y
     from (
       select ${columnQuery} as t
-      from ${table} website_event
+      from website_event_stats_hourly website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}

src/queries/analytics/pageviews/getPageviewStats.ts

@@ -46,7 +46,6 @@ async function clickhouseQuery(
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });
-  const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
 
   return rawQuery(
     `
@@ -57,7 +56,7 @@ async function clickhouseQuery(
       select 
         ${getDateSQL('created_at', unit, timezone)} as t,
         sum(views) as y
-      from ${table} website_event
+      from website_event_stats_hourly website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}

src/queries/analytics/sessions/getSessionMetrics.ts

@@ -1,19 +1,11 @@
-/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
-import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
-import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
 import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
 import { QueryFilters } from 'lib/types';
 
 export async function getSessionMetrics(
-  ...args: [
+  ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]
-    websiteId: string,
-    type: string,
-    filters: QueryFilters,
-    limit?: number,
-    offset?: number,
-    unit?: string,
-  ]
 ) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -27,7 +19,6 @@ async function relationalQuery(
   filters: QueryFilters,
   limit: number = 500,
   offset: number = 0,
-  unit: string,
 ) {
   const column = FILTER_COLUMNS[type] || type;
   const { parseFilters, rawQuery } = prisma;
@@ -71,7 +62,6 @@ async function clickhouseQuery(
   filters: QueryFilters,
   limit: number = 500,
   offset: number = 0,
-  unit: string,
 ): Promise<{ x: string; y: number }[]> {
   const column = FILTER_COLUMNS[type] || type;
   const { parseFilters, rawQuery } = clickhouse;
@@ -80,7 +70,6 @@ async function clickhouseQuery(
     eventType: EVENT_TYPE.pageView,
   });
   const includeCountry = column === 'city' || column === 'subdivision1';
-  const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
 
   return rawQuery(
     `
@@ -88,7 +77,7 @@ async function clickhouseQuery(
       ${column} x,
       uniq(session_id) y
       ${includeCountry ? ', country' : ''}
-    from ${table} website_event
+    from website_event_stats_hourly website_event
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and event_type = {eventType:UInt32}

src/queries/analytics/sessions/getSessionStats.ts

@@ -46,7 +46,6 @@ async function clickhouseQuery(
     ...filters,
     eventType: EVENT_TYPE.pageView,
   });
-  const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
 
   return rawQuery(
     `
@@ -57,7 +56,7 @@ async function clickhouseQuery(
       select 
         ${getDateSQL('created_at', unit, timezone)} as t,
         uniq(session_id) as y
-      from ${table} website_event
+      from website_event_stats_hourly website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}