diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 9082884b..29cdeabc 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -78,6 +78,8 @@ CREATE TABLE umami.website_event_stats_hourly browser LowCardinality(String), os LowCardinality(String), device LowCardinality(String), + screen LowCardinality(String), + language LowCardinality(String), country LowCardinality(String), subdivision1 LowCardinality(String), city String, @@ -103,8 +105,7 @@ ORDER BY ( visit_id, event_type ) -SAMPLE BY cityHash64(visit_id) -TTL created_at + INTERVAL 10 DAY; +SAMPLE BY cityHash64(visit_id); CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv TO umami.website_event_stats_hourly @@ -117,6 +118,8 @@ SELECT browser, os, device, + screen, + language, country, subdivision1, city, @@ -140,6 +143,8 @@ FROM (SELECT browser, os, device, + screen, + language, country, subdivision1, city, @@ -163,106 +168,8 @@ GROUP BY website_id, browser, os, device, - country, - subdivision1, - city, - event_type, - timestamp); - --- stats daily -CREATE TABLE umami.website_event_stats_daily -( - website_id UUID, - session_id UUID, - visit_id UUID, - hostname LowCardinality(String), - browser LowCardinality(String), - os LowCardinality(String), - device LowCardinality(String), - country LowCardinality(String), - subdivision1 LowCardinality(String), - city String, - entry_url AggregateFunction(argMin, String, DateTime('UTC')), - exit_url AggregateFunction(argMax, String, DateTime('UTC')), - url_path SimpleAggregateFunction(groupArrayArray, Array(String)), - url_query SimpleAggregateFunction(groupArrayArray, Array(String)), - referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)), - page_title SimpleAggregateFunction(groupArrayArray, Array(String)), - event_type UInt32, - event_name SimpleAggregateFunction(groupArrayArray, Array(String)), - views SimpleAggregateFunction(sum, UInt64), - min_time SimpleAggregateFunction(min, DateTime('UTC')), - max_time SimpleAggregateFunction(max, DateTime('UTC')), - created_at Datetime('UTC') -) -ENGINE = AggregatingMergeTree -PARTITION BY toYYYYMM(created_at) -ORDER BY ( - website_id, - toStartOfDay(created_at), - cityHash64(visit_id), - visit_id, - event_type -) -SAMPLE BY cityHash64(visit_id); - -CREATE MATERIALIZED VIEW umami.website_event_stats_daily_mv -TO umami.website_event_stats_daily -AS -SELECT - website_id, - session_id, - visit_id, - hostname, - browser, - os, - device, - country, - subdivision1, - city, - entry_url, - exit_url, - url_paths as url_path, - url_query, - referrer_domain, - page_title, - event_type, - event_name, - views, - min_time, - max_time, - timestamp as created_at -FROM (SELECT - website_id, - session_id, - visit_id, - hostname, - browser, - os, - device, - country, - subdivision1, - city, - argMinState(url_path, created_at) entry_url, - argMaxState(url_path, created_at) exit_url, - arrayFilter(x -> x != '', groupArray(url_path)) as url_paths, - arrayFilter(x -> x != '', groupArray(url_query)) url_query, - arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain, - arrayFilter(x -> x != '', groupArray(page_title)) page_title, - event_type, - if(event_type = 2, groupArray(event_name), []) event_name, - sumIf(1, event_type = 1) views, - min(created_at) min_time, - max(created_at) max_time, - toStartOfDay(created_at) timestamp -FROM umami.website_event -GROUP BY website_id, - session_id, - visit_id, - hostname, - browser, - os, - device, + screen, + language, country, subdivision1, city, diff --git a/src/components/charts/Chart.tsx b/src/components/charts/Chart.tsx index 926defaf..a4badbce 100644 --- a/src/components/charts/Chart.tsx +++ b/src/components/charts/Chart.tsx @@ -79,18 +79,20 @@ export function Chart({ }; const updateChart = (data: any) => { - if ((data?.datasets?.length || 0) === chart.current.data.datasets.length) { - chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => { - if (data?.datasets[index]) { - dataset.data = data?.datasets[index]?.data; + if (data.datasets) { + if (data.datasets.length === chart.current.data.datasets.length) { + chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => { + if (data?.datasets[index]) { + dataset.data = data?.datasets[index]?.data; - if (chart.current.legend.legendItems[index]) { - chart.current.legend.legendItems[index].text = data?.datasets[index]?.label; + if (chart.current.legend.legendItems[index]) { + chart.current.legend.legendItems[index].text = data?.datasets[index]?.label; + } } - } - }); - } else { - chart.current.data.datasets = data.datasets; + }); + } else { + chart.current.data.datasets = data.datasets; + } } chart.current.options = options; diff --git a/src/pages/api/websites/[websiteId]/metrics.ts b/src/pages/api/websites/[websiteId]/metrics.ts index b37c38f7..3dac217b 100644 --- a/src/pages/api/websites/[websiteId]/metrics.ts +++ b/src/pages/api/websites/[websiteId]/metrics.ts @@ -64,7 +64,7 @@ export default async ( await useAuth(req, res); await useValidate(schema, req, res); - const { websiteId, type, limit, offset, search, unit } = req.query; + const { websiteId, type, limit, offset, search } = req.query; if (req.method === 'GET') { if (!(await canViewWebsite(req.auth, websiteId))) { @@ -89,7 +89,7 @@ export default async ( } if (SESSION_COLUMNS.includes(type)) { - const data = await getSessionMetrics(websiteId, type, filters, limit, offset, unit as string); + const data = await getSessionMetrics(websiteId, type, filters, limit, offset); if (type === 'language') { const combined = {}; @@ -111,14 +111,7 @@ export default async ( } if (EVENT_COLUMNS.includes(type)) { - const data = await getPageviewMetrics( - websiteId, - type, - filters, - limit, - offset, - unit as string, - ); + const data = await getPageviewMetrics(websiteId, type, filters, limit, offset); return ok(res, data); } diff --git a/src/pages/api/websites/[websiteId]/stats.ts b/src/pages/api/websites/[websiteId]/stats.ts index 1c684dbe..9ca84c74 100644 --- a/src/pages/api/websites/[websiteId]/stats.ts +++ b/src/pages/api/websites/[websiteId]/stats.ts @@ -56,7 +56,7 @@ export default async ( await useAuth(req, res); await useValidate(schema, req, res); - const { websiteId, compare, unit } = req.query; + const { websiteId, compare } = req.query; if (req.method === 'GET') { if (!(await canViewWebsite(req.auth, websiteId))) { @@ -72,13 +72,13 @@ export default async ( const filters = getRequestFilters(req); - const metrics = await getWebsiteStats(websiteId, unit as string, { + const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate, }); - const prevPeriod = await getWebsiteStats(websiteId, unit as string, { + const prevPeriod = await getWebsiteStats(websiteId, { ...filters, startDate: compareStartDate, endDate: compareEndDate, diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts index c97c11ad..36232135 100644 --- a/src/queries/analytics/events/getEventMetrics.ts +++ b/src/queries/analytics/events/getEventMetrics.ts @@ -51,8 +51,6 @@ async function clickhouseQuery( eventType: EVENT_TYPE.customEvent, }); - const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; - return rawQuery( ` select @@ -62,7 +60,7 @@ async function clickhouseQuery( from ( select arrayJoin(event_name) as event_name, created_at - from ${table} website_event + from website_event_stats_hourly website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 125715ca..ebe711a0 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -6,7 +6,7 @@ import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; export async function getWebsiteStats( - ...args: [websiteId: string, unit: string, filters: QueryFilters] + ...args: [websiteId: string, filters: QueryFilters] ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { @@ -18,7 +18,6 @@ export async function getWebsiteStats( async function relationalQuery( websiteId: string, - unit: string, filters: QueryFilters, ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] @@ -59,7 +58,6 @@ async function relationalQuery( async function clickhouseQuery( websiteId: string, - unit: string, filters: QueryFilters, ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] @@ -69,7 +67,6 @@ async function clickhouseQuery( ...filters, eventType: EVENT_TYPE.pageView, }); - const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` @@ -79,7 +76,7 @@ async function clickhouseQuery( uniq(visit_id) as "visits", sumIf(1, views = 1) as "bounces", sum(max_time-min_time) as "totaltime" - from ${table} "website_event" + from website_event_stats_hourly "website_event" where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts index d66ec6aa..ccfe4ef0 100644 --- a/src/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */ import clickhouse from 'lib/clickhouse'; import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; @@ -6,14 +5,7 @@ import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; export async function getPageviewMetrics( - ...args: [ - websiteId: string, - type: string, - filters: QueryFilters, - limit?: number, - offset?: number, - unit?: string, - ] + ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -27,7 +19,6 @@ async function relationalQuery( filters: QueryFilters, limit: number = 500, offset: number = 0, - unit: string, ) { const column = FILTER_COLUMNS[type] || type; const { rawQuery, parseFilters } = prisma; @@ -91,7 +82,6 @@ async function clickhouseQuery( filters: QueryFilters, limit: number = 500, offset: number = 0, - unit: string, ): Promise<{ x: string; y: number }[]> { const column = FILTER_COLUMNS[type] || type; const { rawQuery, parseFilters } = clickhouse; @@ -121,15 +111,13 @@ async function clickhouseQuery( groupByQuery = 'group by x'; } - const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; - return rawQuery( ` select g.t as x, count(*) as y from ( select ${columnQuery} as t - from ${table} website_event + from website_event_stats_hourly website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts index 0bb16ca9..8d757de3 100644 --- a/src/queries/analytics/pageviews/getPageviewStats.ts +++ b/src/queries/analytics/pageviews/getPageviewStats.ts @@ -46,7 +46,6 @@ async function clickhouseQuery( ...filters, eventType: EVENT_TYPE.pageView, }); - const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` @@ -57,7 +56,7 @@ async function clickhouseQuery( select ${getDateSQL('created_at', unit, timezone)} as t, sum(views) as y - from ${table} website_event + from website_event_stats_hourly website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts index 93e36a55..9baf2a5c 100644 --- a/src/queries/analytics/sessions/getSessionMetrics.ts +++ b/src/queries/analytics/sessions/getSessionMetrics.ts @@ -1,19 +1,11 @@ -/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */ -import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; export async function getSessionMetrics( - ...args: [ - websiteId: string, - type: string, - filters: QueryFilters, - limit?: number, - offset?: number, - unit?: string, - ] + ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -27,7 +19,6 @@ async function relationalQuery( filters: QueryFilters, limit: number = 500, offset: number = 0, - unit: string, ) { const column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = prisma; @@ -71,7 +62,6 @@ async function clickhouseQuery( filters: QueryFilters, limit: number = 500, offset: number = 0, - unit: string, ): Promise<{ x: string; y: number }[]> { const column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = clickhouse; @@ -80,7 +70,6 @@ async function clickhouseQuery( eventType: EVENT_TYPE.pageView, }); const includeCountry = column === 'city' || column === 'subdivision1'; - const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` @@ -88,7 +77,7 @@ async function clickhouseQuery( ${column} x, uniq(session_id) y ${includeCountry ? ', country' : ''} - from ${table} website_event + from website_event_stats_hourly website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts index 2b57b922..0fe0b8a5 100644 --- a/src/queries/analytics/sessions/getSessionStats.ts +++ b/src/queries/analytics/sessions/getSessionStats.ts @@ -46,7 +46,6 @@ async function clickhouseQuery( ...filters, eventType: EVENT_TYPE.pageView, }); - const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily'; return rawQuery( ` @@ -57,7 +56,7 @@ async function clickhouseQuery( select ${getDateSQL('created_at', unit, timezone)} as t, uniq(session_id) as y - from ${table} website_event + from website_event_stats_hourly website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32}