m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2025-02-05 17:05:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

only use hourly table, remove daily table logic, fix updatechart undefined

Browse Source
This commit is contained in:
Francis Cao 2024-07-23 22:35:11 -07:00
parent 038ecdb592
commit 174b9e4376
10 changed files with 38 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
db/clickhouse/schema.sql
View File

@ -78,6 +78,8 @@ CREATE TABLE umami.website_event_stats_hourly
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
city String,
@ -103,8 +105,7 @@ ORDER BY (
visit_id,
event_type
)
SAMPLE BY cityHash64(visit_id)
TTL created_at + INTERVAL 10 DAY;
SAMPLE BY cityHash64(visit_id);
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
@ -117,6 +118,8 @@ SELECT
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
@ -140,6 +143,8 @@ FROM (SELECT
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
@ -163,106 +168,8 @@ GROUP BY website_id,
browser,
os,
device,
country,
subdivision1,
city,
event_type,
timestamp);
-- stats daily
CREATE TABLE umami.website_event_stats_daily
(
website_id UUID,
session_id UUID,
visit_id UUID,
hostname LowCardinality(String),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
city String,
entry_url AggregateFunction(argMin, String, DateTime('UTC')),
exit_url AggregateFunction(argMax, String, DateTime('UTC')),
url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
event_type UInt32,
event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
views SimpleAggregateFunction(sum, UInt64),
min_time SimpleAggregateFunction(min, DateTime('UTC')),
max_time SimpleAggregateFunction(max, DateTime('UTC')),
created_at Datetime('UTC')
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (
website_id,
toStartOfDay(created_at),
cityHash64(visit_id),
visit_id,
event_type
)
SAMPLE BY cityHash64(visit_id);
CREATE MATERIALIZED VIEW umami.website_event_stats_daily_mv
TO umami.website_event_stats_daily
AS
SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
country,
subdivision1,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
referrer_domain,
page_title,
event_type,
event_name,
views,
min_time,
max_time,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
country,
subdivision1,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
toStartOfDay(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,

22
src/components/charts/Chart.tsx
View File

@ -79,18 +79,20 @@ export function Chart({
};
const updateChart = (data: any) => {
if ((data?.datasets?.length || 0) === chart.current.data.datasets.length) {
chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => {
if (data?.datasets[index]) {
dataset.data = data?.datasets[index]?.data;
if (data.datasets) {
if (data.datasets.length === chart.current.data.datasets.length) {
chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => {
if (data?.datasets[index]) {
dataset.data = data?.datasets[index]?.data;
if (chart.current.legend.legendItems[index]) {
chart.current.legend.legendItems[index].text = data?.datasets[index]?.label;
if (chart.current.legend.legendItems[index]) {
chart.current.legend.legendItems[index].text = data?.datasets[index]?.label;
}
}
}
});
} else {
chart.current.data.datasets = data.datasets;
});
} else {
chart.current.data.datasets = data.datasets;
}
}
chart.current.options = options;

13
src/pages/api/websites/[websiteId]/metrics.ts
View File

@ -64,7 +64,7 @@ export default async (
await useAuth(req, res);
await useValidate(schema, req, res);
const { websiteId, type, limit, offset, search, unit } = req.query;
const { websiteId, type, limit, offset, search } = req.query;
if (req.method === 'GET') {
if (!(await canViewWebsite(req.auth, websiteId))) {
@ -89,7 +89,7 @@ export default async (
}
if (SESSION_COLUMNS.includes(type)) {
const data = await getSessionMetrics(websiteId, type, filters, limit, offset, unit as string);
const data = await getSessionMetrics(websiteId, type, filters, limit, offset);
if (type === 'language') {
const combined = {};
@ -111,14 +111,7 @@ export default async (
}
if (EVENT_COLUMNS.includes(type)) {
const data = await getPageviewMetrics(
websiteId,
type,
filters,
limit,
offset,
unit as string,
);
const data = await getPageviewMetrics(websiteId, type, filters, limit, offset);
return ok(res, data);
}

6
src/pages/api/websites/[websiteId]/stats.ts
View File

@ -56,7 +56,7 @@ export default async (
await useAuth(req, res);
await useValidate(schema, req, res);
const { websiteId, compare, unit } = req.query;
const { websiteId, compare } = req.query;
if (req.method === 'GET') {
if (!(await canViewWebsite(req.auth, websiteId))) {
@ -72,13 +72,13 @@ export default async (
const filters = getRequestFilters(req);
const metrics = await getWebsiteStats(websiteId, unit as string, {
const metrics = await getWebsiteStats(websiteId, {
...filters,
startDate,
endDate,
});
const prevPeriod = await getWebsiteStats(websiteId, unit as string, {
const prevPeriod = await getWebsiteStats(websiteId, {
...filters,
startDate: compareStartDate,
endDate: compareEndDate,

4
src/queries/analytics/events/getEventMetrics.ts
View File

@ -51,8 +51,6 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.customEvent,
});
const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
select
@ -62,7 +60,7 @@ async function clickhouseQuery(
from (
select arrayJoin(event_name) as event_name,
created_at
from ${table} website_event
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}

7
src/queries/analytics/getWebsiteStats.ts
View File

@ -6,7 +6,7 @@ import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export async function getWebsiteStats(
...args: [websiteId: string, unit: string, filters: QueryFilters]
...args: [websiteId: string, filters: QueryFilters]
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
> {
@ -18,7 +18,6 @@ export async function getWebsiteStats(
async function relationalQuery(
websiteId: string,
unit: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
@ -59,7 +58,6 @@ async function relationalQuery(
async function clickhouseQuery(
websiteId: string,
unit: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
@ -69,7 +67,6 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@ -79,7 +76,7 @@ async function clickhouseQuery(
uniq(visit_id) as "visits",
sumIf(1, views = 1) as "bounces",
sum(max_time-min_time) as "totaltime"
from ${table} "website_event"
from website_event_stats_hourly "website_event"
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}

16
src/queries/analytics/pageviews/getPageviewMetrics.ts
View File

@ -1,4 +1,3 @@
/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
import clickhouse from 'lib/clickhouse';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
@ -6,14 +5,7 @@ import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export async function getPageviewMetrics(
...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number,
offset?: number,
unit?: string,
]
...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -27,7 +19,6 @@ async function relationalQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
unit: string,
) {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = prisma;
@ -91,7 +82,6 @@ async function clickhouseQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
unit: string,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse;
@ -121,15 +111,13 @@ async function clickhouseQuery(
groupByQuery = 'group by x';
}
const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
select g.t as x,
count(*) as y
from (
select ${columnQuery} as t
from ${table} website_event
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}

3
src/queries/analytics/pageviews/getPageviewStats.ts
View File

@ -46,7 +46,6 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@ -57,7 +56,7 @@ async function clickhouseQuery(
select
${getDateSQL('created_at', unit, timezone)} as t,
sum(views) as y
from ${table} website_event
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}

19
src/queries/analytics/sessions/getSessionMetrics.ts
View File

@ -1,19 +1,11 @@
/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export async function getSessionMetrics(
...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number,
offset?: number,
unit?: string,
]
...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -27,7 +19,6 @@ async function relationalQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
unit: string,
) {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = prisma;
@ -71,7 +62,6 @@ async function clickhouseQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
unit: string,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
@ -80,7 +70,6 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView,
});
const includeCountry = column === 'city' || column === 'subdivision1';
const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@ -88,7 +77,7 @@ async function clickhouseQuery(
${column} x,
uniq(session_id) y
${includeCountry ? ', country' : ''}
from ${table} website_event
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}

3
src/queries/analytics/sessions/getSessionStats.ts
View File

@ -46,7 +46,6 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@ -57,7 +56,7 @@ async function clickhouseQuery(
select
${getDateSQL('created_at', unit, timezone)} as t,
uniq(session_id) as y
from ${table} website_event
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}