From 51139c1918abf53517ec6d07139748bb33b40d0b Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 26 Mar 2024 10:43:01 -0700 Subject: [PATCH 1/2] include visitors metrics in websiteStats --- src/lib/crypto.ts | 2 +- src/lib/session.ts | 4 +-- src/pages/api/send.ts | 4 +-- src/queries/analytics/getWebsiteStats.ts | 41 +++++++++++++++--------- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index d47c9fd8..1fbcf031 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -12,7 +12,7 @@ export function salt() { return hash(secret(), ROTATING_SALT); } -export function sessionSalt() { +export function visitSalt() { const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString()); return hash(secret(), ROTATING_SALT); diff --git a/src/lib/session.ts b/src/lib/session.ts index 6e2cbcc3..4eab67ff 100644 --- a/src/lib/session.ts +++ b/src/lib/session.ts @@ -1,4 +1,4 @@ -import { isUuid, secret, sessionSalt, uuid } from 'lib/crypto'; +import { isUuid, secret, uuid, visitSalt } from 'lib/crypto'; import { getClientInfo } from 'lib/detect'; import { parseToken } from 'next-basics'; import { NextApiRequestCollect } from 'pages/api/send'; @@ -68,7 +68,7 @@ export async function findSession(req: NextApiRequestCollect): Promise<{ await getClientInfo(req, payload); const sessionId = uuid(websiteId, hostname, ip, userAgent); - const visitId = uuid(sessionId, sessionSalt()); + const visitId = uuid(sessionId, visitSalt()); // Clickhouse does not require session lookup if (clickhouse.enabled) { diff --git a/src/pages/api/send.ts b/src/pages/api/send.ts index 46f534fd..1ca23a41 100644 --- a/src/pages/api/send.ts +++ b/src/pages/api/send.ts @@ -1,7 +1,7 @@ import ipaddr from 'ipaddr.js'; import { isbot } from 'isbot'; import { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants'; -import { secret, sessionSalt, uuid } from 'lib/crypto'; +import { secret, visitSalt, uuid } from 'lib/crypto'; import { getIpAddress } from 'lib/detect'; import { useCors, useSession, useValidate } from 'lib/middleware'; import { CollectionType, YupRequest } from 'lib/types'; @@ -98,7 +98,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { // expire visitId after 30 minutes session.visitId = !!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800 - ? uuid(session.id, sessionSalt()) + ? uuid(session.id, visitSalt()) : session.visitId; session.iat = Math.floor(new Date().getTime() / 1000); diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 51716b30..cd70ab78 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -1,19 +1,27 @@ -import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; import { QueryFilters } from 'lib/types'; -export async function getWebsiteStats(...args: [websiteId: string, filters: QueryFilters]) { +export async function getWebsiteStats( + ...args: [websiteId: string, filters: QueryFilters] +): Promise< + { pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[] +> { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } -async function relationalQuery(websiteId: string, filters: QueryFilters) { - const { getDateQuery, getAddIntervalQuery, getTimestampDiffQuery, parseFilters, rawQuery } = - prisma; +async function relationalQuery( + websiteId: string, + filters: QueryFilters, +): Promise< + { pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[] +> { + const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma; const { filterQuery, joinSession, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -24,14 +32,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", + count(distinct t.visit_id) as "visitors", sum(case when t.c = 1 then 1 else 0 end) as "bounces", - sum(case when t.max_time < ${getAddIntervalQuery('t.min_time', '1 hour')} - then ${getTimestampDiffQuery('t.min_time', 't.max_time')} - else 0 end) as "totaltime" + sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as "totaltime" from ( select website_event.session_id, - ${getDateQuery('website_event.created_at', 'hour')}, + website_event.visit_id, count(*) as "c", min(website_event.created_at) as "min_time", max(website_event.created_at) as "max_time" @@ -53,8 +60,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery( websiteId: string, filters: QueryFilters, -): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> { - const { rawQuery, getDateQuery, parseFilters } = clickhouse; +): Promise< + { pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[] +> { + const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -65,12 +74,13 @@ async function clickhouseQuery( select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", + count(distinct t.visit_id) as "visitors", sum(if(t.c = 1, 1, 0)) as "bounces", - sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" + sum(max_time-min_time) as "totaltime" from ( select session_id, - ${getDateQuery('created_at', 'hour')} time_series, + visit_id, count(*) c, min(created_at) min_time, max(created_at) max_time @@ -79,7 +89,7 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} - group by session_id, time_series + group by session_id, visit_id ) as t; `, params, @@ -88,6 +98,7 @@ async function clickhouseQuery( return { pageviews: Number(a.pageviews), uniques: Number(a.uniques), + visitors: Number(a.visitors), bounces: Number(a.bounces), totaltime: Number(a.totaltime), }; From dcf2457e450cb294c3a16f64a98718da21f3a80d Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 26 Mar 2024 11:15:39 -0700 Subject: [PATCH 2/2] update terminology, add metric cards to new metrics --- .../[websiteId]/WebsiteMetricsBar.tsx | 51 ++++++++++++------- src/components/messages.ts | 2 + src/queries/analytics/getWebsiteStats.ts | 22 ++++---- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/app/(main)/websites/[websiteId]/WebsiteMetricsBar.tsx b/src/app/(main)/websites/[websiteId]/WebsiteMetricsBar.tsx index 2dcb2b81..c488464d 100644 --- a/src/app/(main)/websites/[websiteId]/WebsiteMetricsBar.tsx +++ b/src/app/(main)/websites/[websiteId]/WebsiteMetricsBar.tsx @@ -21,11 +21,12 @@ export function WebsiteMetricsBar({ const { ref, isSticky } = useSticky({ enabled: sticky }); const { data, isLoading, isFetched, error } = useWebsiteStats(websiteId); - const { pageviews, uniques, bounces, totaltime } = data || {}; - const num = Math.min(data && uniques.value, data && bounces.value); + const { views, visitors, visits, bounces, totaltime } = data || {}; + const num = Math.min(data && visitors.value, data && bounces.value); const diffs = data && { - pageviews: pageviews.value - pageviews.change, - uniques: uniques.value - uniques.change, + views: views.value - views.change, + visitors: visitors.value - visitors.change, + visits: visits.value - visits.change, bounces: bounces.value - bounces.change, totaltime: totaltime.value - totaltime.change, }; @@ -39,25 +40,39 @@ export function WebsiteMetricsBar({ })} > - {pageviews && uniques && ( + {views && visitors && ( <> + + Number(n).toFixed(0) + '%'} @@ -66,14 +81,12 @@ export function WebsiteMetricsBar({ { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -19,7 +19,7 @@ async function relationalQuery( websiteId: string, filters: QueryFilters, ): Promise< - { pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[] + { views: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma; const { filterQuery, joinSession, params } = await parseFilters(websiteId, { @@ -30,9 +30,9 @@ async function relationalQuery( return rawQuery( ` select - sum(t.c) as "pageviews", - count(distinct t.session_id) as "uniques", - count(distinct t.visit_id) as "visitors", + sum(t.c) as "views", + count(distinct t.session_id) as "visitors", + count(distinct t.visit_id) as "visits", sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as "totaltime" from ( @@ -61,7 +61,7 @@ async function clickhouseQuery( websiteId: string, filters: QueryFilters, ): Promise< - { pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[] + { views: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { @@ -72,9 +72,9 @@ async function clickhouseQuery( return rawQuery( ` select - sum(t.c) as "pageviews", - count(distinct t.session_id) as "uniques", - count(distinct t.visit_id) as "visitors", + sum(t.c) as "views", + count(distinct t.session_id) as "visitors", + count(distinct t.visit_id) as "visits", sum(if(t.c = 1, 1, 0)) as "bounces", sum(max_time-min_time) as "totaltime" from ( @@ -96,9 +96,9 @@ async function clickhouseQuery( ).then(a => { return Object.values(a).map(a => { return { - pageviews: Number(a.pageviews), - uniques: Number(a.uniques), + views: Number(a.views), visitors: Number(a.visitors), + visits: Number(a.visits), bounces: Number(a.bounces), totaltime: Number(a.totaltime), };