include visitors metrics in websiteStats

This commit is contained in:
Francis Cao 2024-03-26 10:43:01 -07:00
parent 1a839d1cae
commit 51139c1918
4 changed files with 31 additions and 20 deletions

View File

@ -12,7 +12,7 @@ export function salt() {
return hash(secret(), ROTATING_SALT); return hash(secret(), ROTATING_SALT);
} }
export function sessionSalt() { export function visitSalt() {
const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString()); const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString());
return hash(secret(), ROTATING_SALT); return hash(secret(), ROTATING_SALT);

View File

@ -1,4 +1,4 @@
import { isUuid, secret, sessionSalt, uuid } from 'lib/crypto'; import { isUuid, secret, uuid, visitSalt } from 'lib/crypto';
import { getClientInfo } from 'lib/detect'; import { getClientInfo } from 'lib/detect';
import { parseToken } from 'next-basics'; import { parseToken } from 'next-basics';
import { NextApiRequestCollect } from 'pages/api/send'; import { NextApiRequestCollect } from 'pages/api/send';
@ -68,7 +68,7 @@ export async function findSession(req: NextApiRequestCollect): Promise<{
await getClientInfo(req, payload); await getClientInfo(req, payload);
const sessionId = uuid(websiteId, hostname, ip, userAgent); const sessionId = uuid(websiteId, hostname, ip, userAgent);
const visitId = uuid(sessionId, sessionSalt()); const visitId = uuid(sessionId, visitSalt());
// Clickhouse does not require session lookup // Clickhouse does not require session lookup
if (clickhouse.enabled) { if (clickhouse.enabled) {

View File

@ -1,7 +1,7 @@
import ipaddr from 'ipaddr.js'; import ipaddr from 'ipaddr.js';
import { isbot } from 'isbot'; import { isbot } from 'isbot';
import { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants'; import { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants';
import { secret, sessionSalt, uuid } from 'lib/crypto'; import { secret, visitSalt, uuid } from 'lib/crypto';
import { getIpAddress } from 'lib/detect'; import { getIpAddress } from 'lib/detect';
import { useCors, useSession, useValidate } from 'lib/middleware'; import { useCors, useSession, useValidate } from 'lib/middleware';
import { CollectionType, YupRequest } from 'lib/types'; import { CollectionType, YupRequest } from 'lib/types';
@ -98,7 +98,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
// expire visitId after 30 minutes // expire visitId after 30 minutes
session.visitId = session.visitId =
!!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800 !!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800
? uuid(session.id, sessionSalt()) ? uuid(session.id, visitSalt())
: session.visitId; : session.visitId;
session.iat = Math.floor(new Date().getTime() / 1000); session.iat = Math.floor(new Date().getTime() / 1000);

View File

@ -1,19 +1,27 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types'; import { QueryFilters } from 'lib/types';
export async function getWebsiteStats(...args: [websiteId: string, filters: QueryFilters]) { export async function getWebsiteStats(
...args: [websiteId: string, filters: QueryFilters]
): Promise<
{ pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[]
> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
async function relationalQuery(websiteId: string, filters: QueryFilters) { async function relationalQuery(
const { getDateQuery, getAddIntervalQuery, getTimestampDiffQuery, parseFilters, rawQuery } = websiteId: string,
prisma; filters: QueryFilters,
): Promise<
{ pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[]
> {
const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, { const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters, ...filters,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
@ -24,14 +32,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
select select
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques", count(distinct t.session_id) as "uniques",
count(distinct t.visit_id) as "visitors",
sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(case when t.max_time < ${getAddIntervalQuery('t.min_time', '1 hour')} sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as "totaltime"
then ${getTimestampDiffQuery('t.min_time', 't.max_time')}
else 0 end) as "totaltime"
from ( from (
select select
website_event.session_id, website_event.session_id,
${getDateQuery('website_event.created_at', 'hour')}, website_event.visit_id,
count(*) as "c", count(*) as "c",
min(website_event.created_at) as "min_time", min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time" max(website_event.created_at) as "max_time"
@ -53,8 +60,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
async function clickhouseQuery( async function clickhouseQuery(
websiteId: string, websiteId: string,
filters: QueryFilters, filters: QueryFilters,
): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> { ): Promise<
const { rawQuery, getDateQuery, parseFilters } = clickhouse; { pageviews: number; uniques: number; visitors: number; bounces: number; totaltime: number }[]
> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
...filters, ...filters,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
@ -65,12 +74,13 @@ async function clickhouseQuery(
select select
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques", count(distinct t.session_id) as "uniques",
count(distinct t.visit_id) as "visitors",
sum(if(t.c = 1, 1, 0)) as "bounces", sum(if(t.c = 1, 1, 0)) as "bounces",
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" sum(max_time-min_time) as "totaltime"
from ( from (
select select
session_id, session_id,
${getDateQuery('created_at', 'hour')} time_series, visit_id,
count(*) c, count(*) c,
min(created_at) min_time, min(created_at) min_time,
max(created_at) max_time max(created_at) max_time
@ -79,7 +89,7 @@ async function clickhouseQuery(
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${filterQuery} ${filterQuery}
group by session_id, time_series group by session_id, visit_id
) as t; ) as t;
`, `,
params, params,
@ -88,6 +98,7 @@ async function clickhouseQuery(
return { return {
pageviews: Number(a.pageviews), pageviews: Number(a.pageviews),
uniques: Number(a.uniques), uniques: Number(a.uniques),
visitors: Number(a.visitors),
bounces: Number(a.bounces), bounces: Number(a.bounces),
totaltime: Number(a.totaltime), totaltime: Number(a.totaltime),
}; };