From 0aaf2c0b3bb508a739e9bc91fa99da9345269bec Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 25 Mar 2024 17:47:53 -0700 Subject: [PATCH] update visitId hash and expiration logic --- .../migrations/05_add_visit_id/migration.sql | 14 +++++++++++--- src/lib/crypto.ts | 8 +++++++- src/lib/session.ts | 7 +++++-- src/pages/api/send.ts | 12 +++++++++++- src/queries/analytics/events/saveEvent.ts | 13 ++++++++++--- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/db/postgresql/migrations/05_add_visit_id/migration.sql b/db/postgresql/migrations/05_add_visit_id/migration.sql index 8274d1fb..fd2f1b90 100644 --- a/db/postgresql/migrations/05_add_visit_id/migration.sql +++ b/db/postgresql/migrations/05_add_visit_id/migration.sql @@ -1,9 +1,17 @@ -- AlterTable ALTER TABLE "website_event" ADD COLUMN "visit_id" UUID NULL; -UPDATE "website_event" -SET visit_id = uuid_in(overlay(overlay(md5(CONCAT(session_id::text, to_char(date_trunc('hour', created_at), 'YYYY-MM-DD HH24:00:00'))) placing '4' from 13) placing '8' from 17)::cstring) -WHERE visit_id IS NULL; +UPDATE "website_event" we +SET visit_id = a.uuid +FROM (SELECT DISTINCT + s.session_id, + s.visit_time, + gen_random_uuid() uuid + FROM (SELECT DISTINCT session_id, + date_trunc('hour', created_at) visit_time + FROM "website_event") s) a +WHERE we.session_id = a.session_id + and date_trunc('hour', we.created_at) = a.visit_time; ALTER TABLE "website_event" ALTER COLUMN "visit_id" SET NOT NULL; diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index a2763352..d47c9fd8 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -1,4 +1,4 @@ -import { startOfMonth } from 'date-fns'; +import { startOfHour, startOfMonth } from 'date-fns'; import { hash } from 'next-basics'; import { v4, v5, validate } from 'uuid'; @@ -12,6 +12,12 @@ export function salt() { return hash(secret(), ROTATING_SALT); } +export function sessionSalt() { + const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString()); + + return hash(secret(), ROTATING_SALT); +} + export function uuid(...args: any) { if (!args.length) return v4(); diff --git a/src/lib/session.ts b/src/lib/session.ts index 0f388db9..6e2cbcc3 100644 --- a/src/lib/session.ts +++ b/src/lib/session.ts @@ -1,4 +1,4 @@ -import { isUuid, secret, uuid } from 'lib/crypto'; +import { isUuid, secret, sessionSalt, uuid } from 'lib/crypto'; import { getClientInfo } from 'lib/detect'; import { parseToken } from 'next-basics'; import { NextApiRequestCollect } from 'pages/api/send'; @@ -10,6 +10,7 @@ import { loadSession, loadWebsite } from './load'; export async function findSession(req: NextApiRequestCollect): Promise<{ id: any; websiteId: string; + visitId: string; hostname: string; browser: string; os: any; @@ -67,12 +68,14 @@ export async function findSession(req: NextApiRequestCollect): Promise<{ await getClientInfo(req, payload); const sessionId = uuid(websiteId, hostname, ip, userAgent); + const visitId = uuid(sessionId, sessionSalt()); // Clickhouse does not require session lookup if (clickhouse.enabled) { return { id: sessionId, websiteId, + visitId, hostname, browser, os: os as any, @@ -114,7 +117,7 @@ export async function findSession(req: NextApiRequestCollect): Promise<{ } } - return { ...session, ownerId: website.userId }; + return { ...session, ownerId: website.userId, visitId: visitId }; } async function checkUserBlock(userId: string) { diff --git a/src/pages/api/send.ts b/src/pages/api/send.ts index 5aa367f0..726a6fcc 100644 --- a/src/pages/api/send.ts +++ b/src/pages/api/send.ts @@ -1,7 +1,7 @@ import ipaddr from 'ipaddr.js'; import { isbot } from 'isbot'; import { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants'; -import { secret } from 'lib/crypto'; +import { secret, sessionSalt, uuid } from 'lib/crypto'; import { getIpAddress } from 'lib/detect'; import { useCors, useSession, useValidate } from 'lib/middleware'; import { CollectionType, YupRequest } from 'lib/types'; @@ -31,6 +31,7 @@ export interface NextApiRequestCollect extends NextApiRequest { session: { id: string; websiteId: string; + visitId: string; ownerId: string; hostname: string; browser: string; @@ -93,6 +94,14 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { const session = req.session; + // expire visitId after 30 minutes + session.visitId = + !!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800 + ? uuid(session.id, sessionSalt()) + : session.visitId; + + session.iat = Math.floor(new Date().getTime() / 1000); + if (type === COLLECTION_TYPE.event) { // eslint-disable-next-line prefer-const let [urlPath, urlQuery] = url?.split('?') || []; @@ -125,6 +134,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { eventData, ...session, sessionId: session.id, + visitId: session.visitId, }); } diff --git a/src/queries/analytics/events/saveEvent.ts b/src/queries/analytics/events/saveEvent.ts index 0596023b..25bcf9e7 100644 --- a/src/queries/analytics/events/saveEvent.ts +++ b/src/queries/analytics/events/saveEvent.ts @@ -6,8 +6,9 @@ import { uuid } from 'lib/crypto'; import { saveEventData } from 'queries/analytics/eventData/saveEventData'; export async function saveEvent(args: { - sessionId: string; websiteId: string; + sessionId: string; + visitId: string; urlPath: string; urlQuery?: string; referrerPath?: string; @@ -34,8 +35,9 @@ export async function saveEvent(args: { } async function relationalQuery(data: { - sessionId: string; websiteId: string; + sessionId: string; + visitId: string; urlPath: string; urlQuery?: string; referrerPath?: string; @@ -48,6 +50,7 @@ async function relationalQuery(data: { const { websiteId, sessionId, + visitId, urlPath, urlQuery, referrerPath, @@ -64,6 +67,7 @@ async function relationalQuery(data: { id: websiteEventId, websiteId, sessionId, + visitId, urlPath: urlPath?.substring(0, URL_LENGTH), urlQuery: urlQuery?.substring(0, URL_LENGTH), referrerPath: referrerPath?.substring(0, URL_LENGTH), @@ -90,8 +94,9 @@ async function relationalQuery(data: { } async function clickhouseQuery(data: { - sessionId: string; websiteId: string; + sessionId: string; + visitId: string; urlPath: string; urlQuery?: string; referrerPath?: string; @@ -114,6 +119,7 @@ async function clickhouseQuery(data: { const { websiteId, sessionId, + visitId, urlPath, urlQuery, referrerPath, @@ -136,6 +142,7 @@ async function clickhouseQuery(data: { ...args, website_id: websiteId, session_id: sessionId, + visit_id: visitId, event_id: uuid(), country: country, subdivision1: