Merge remote-tracking branch 'origin/dev' into dev

db/clickhouse/migrations/02_add_visit_id.sql

@@ -0,0 +1,91 @@
+CREATE TABLE umami.website_event_join
+(
+    session_id UUID,
+    visit_id UUID,
+    created_at DateTime('UTC')
+)
+    engine = MergeTree
+        ORDER BY (session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_join
+SELECT DISTINCT
+    s.session_id,
+    generateUUIDv4() visit_id,
+    s.created_at
+FROM (SELECT DISTINCT session_id,
+        date_trunc('hour', created_at) created_at
+    FROM website_event) s;
+
+-- create new table
+CREATE TABLE umami.website_event_new
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    event_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    subdivision2 LowCardinality(String),
+    city String,
+    url_path String,
+    url_query String,
+    referrer_path String,
+    referrer_query String,
+    referrer_domain String,
+    page_title String,
+    event_type UInt32,
+    event_name String,
+    created_at DateTime('UTC'),
+    job_id UUID
+)
+    engine = MergeTree
+        ORDER BY (website_id, session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_new
+SELECT we.website_id,
+    we.session_id,
+    j.visit_id,
+    we.event_id,
+    we.hostname,
+    we.browser,
+    we.os,
+    we.device,
+    we.screen,
+    we.language,
+    we.country,
+    we.subdivision1,
+    we.subdivision2,
+    we.city,
+    we.url_path,
+    we.url_query,
+    we.referrer_path,
+    we.referrer_query,
+    we.referrer_domain,
+    we.page_title,
+    we.event_type,
+    we.event_name,
+    we.created_at,
+    we.job_id
+FROM umami.website_event we
+JOIN umami.website_event_join j
+    ON we.session_id = j.session_id
+        and date_trunc('hour', we.created_at) = j.created_at
+WHERE we.created_at > '2023-03-31';
+
+RENAME TABLE umami.website_event TO umami.website_event_old;
+RENAME TABLE umami.website_event_new TO umami.website_event;
+
+/*
+
+ DROP TABLE umami.website_event_old
+ DROP TABLE umami.website_event_join
+
+ */

db/clickhouse/schema.sql

@@ -3,6 +3,7 @@ CREATE TABLE umami.website_event
 (
     website_id UUID,
     session_id UUID,
+    visit_id UUID,
     event_id UUID,
     --sessions
     hostname LowCardinality(String),

db/mysql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE `website_event` ADD COLUMN `visit_id` VARCHAR(36) NULL;
+
+UPDATE `website_event` we
+JOIN (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        BIN_TO_UUID(RANDOM_BYTES(16) & 0xffffffffffff0fff3fffffffffffffff | 0x00000000000040008000000000000000) uuid
+    FROM (SELECT DISTINCT session_id,
+            DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00') visit_time
+        FROM `website_event`) s) a
+    ON we.session_id = a.session_id and DATE_FORMAT(we.created_at, '%Y-%m-%d %H:00:00') = a.visit_time
+SET we.visit_id = a.uuid
+WHERE we.visit_id IS NULL;
+
+ALTER TABLE `website_event` MODIFY `visit_id` VARCHAR(36) NOT NULL;
+
+-- CreateIndex
+CREATE INDEX `website_event_visit_id_idx` ON `website_event`(`visit_id`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_visit_id_created_at_idx` ON `website_event`(`website_id`, `visit_id`, `created_at`);

db/mysql/schema.prisma

@@ -92,6 +92,7 @@ model WebsiteEvent {
   id             String    @id() @map("event_id") @db.VarChar(36)
   websiteId      String    @map("website_id") @db.VarChar(36)
   sessionId      String    @map("session_id") @db.VarChar(36)
+  visitId        String    @map("visit_id") @db.VarChar(36)
   createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
   urlPath        String    @map("url_path") @db.VarChar(500)
   urlQuery       String?   @map("url_query") @db.VarChar(500)
@@ -107,6 +108,7 @@ model WebsiteEvent {
 
   @@index([createdAt])
   @@index([sessionId])
+  @@index([visitId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
   @@index([websiteId, createdAt, urlPath])
@@ -115,6 +117,7 @@ model WebsiteEvent {
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
+  @@index([websiteId, visitId, createdAt])
   @@map("website_event")
 }
 

db/postgresql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE "website_event" ADD COLUMN "visit_id" UUID NULL;
+
+UPDATE "website_event" we
+SET visit_id = a.uuid
+FROM (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        gen_random_uuid() uuid
+    FROM (SELECT DISTINCT session_id,
+            date_trunc('hour', created_at) visit_time
+        FROM "website_event") s) a
+WHERE we.session_id = a.session_id 
+    and date_trunc('hour', we.created_at) = a.visit_time;
+
+ALTER TABLE "website_event" ALTER COLUMN "visit_id" SET NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "website_event_visit_id_idx" ON "website_event"("visit_id");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_visit_id_created_at_idx" ON "website_event"("website_id", "visit_id", "created_at");

db/postgresql/schema.prisma

@@ -92,6 +92,7 @@ model WebsiteEvent {
   id             String    @id() @map("event_id") @db.Uuid
   websiteId      String    @map("website_id") @db.Uuid
   sessionId      String    @map("session_id") @db.Uuid
+  visitId        String    @map("visit_id") @db.Uuid
   createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
   urlPath        String    @map("url_path") @db.VarChar(500)
   urlQuery       String?   @map("url_query") @db.VarChar(500)
@@ -107,6 +108,7 @@ model WebsiteEvent {
 
   @@index([createdAt])
   @@index([sessionId])
+  @@index([visitId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
   @@index([websiteId, createdAt, urlPath])
@@ -115,6 +117,7 @@ model WebsiteEvent {
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
+  @@index([websiteId, visitId, createdAt])
   @@map("website_event")
 }
 

src/components/metrics/DatePickerForm.module.css

@@ -9,10 +9,6 @@
   justify-content: center;
 }
 
-.calendars > div {
-  width: 380px;
-}
-
 .calendars > div + div {
   margin-inline-start: 20px;
   padding-inline-start: 20px;

src/lib/crypto.ts

@@ -1,4 +1,4 @@
-import { startOfMonth } from 'date-fns';
+import { startOfHour, startOfMonth } from 'date-fns';
 import { hash } from 'next-basics';
 import { v4, v5, validate } from 'uuid';
 
@@ -12,6 +12,12 @@ export function salt() {
   return hash(secret(), ROTATING_SALT);
 }
 
+export function sessionSalt() {
+  const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString());
+
+  return hash(secret(), ROTATING_SALT);
+}
+
 export function uuid(...args: any) {
   if (!args.length) return v4();
 

src/lib/session.ts

@@ -1,4 +1,4 @@
-import { isUuid, secret, uuid } from 'lib/crypto';
+import { isUuid, secret, sessionSalt, uuid } from 'lib/crypto';
 import { getClientInfo } from 'lib/detect';
 import { parseToken } from 'next-basics';
 import { NextApiRequestCollect } from 'pages/api/send';
@@ -10,6 +10,7 @@ import { loadSession, loadWebsite } from './load';
 export async function findSession(req: NextApiRequestCollect): Promise<{
   id: any;
   websiteId: string;
+  visitId: string;
   hostname: string;
   browser: string;
   os: any;
@@ -67,12 +68,14 @@ export async function findSession(req: NextApiRequestCollect): Promise<{
     await getClientInfo(req, payload);
 
   const sessionId = uuid(websiteId, hostname, ip, userAgent);
+  const visitId = uuid(sessionId, sessionSalt());
 
   // Clickhouse does not require session lookup
   if (clickhouse.enabled) {
     return {
       id: sessionId,
       websiteId,
+      visitId,
       hostname,
       browser,
       os: os as any,
@@ -114,7 +117,7 @@ export async function findSession(req: NextApiRequestCollect): Promise<{
     }
   }
 
-  return { ...session, ownerId: website.userId };
+  return { ...session, ownerId: website.userId, visitId: visitId };
 }
 
 async function checkUserBlock(userId: string) {

src/pages/api/send.ts

@@ -1,7 +1,7 @@
 import ipaddr from 'ipaddr.js';
 import { isbot } from 'isbot';
 import { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants';
-import { secret } from 'lib/crypto';
+import { secret, sessionSalt, uuid } from 'lib/crypto';
 import { getIpAddress } from 'lib/detect';
 import { useCors, useSession, useValidate } from 'lib/middleware';
 import { CollectionType, YupRequest } from 'lib/types';
@@ -31,6 +31,7 @@ export interface NextApiRequestCollect extends NextApiRequest {
   session: {
     id: string;
     websiteId: string;
+    visitId: string;
     ownerId: string;
     hostname: string;
     browser: string;
@@ -42,6 +43,7 @@ export interface NextApiRequestCollect extends NextApiRequest {
     subdivision1: string;
     subdivision2: string;
     city: string;
+    iat: number;
   };
   headers: { [key: string]: any };
   yup: YupRequest;
@@ -93,6 +95,14 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
 
     const session = req.session;
 
+    // expire visitId after 30 minutes
+    session.visitId =
+      !!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800
+        ? uuid(session.id, sessionSalt())
+        : session.visitId;
+
+    session.iat = Math.floor(new Date().getTime() / 1000);
+
     if (type === COLLECTION_TYPE.event) {
       // eslint-disable-next-line prefer-const
       let [urlPath, urlQuery] = url?.split('?') || [];
@@ -125,6 +135,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
         eventData,
         ...session,
         sessionId: session.id,
+        visitId: session.visitId,
       });
     }
 

src/queries/analytics/events/saveEvent.ts

@@ -6,8 +6,9 @@ import { uuid } from 'lib/crypto';
 import { saveEventData } from 'queries/analytics/eventData/saveEventData';
 
 export async function saveEvent(args: {
-  sessionId: string;
   websiteId: string;
+  sessionId: string;
+  visitId: string;
   urlPath: string;
   urlQuery?: string;
   referrerPath?: string;
@@ -34,8 +35,9 @@ export async function saveEvent(args: {
 }
 
 async function relationalQuery(data: {
-  sessionId: string;
   websiteId: string;
+  sessionId: string;
+  visitId: string;
   urlPath: string;
   urlQuery?: string;
   referrerPath?: string;
@@ -48,6 +50,7 @@ async function relationalQuery(data: {
   const {
     websiteId,
     sessionId,
+    visitId,
     urlPath,
     urlQuery,
     referrerPath,
@@ -64,6 +67,7 @@ async function relationalQuery(data: {
       id: websiteEventId,
       websiteId,
       sessionId,
+      visitId,
       urlPath: urlPath?.substring(0, URL_LENGTH),
       urlQuery: urlQuery?.substring(0, URL_LENGTH),
       referrerPath: referrerPath?.substring(0, URL_LENGTH),
@@ -90,8 +94,9 @@ async function relationalQuery(data: {
 }
 
 async function clickhouseQuery(data: {
-  sessionId: string;
   websiteId: string;
+  sessionId: string;
+  visitId: string;
   urlPath: string;
   urlQuery?: string;
   referrerPath?: string;
@@ -114,6 +119,7 @@ async function clickhouseQuery(data: {
   const {
     websiteId,
     sessionId,
+    visitId,
     urlPath,
     urlQuery,
     referrerPath,
@@ -136,6 +142,7 @@ async function clickhouseQuery(data: {
     ...args,
     website_id: websiteId,
     session_id: sessionId,
+    visit_id: visitId,
     event_id: uuid(),
     country: country,
     subdivision1: