m
/
umami
mirror of https://github.com/kremalicious/umami.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Mike Cao 2024-03-25 22:51:24 -07:00
parent 91efb7f1d0 7a6a598a19
commit 1a839d1cae
11 changed files with 176 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
db/clickhouse/migrations/02_add_visit_id.sql Normal file
View File

@ -0,0 +1,91 @@
CREATE TABLE umami.website_event_join
(
session_id UUID,
visit_id UUID,
created_at DateTime('UTC')
)
engine = MergeTree
ORDER BY (session_id, created_at)
SETTINGS index_granularity = 8192;
INSERT INTO umami.website_event_join
SELECT DISTINCT
s.session_id,
generateUUIDv4() visit_id,
s.created_at
FROM (SELECT DISTINCT session_id,
date_trunc('hour', created_at) created_at
FROM website_event) s;
-- create new table
CREATE TABLE umami.website_event_new
(
website_id UUID,
session_id UUID,
visit_id UUID,
event_id UUID,
hostname LowCardinality(String),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
subdivision2 LowCardinality(String),
city String,
url_path String,
url_query String,
referrer_path String,
referrer_query String,
referrer_domain String,
page_title String,
event_type UInt32,
event_name String,
created_at DateTime('UTC'),
job_id UUID
)
engine = MergeTree
ORDER BY (website_id, session_id, created_at)
SETTINGS index_granularity = 8192;
INSERT INTO umami.website_event_new
SELECT we.website_id,
we.session_id,
j.visit_id,
we.event_id,
we.hostname,
we.browser,
we.os,
we.device,
we.screen,
we.language,
we.country,
we.subdivision1,
we.subdivision2,
we.city,
we.url_path,
we.url_query,
we.referrer_path,
we.referrer_query,
we.referrer_domain,
we.page_title,
we.event_type,
we.event_name,
we.created_at,
we.job_id
FROM umami.website_event we
JOIN umami.website_event_join j
ON we.session_id = j.session_id
and date_trunc('hour', we.created_at) = j.created_at
WHERE we.created_at > '2023-03-31';
RENAME TABLE umami.website_event TO umami.website_event_old;
RENAME TABLE umami.website_event_new TO umami.website_event;
/*
DROP TABLE umami.website_event_old
DROP TABLE umami.website_event_join
*/

1
db/clickhouse/schema.sql
View File

@ -3,6 +3,7 @@ CREATE TABLE umami.website_event
(
website_id UUID,
session_id UUID,
visit_id UUID,
event_id UUID,
--sessions
hostname LowCardinality(String),

22
db/mysql/migrations/05_add_visit_id/migration.sql Normal file
View File

@ -0,0 +1,22 @@
-- AlterTable
ALTER TABLE `website_event` ADD COLUMN `visit_id` VARCHAR(36) NULL;
UPDATE `website_event` we
JOIN (SELECT DISTINCT
s.session_id,
s.visit_time,
BIN_TO_UUID(RANDOM_BYTES(16) & 0xffffffffffff0fff3fffffffffffffff | 0x00000000000040008000000000000000) uuid
FROM (SELECT DISTINCT session_id,
DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00') visit_time
FROM `website_event`) s) a
ON we.session_id = a.session_id and DATE_FORMAT(we.created_at, '%Y-%m-%d %H:00:00') = a.visit_time
SET we.visit_id = a.uuid
WHERE we.visit_id IS NULL;
ALTER TABLE `website_event` MODIFY `visit_id` VARCHAR(36) NOT NULL;
-- CreateIndex
CREATE INDEX `website_event_visit_id_idx` ON `website_event`(`visit_id`);
-- CreateIndex
CREATE INDEX `website_event_website_id_visit_id_created_at_idx` ON `website_event`(`website_id`, `visit_id`, `created_at`);

3
db/mysql/schema.prisma
View File

@ -92,6 +92,7 @@ model WebsiteEvent {
id String @id() @map("event_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
sessionId String @map("session_id") @db.VarChar(36)
visitId String @map("visit_id") @db.VarChar(36)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
urlPath String @map("url_path") @db.VarChar(500)
urlQuery String? @map("url_query") @db.VarChar(500)
@ -107,6 +108,7 @@ model WebsiteEvent {
@@index([createdAt])
@@index([sessionId])
@@index([visitId])
@@index([websiteId])
@@index([websiteId, createdAt])
@@index([websiteId, createdAt, urlPath])
@ -115,6 +117,7 @@ model WebsiteEvent {
@@index([websiteId, createdAt, pageTitle])
@@index([websiteId, createdAt, eventName])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, visitId, createdAt])
@@map("website_event")
}

22
db/postgresql/migrations/05_add_visit_id/migration.sql Normal file
View File

@ -0,0 +1,22 @@
-- AlterTable
ALTER TABLE "website_event" ADD COLUMN "visit_id" UUID NULL;
UPDATE "website_event" we
SET visit_id = a.uuid
FROM (SELECT DISTINCT
s.session_id,
s.visit_time,
gen_random_uuid() uuid
FROM (SELECT DISTINCT session_id,
date_trunc('hour', created_at) visit_time
FROM "website_event") s) a
WHERE we.session_id = a.session_id
and date_trunc('hour', we.created_at) = a.visit_time;
ALTER TABLE "website_event" ALTER COLUMN "visit_id" SET NOT NULL;
-- CreateIndex
CREATE INDEX "website_event_visit_id_idx" ON "website_event"("visit_id");
-- CreateIndex
CREATE INDEX "website_event_website_id_visit_id_created_at_idx" ON "website_event"("website_id", "visit_id", "created_at");

3
db/postgresql/schema.prisma
View File

@ -92,6 +92,7 @@ model WebsiteEvent {
id String @id() @map("event_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
sessionId String @map("session_id") @db.Uuid
visitId String @map("visit_id") @db.Uuid
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
urlPath String @map("url_path") @db.VarChar(500)
urlQuery String? @map("url_query") @db.VarChar(500)
@ -107,6 +108,7 @@ model WebsiteEvent {
@@index([createdAt])
@@index([sessionId])
@@index([visitId])
@@index([websiteId])
@@index([websiteId, createdAt])
@@index([websiteId, createdAt, urlPath])
@ -115,6 +117,7 @@ model WebsiteEvent {
@@index([websiteId, createdAt, pageTitle])
@@index([websiteId, createdAt, eventName])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, visitId, createdAt])
@@map("website_event")
}

4
src/components/metrics/DatePickerForm.module.css
View File

@ -9,10 +9,6 @@
justify-content: center;
}
.calendars > div {
width: 380px;
}
.calendars > div + div {
margin-inline-start: 20px;
padding-inline-start: 20px;

8
src/lib/crypto.ts
View File

@ -1,4 +1,4 @@
import { startOfMonth } from 'date-fns';
import { startOfHour, startOfMonth } from 'date-fns';
import { hash } from 'next-basics';
import { v4, v5, validate } from 'uuid';
@ -12,6 +12,12 @@ export function salt() {
return hash(secret(), ROTATING_SALT);
}
export function sessionSalt() {
const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString());
return hash(secret(), ROTATING_SALT);
}
export function uuid(...args: any) {
if (!args.length) return v4();

7
src/lib/session.ts
View File

@ -1,4 +1,4 @@
import { isUuid, secret, uuid } from 'lib/crypto';
import { isUuid, secret, sessionSalt, uuid } from 'lib/crypto';
import { getClientInfo } from 'lib/detect';
import { parseToken } from 'next-basics';
import { NextApiRequestCollect } from 'pages/api/send';
@ -10,6 +10,7 @@ import { loadSession, loadWebsite } from './load';
export async function findSession(req: NextApiRequestCollect): Promise<{
id: any;
websiteId: string;
visitId: string;
hostname: string;
browser: string;
os: any;
@ -67,12 +68,14 @@ export async function findSession(req: NextApiRequestCollect): Promise<{
await getClientInfo(req, payload);
const sessionId = uuid(websiteId, hostname, ip, userAgent);
const visitId = uuid(sessionId, sessionSalt());
// Clickhouse does not require session lookup
if (clickhouse.enabled) {
return {
id: sessionId,
websiteId,
visitId,
hostname,
browser,
os: os as any,
@ -114,7 +117,7 @@ export async function findSession(req: NextApiRequestCollect): Promise<{
}
}
return { ...session, ownerId: website.userId };
return { ...session, ownerId: website.userId, visitId: visitId };
}
async function checkUserBlock(userId: string) {

13
src/pages/api/send.ts
View File

@ -1,7 +1,7 @@
import ipaddr from 'ipaddr.js';
import { isbot } from 'isbot';
import { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants';
import { secret } from 'lib/crypto';
import { secret, sessionSalt, uuid } from 'lib/crypto';
import { getIpAddress } from 'lib/detect';
import { useCors, useSession, useValidate } from 'lib/middleware';
import { CollectionType, YupRequest } from 'lib/types';
@ -31,6 +31,7 @@ export interface NextApiRequestCollect extends NextApiRequest {
session: {
id: string;
websiteId: string;
visitId: string;
ownerId: string;
hostname: string;
browser: string;
@ -42,6 +43,7 @@ export interface NextApiRequestCollect extends NextApiRequest {
subdivision1: string;
subdivision2: string;
city: string;
iat: number;
};
headers: { [key: string]: any };
yup: YupRequest;
@ -93,6 +95,14 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
const session = req.session;
// expire visitId after 30 minutes
session.visitId =
!!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800
? uuid(session.id, sessionSalt())
: session.visitId;
session.iat = Math.floor(new Date().getTime() / 1000);
if (type === COLLECTION_TYPE.event) {
// eslint-disable-next-line prefer-const
let [urlPath, urlQuery] = url?.split('?') || [];
@ -125,6 +135,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
eventData,
...session,
sessionId: session.id,
visitId: session.visitId,
});
}

13
src/queries/analytics/events/saveEvent.ts
View File

@ -6,8 +6,9 @@ import { uuid } from 'lib/crypto';
import { saveEventData } from 'queries/analytics/eventData/saveEventData';
export async function saveEvent(args: {
sessionId: string;
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
urlQuery?: string;
referrerPath?: string;
@ -34,8 +35,9 @@ export async function saveEvent(args: {
}
async function relationalQuery(data: {
sessionId: string;
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
urlQuery?: string;
referrerPath?: string;
@ -48,6 +50,7 @@ async function relationalQuery(data: {
const {
websiteId,
sessionId,
visitId,
urlPath,
urlQuery,
referrerPath,
@ -64,6 +67,7 @@ async function relationalQuery(data: {
id: websiteEventId,
websiteId,
sessionId,
visitId,
urlPath: urlPath?.substring(0, URL_LENGTH),
urlQuery: urlQuery?.substring(0, URL_LENGTH),
referrerPath: referrerPath?.substring(0, URL_LENGTH),
@ -90,8 +94,9 @@ async function relationalQuery(data: {
}
async function clickhouseQuery(data: {
sessionId: string;
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
urlQuery?: string;
referrerPath?: string;
@ -114,6 +119,7 @@ async function clickhouseQuery(data: {
const {
websiteId,
sessionId,
visitId,
urlPath,
urlQuery,
referrerPath,
@ -136,6 +142,7 @@ async function clickhouseQuery(data: {
...args,
website_id: websiteId,
session_id: sessionId,
visit_id: visitId,
event_id: uuid(),
country: country,
subdivision1: