m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2024-11-15 17:55:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

Feat/um 305 unique session ch (#2065)

Browse Source 
* Add session_data / session redis to CH.

* Add mysql migration.
This commit is contained in:
Brian Cao 2023-05-31 21:46:49 -07:00 committed by GitHub
parent 1038a54fe4
commit b484286523
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 405 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
components/pages/console/TestConsole.js
View File

@ -28,22 +28,41 @@ export function TestConsole() {
window.umami.track({ url: '/page-view', referrer: 'https://www.google.com' });
window.umami.track('track-event-no-data');
window.umami.track('track-event-with-data', {
data: {
test: 'test-data',
boolean: true,
booleanError: 'true',
time: new Date(),
number: 1,
time2: new Date().toISOString(),
nested: {
test: 'test-data',
boolean: true,
booleanError: 'true',
time: new Date(),
number: 1,
time2: new Date().toISOString(),
nested: {
object: {
test: 'test-data',
number: 1,
object: {
test: 'test-data',
},
},
array: [1, 2, 3],
},
array: [1, 2, 3],
});
}
function handleIdentifyClick() {
window.umami.identify({
userId: 123,
name: 'brian',
number: Math.random() * 100,
test: 'test-data',
boolean: true,
booleanError: 'true',
time: new Date(),
time2: new Date().toISOString(),
nested: {
test: 'test-data',
number: 1,
object: {
test: 'test-data',
},
},
array: [1, 2, 3],
});
}
@ -116,6 +135,10 @@ export function TestConsole() {
<Button id="manual-button" variant="action" onClick={handleClick}>
Run script
</Button>
<p />
<Button id="manual-button" variant="action" onClick={handleIdentifyClick}>
Run identify
</Button>
</Column>
</Row>
<Row>

4
db/clickhouse/migrations/01_edit_keys.sql Normal file
View File

@ -0,0 +1,4 @@
ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value";
ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value";
ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value";
ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";

24
db/clickhouse/schema.sql
View File

@ -117,10 +117,10 @@ CREATE TABLE umami.event_data
url_path String,
event_name String,
event_key String,
event_string_value Nullable(String),
event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
event_date_value Nullable(DateTime('UTC')),
event_data_type UInt32,
string_value Nullable(String),
numeric_value Nullable(Decimal64(4)), --922337203685477.5625
date_value Nullable(DateTime('UTC')),
data_type UInt32,
created_at DateTime('UTC')
)
engine = MergeTree
@ -134,10 +134,10 @@ CREATE TABLE umami.event_data_queue (
url_path String,
event_name String,
event_key String,
event_string_value Nullable(String),
event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
event_date_value Nullable(DateTime('UTC')),
event_data_type UInt32,
string_value Nullable(String),
numeric_value Nullable(Decimal64(4)), --922337203685477.5625
date_value Nullable(DateTime('UTC')),
data_type UInt32,
created_at DateTime('UTC'),
--virtual columns
_error String,
@ -158,10 +158,10 @@ SELECT website_id,
url_path,
event_name,
event_key,
event_string_value,
event_numeric_value,
event_date_value,
event_data_type,
string_value,
numeric_value,
date_value,
data_type,
created_at
FROM umami.event_data_queue;

37
db/mysql/migrations/03_session_data/migration.sql Normal file
View File

@ -0,0 +1,37 @@
/*
Warnings:
- The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to drop the column `event_data_type` on the `event_data` table. All the data in the column will be lost.
- You are about to drop the column `event_date_value` on the `event_data` table. All the data in the column will be lost.
- You are about to drop the column `event_id` on the `event_data` table. All the data in the column will be lost.
- You are about to drop the column `event_numeric_value` on the `event_data` table. All the data in the column will be lost.
- You are about to drop the column `event_string_value` on the `event_data` table. All the data in the column will be lost.
- Added the required column `data_type` to the `event_data` table without a default value. This is not possible if the table is not empty.
- Added the required column `event_data_id` to the `event_data` table without a default value. This is not possible if the table is not empty.
*/
-- AlterTable
ALTER TABLE `event_data` RENAME COLUMN `event_data_type` TO `data_type`;
ALTER TABLE `event_data` RENAME COLUMN `event_date_value` TO `date_value`;
ALTER TABLE `event_data` RENAME COLUMN `event_id` TO `event_data_id`;
ALTER TABLE `event_data` RENAME COLUMN `event_numeric_value` TO `numeric_value`;
ALTER TABLE `event_data` RENAME COLUMN `event_string_value` TO `string_value`;
-- CreateTable
CREATE TABLE `session_data` (
`session_data_id` VARCHAR(36) NOT NULL,
`website_id` VARCHAR(36) NOT NULL,
`session_id` VARCHAR(36) NOT NULL,
`event_key` VARCHAR(500) NOT NULL,
`event_string_value` VARCHAR(500) NULL,
`event_numeric_value` DECIMAL(19, 4) NULL,
`event_date_value` TIMESTAMP(0) NULL,
`event_data_type` INTEGER UNSIGNED NOT NULL,
`created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
INDEX `session_data_created_at_idx`(`created_at`),
INDEX `session_data_website_id_idx`(`website_id`),
INDEX `session_data_session_id_idx`(`session_id`),
PRIMARY KEY (`session_data_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

54
db/mysql/schema.prisma
View File

@ -14,12 +14,12 @@ model User {
password String @db.VarChar(60)
role String @map("role") @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
website Website[]
teamUser TeamUser[]
Report Report[]
report Report[]
@@map("user")
}
@ -40,6 +40,7 @@ model Session {
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
websiteEvent WebsiteEvent[]
sessionData SessionData[]
@@index([createdAt])
@@index([websiteId])
@ -54,13 +55,14 @@ model Website {
resetAt DateTime? @map("reset_at") @db.Timestamp(0)
userId String? @map("user_id") @db.VarChar(36)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
user User? @relation(fields: [userId], references: [id])
teamWebsite TeamWebsite[]
eventData EventData[]
Report Report[]
report Report[]
sessionData SessionData[]
@@index([userId])
@@index([createdAt])
@ -94,15 +96,15 @@ model WebsiteEvent {
}
model EventData {
id String @id() @map("event_id") @db.VarChar(36)
websiteEventId String @map("website_event_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
eventKey String @map("event_key") @db.VarChar(500)
eventStringValue String? @map("event_string_value") @db.VarChar(500)
eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)
eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0)
eventDataType Int @map("event_data_type") @db.UnsignedInt
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
id String @id() @map("event_data_id") @db.VarChar(36)
websiteEventId String @map("website_event_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
eventKey String @map("event_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamp(0)
dataType Int @map("data_type") @db.UnsignedInt
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
website Website @relation(fields: [websiteId], references: [id])
websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
@ -114,12 +116,32 @@ model EventData {
@@map("event_data")
}
model SessionData {
id String @id() @map("session_data_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
sessionId String @map("session_id") @db.VarChar(36)
eventKey String @map("event_key") @db.VarChar(500)
eventStringValue String? @map("event_string_value") @db.VarChar(500)
eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)
eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0)
eventDataType Int @map("event_data_type") @db.UnsignedInt
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
website Website @relation(fields: [websiteId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([createdAt])
@@index([websiteId])
@@index([sessionId])
@@map("session_data")
}
model Team {
id String @id() @unique() @map("team_id") @db.VarChar(36)
name String @db.VarChar(50)
accessCode String? @unique @map("access_code") @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
teamUser TeamUser[]
teamWebsite TeamWebsite[]
@ -134,7 +156,7 @@ model TeamUser {
userId String @map("user_id") @db.VarChar(36)
role String @map("role") @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
team Team @relation(fields: [teamId], references: [id])
user User @relation(fields: [userId], references: [id])
@ -177,4 +199,4 @@ model Report {
@@index([type])
@@index([name])
@@map("report")
}
}

31
db/postgresql/migrations/03_session_data/migration.sql Normal file
View File

@ -0,0 +1,31 @@
-- AlterTable
ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";
ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value";
ALTER TABLE "event_data" RENAME COLUMN "event_id" TO "event_data_id";
ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value";
ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value";
-- CreateTable
CREATE TABLE "session_data" (
"session_data_id" UUID NOT NULL,
"website_id" UUID NOT NULL,
"session_id" UUID NOT NULL,
"session_key" VARCHAR(500) NOT NULL,
"string_value" VARCHAR(500),
"numeric_value" DECIMAL(19,4),
"date_value" TIMESTAMPTZ(6),
"data_type" INTEGER NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
"deleted_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "session_data_pkey" PRIMARY KEY ("session_data_id")
);
-- CreateIndex
CREATE INDEX "session_data_created_at_idx" ON "session_data"("created_at");
-- CreateIndex
CREATE INDEX "session_data_website_id_idx" ON "session_data"("website_id");
-- CreateIndex
CREATE INDEX "session_data_session_id_idx" ON "session_data"("session_id");

45
db/postgresql/schema.prisma
View File

@ -19,7 +19,7 @@ model User {
website Website[]
teamUser TeamUser[]
Report Report[]
report Report[]
@@map("user")
}
@ -40,6 +40,7 @@ model Session {
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
websiteEvent WebsiteEvent[]
sessionData SessionData[]
@@index([createdAt])
@@index([websiteId])
@ -60,7 +61,8 @@ model Website {
user User? @relation(fields: [userId], references: [id])
teamWebsite TeamWebsite[]
eventData EventData[]
Report Report[]
report Report[]
sessionData SessionData[]
@@index([userId])
@@index([createdAt])
@ -94,15 +96,15 @@ model WebsiteEvent {
}
model EventData {
id String @id() @map("event_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
websiteEventId String @map("website_event_id") @db.Uuid
eventKey String @map("event_key") @db.VarChar(500)
eventStringValue String? @map("event_string_value") @db.VarChar(500)
eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)
eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6)
eventDataType Int @map("event_data_type") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
id String @id() @map("event_data_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
websiteEventId String @map("website_event_id") @db.Uuid
eventKey String @map("event_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamptz(6)
dataType Int @map("data_type") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
@ -113,6 +115,27 @@ model EventData {
@@map("event_data")
}
model SessionData {
id String @id() @map("session_data_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
sessionId String @map("session_id") @db.Uuid
sessionKey String @map("session_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamptz(6)
dataType Int @map("data_type") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
deletedAt DateTime? @default(now()) @map("deleted_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([createdAt])
@@index([websiteId])
@@index([sessionId])
@@map("session_data")
}
model Team {
id String @id() @unique() @map("team_id") @db.Uuid
name String @db.VarChar(50)

4
lib/clickhouse.ts
View File

@ -2,7 +2,7 @@ import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import debug from 'debug';
import { CLICKHOUSE } from 'lib/db';
import { getEventDataType } from './eventData';
import { getDynamicDataType } from './dynamicData';
import { WebsiteMetricFilter } from './types';
import { FILTER_COLUMNS } from './constants';
@ -74,7 +74,7 @@ function getEventDataFilterQuery(
params: any,
) {
const query = filters.reduce((ac, cv, i) => {
const type = getEventDataType(cv.eventValue);
const type = getDynamicDataType(cv.eventValue);
let value = cv.eventValue;

7
lib/constants.ts
View File

@ -42,6 +42,11 @@ export const SESSION_COLUMNS = [
'city',
];
export const COLLECTION_TYPE = {
event: 'event',
identify: 'identify',
};
export const FILTER_COLUMNS = {
url: 'url_path',
referrer: 'referrer_domain',
@ -56,7 +61,7 @@ export const EVENT_TYPE = {
customEvent: 2,
} as const;
export const EVENT_DATA_TYPE = {
export const DYNAMIC_DATA_TYPE = {
string: 1,
number: 2,
boolean: 3,

28
lib/eventData.ts → lib/dynamicData.ts
View File

@ -1,12 +1,12 @@
import { isValid, parseISO } from 'date-fns';
import { EVENT_DATA_TYPE } from './constants';
import { EventDataTypes } from './types';
import { DYNAMIC_DATA_TYPE } from './constants';
import { DynamicDataType } from './types';
export function flattenJSON(
eventData: { [key: string]: any },
keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [],
keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],
parentKey = '',
): { key: string; value: any; eventDataType: EventDataTypes }[] {
): { key: string; value: any; dynamicDataType: DynamicDataType }[] {
return Object.keys(eventData).reduce(
(acc, key) => {
const value = eventData[key];
@ -25,7 +25,7 @@ export function flattenJSON(
).keyValues;
}
export function getEventDataType(value: any): string {
export function getDynamicDataType(value: any): string {
let type: string = typeof value;
if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
@ -36,34 +36,34 @@ export function getEventDataType(value: any): string {
}
function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
const type = getEventDataType(value);
const type = getDynamicDataType(value);
let eventDataType = null;
let dynamicDataType = null;
switch (type) {
case 'number':
eventDataType = EVENT_DATA_TYPE.number;
dynamicDataType = DYNAMIC_DATA_TYPE.number;
break;
case 'string':
eventDataType = EVENT_DATA_TYPE.string;
dynamicDataType = DYNAMIC_DATA_TYPE.string;
break;
case 'boolean':
eventDataType = EVENT_DATA_TYPE.boolean;
dynamicDataType = DYNAMIC_DATA_TYPE.boolean;
value = value ? 'true' : 'false';
break;
case 'date':
eventDataType = EVENT_DATA_TYPE.date;
dynamicDataType = DYNAMIC_DATA_TYPE.date;
break;
case 'object':
eventDataType = EVENT_DATA_TYPE.array;
dynamicDataType = DYNAMIC_DATA_TYPE.array;
value = JSON.stringify(value);
break;
default:
eventDataType = EVENT_DATA_TYPE.string;
dynamicDataType = DYNAMIC_DATA_TYPE.string;
break;
}
acc.keyValues.push({ key, value, eventDataType });
acc.keyValues.push({ key, value, dynamicDataType });
}
function getKeyName(key, parentKey) {

4
lib/prisma.ts
View File

@ -1,7 +1,7 @@
import prisma from '@umami/prisma-client';
import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { getEventDataType } from './eventData';
import { getDynamicDataType } from './dynamicData';
import { FILTER_COLUMNS } from './constants';
const MYSQL_DATE_FORMATS = {
@ -85,7 +85,7 @@ function getEventDataFilterQuery(
params: any[],
) {
const query = filters.reduce((ac, cv) => {
const type = getEventDataType(cv.eventValue);
const type = getDynamicDataType(cv.eventValue);
let value = cv.eventValue;

23
lib/session.ts
View File

@ -1,12 +1,11 @@
import clickhouse from 'lib/clickhouse';
import { secret, uuid } from 'lib/crypto';
import { getClientInfo, getJsonBody } from 'lib/detect';
import { parseToken } from 'next-basics';
import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
import { createSession } from 'queries';
import { validate } from 'uuid';
import { loadSession, loadWebsite } from './query';
import cache from './cache';
import { loadSession, loadWebsite } from './query';
export async function findSession(req: NextApiRequestCollect) {
const { payload } = getJsonBody<CollectRequestBody>(req);
@ -46,26 +45,8 @@ export async function findSession(req: NextApiRequestCollect) {
const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
await getClientInfo(req, payload);
const sessionId = uuid(websiteId, hostname, ip, userAgent);
// Clickhouse does not require session lookup
if (clickhouse.enabled) {
return {
id: sessionId,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city,
ownerId: website.userId,
};
}
const sessionId = uuid(websiteId, hostname, ip, userAgent);
// Find session
let session = await loadSession(sessionId);

16
lib/types.ts
View File

@ -1,18 +1,20 @@
import { NextApiRequest } from 'next';
import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
import { COLLECTION_TYPE, DYNAMIC_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
type ObjectValues<T> = T[keyof T];
export type Roles = ObjectValues<typeof ROLES>;
export type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;
export type EventTypes = ObjectValues<typeof EVENT_TYPE>;
export type Role = ObjectValues<typeof ROLES>;
export type EventDataTypes = ObjectValues<typeof EVENT_DATA_TYPE>;
export type EventType = ObjectValues<typeof EVENT_TYPE>;
export type KafkaTopics = ObjectValues<typeof KAFKA_TOPIC>;
export type DynamicDataType = ObjectValues<typeof DYNAMIC_DATA_TYPE>;
export interface EventData {
[key: string]: number | string | EventData | number[] | string[] | EventData[];
export type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;
export interface DynamicData {
[key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];
}
export interface Auth {

124
pages/api/send.ts
View File

@ -7,6 +7,9 @@ import { getJsonBody, getIpAddress } from 'lib/detect';
import { secret } from 'lib/crypto';
import { NextApiRequest, NextApiResponse } from 'next';
import { Resolver } from 'dns/promises';
import { CollectionType } from 'lib/types';
import { COLLECTION_TYPE } from 'lib/constants';
import { saveSessionData } from 'queries/analytics/session/saveSessionData';
export interface CollectRequestBody {
payload: {
@ -20,7 +23,7 @@ export interface CollectRequestBody {
website: string;
name: string;
};
type: string;
type: CollectionType;
}
export interface NextApiRequestCollect extends NextApiRequest {
@ -52,17 +55,81 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
const { type, payload } = getJsonBody<CollectRequestBody>(req);
if (type !== 'event') {
validateBody(res, { type, payload });
if (await hasBlockedIp(req)) {
return forbidden(res);
}
const { url, referrer, name: eventName, data: dynamicData, title: pageTitle } = payload;
await useSession(req, res);
const session = req.session;
if (type === COLLECTION_TYPE.event) {
// eslint-disable-next-line prefer-const
let [urlPath, urlQuery] = url?.split('?') || [];
let [referrerPath, referrerQuery] = referrer?.split('?') || [];
let referrerDomain;
if (!urlPath) {
urlPath = '/';
}
if (referrerPath?.startsWith('http')) {
const refUrl = new URL(referrer);
referrerPath = refUrl.pathname;
referrerQuery = refUrl.search.substring(1);
referrerDomain = refUrl.hostname.replace(/www\./, '');
}
if (process.env.REMOVE_TRAILING_SLASH) {
urlPath = urlPath.replace(/.+\/$/, '');
}
await saveEvent({
urlPath,
urlQuery,
referrerPath,
referrerQuery,
referrerDomain,
pageTitle,
eventName,
eventData: dynamicData,
...session,
sessionId: session.id,
});
}
if (type === COLLECTION_TYPE.identify) {
if (!dynamicData) {
return badRequest(res, 'Data required.');
}
await saveSessionData({ ...session, sessionData: dynamicData, sessionId: session.id });
}
const token = createToken(session, secret());
return send(res, token);
};
function validateBody(res: NextApiResponse, { type, payload }: CollectRequestBody) {
const { data } = payload;
// Validate type
if (type !== COLLECTION_TYPE.event && type !== COLLECTION_TYPE.identify) {
return badRequest(res, 'Wrong payload type.');
}
const { url, referrer, name: eventName, data: eventData, title: pageTitle } = payload;
// Validate eventData is JSON
if (eventData && !(typeof eventData === 'object' && !Array.isArray(eventData))) {
if (data && !(typeof data === 'object' && !Array.isArray(data))) {
return badRequest(res, 'Invalid event data.');
}
}
async function hasBlockedIp(req: NextApiRequestCollect) {
const ignoreIps = process.env.IGNORE_IP;
const ignoreHostnames = process.env.IGNORE_HOSTNAME;
@ -100,49 +167,6 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
return false;
});
if (blocked) {
return forbidden(res);
}
return blocked;
}
await useSession(req, res);
const session = req.session;
// eslint-disable-next-line prefer-const
let [urlPath, urlQuery] = url?.split('?') || [];
let [referrerPath, referrerQuery] = referrer?.split('?') || [];
let referrerDomain;
if (!urlPath) {
urlPath = '/';
}
if (referrerPath?.startsWith('http')) {
const refUrl = new URL(referrer);
referrerPath = refUrl.pathname;
referrerQuery = refUrl.search.substring(1);
referrerDomain = refUrl.hostname.replace(/www\./, '');
}
if (process.env.REMOVE_TRAILING_SLASH) {
urlPath = urlPath.replace(/.+\/$/, '');
}
await saveEvent({
urlPath,
urlQuery,
referrerPath,
referrerQuery,
referrerDomain,
pageTitle,
eventName,
eventData,
...session,
sessionId: session.id,
});
const token = createToken(session, secret());
return send(res, token);
};
}

4
pages/api/users/[id]/index.ts
View File

@ -1,4 +1,4 @@
import { NextApiRequestQueryBody, Roles, User } from 'lib/types';
import { NextApiRequestQueryBody, Role, User } from 'lib/types';
import { canDeleteUser, canUpdateUser, canViewUser } from 'lib/auth';
import { useAuth } from 'lib/middleware';
import { NextApiResponse } from 'next';
@ -12,7 +12,7 @@ export interface UserRequestQuery {
export interface UserRequestBody {
username: string;
password: string;
role: Roles;
role: Role;
}
export default async (

4
pages/api/users/index.ts
View File

@ -2,7 +2,7 @@ import { canCreateUser, canViewUsers } from 'lib/auth';
import { ROLES } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { useAuth } from 'lib/middleware';
import { NextApiRequestQueryBody, Roles, User } from 'lib/types';
import { NextApiRequestQueryBody, Role, User } from 'lib/types';
import { NextApiResponse } from 'next';
import { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics';
import { createUser, getUser, getUsers } from 'queries';
@ -11,7 +11,7 @@ export interface UsersRequestBody {
username: string;
password: string;
id: string;
role?: Roles;
role?: Role;
}
export default async (

4
queries/admin/user.ts
View File

@ -3,7 +3,7 @@ import { getRandomChars } from 'next-basics';
import cache from 'lib/cache';
import { ROLES } from 'lib/constants';
import prisma from 'lib/prisma';
import { Website, User, Roles } from 'lib/types';
import { Website, User, Role } from 'lib/types';
export async function getUser(
where: Prisma.UserWhereInput | Prisma.UserWhereUniqueInput,
@ -91,7 +91,7 @@ export async function createUser(data: {
id: string;
username: string;
password: string;
role: Roles;
role: Role;
}): Promise<{
id: string;
username: string;

4
queries/analytics/event/saveEvent.ts
View File

@ -133,9 +133,10 @@ async function clickhouseQuery(data: {
const createdAt = getDateFormat(new Date());
const message = {
...args,
website_id: websiteId,
session_id: sessionId,
event_id: eventId,
event_id: uuid(),
country: country ? country : null,
subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,
subdivision2: subdivision2 ? subdivision2 : null,
@ -149,7 +150,6 @@ async function clickhouseQuery(data: {
event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
created_at: createdAt,
...args,
};
await sendMessage(message, 'event');

42
queries/analytics/eventData/saveEventData.ts
View File

@ -1,11 +1,11 @@
import { Prisma } from '@prisma/client';
import { EVENT_DATA_TYPE } from 'lib/constants';
import { DYNAMIC_DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { flattenJSON } from 'lib/eventData';
import { flattenJSON } from 'lib/dynamicData';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
import { EventData } from 'lib/types';
import { DynamicData } from 'lib/types';
export async function saveEventData(args: {
websiteId: string;
@ -13,7 +13,7 @@ export async function saveEventData(args: {
sessionId?: string;
urlPath?: string;
eventName?: string;
eventData: EventData;
eventData: DynamicData;
createdAt?: string;
}) {
return runQuery({
@ -25,7 +25,7 @@ export async function saveEventData(args: {
async function relationalQuery(data: {
websiteId: string;
eventId: string;
eventData: EventData;
eventData: DynamicData;
}): Promise<Prisma.BatchPayload> {
const { websiteId, eventId, eventData } = data;
@ -36,16 +36,16 @@ async function relationalQuery(data: {
id: uuid(),
websiteEventId: eventId,
websiteId,
eventKey: a.key,
eventStringValue:
a.eventDataType === EVENT_DATA_TYPE.string ||
a.eventDataType === EVENT_DATA_TYPE.boolean ||
a.eventDataType === EVENT_DATA_TYPE.array
key: a.key,
stringValue:
a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||
a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||
a.dynamicDataType === DYNAMIC_DATA_TYPE.array
? a.value
: null,
eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null,
eventDataType: a.eventDataType,
numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,
dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dynamicDataType,
}));
return prisma.client.eventData.createMany({
@ -59,7 +59,7 @@ async function clickhouseQuery(data: {
sessionId?: string;
urlPath?: string;
eventName?: string;
eventData: EventData;
eventData: DynamicData;
createdAt?: string;
}) {
const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
@ -75,15 +75,15 @@ async function clickhouseQuery(data: {
url_path: urlPath,
event_name: eventName,
event_key: a.key,
event_string_value:
a.eventDataType === EVENT_DATA_TYPE.string ||
a.eventDataType === EVENT_DATA_TYPE.boolean ||
a.eventDataType === EVENT_DATA_TYPE.array
string_value:
a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||
a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||
a.dynamicDataType === DYNAMIC_DATA_TYPE.array
? a.value
: null,
event_numeric_value: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
event_date_value: a.eventDataType === EVENT_DATA_TYPE.date ? getDateFormat(a.value) : null,
event_data_type: a.eventDataType,
numeric_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,
date_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? getDateFormat(a.value) : null,
data_type: a.dynamicDataType,
created_at: createdAt,
}));

112
queries/analytics/session/createSession.ts
View File

@ -1,23 +1,8 @@
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
import cache from 'lib/cache';
import { Prisma } from '@prisma/client';
import cache from 'lib/cache';
import prisma from 'lib/prisma';
export async function createSession(args: Prisma.SessionCreateInput) {
return runQuery({
[PRISMA]: () => relationalQuery(args),
[CLICKHOUSE]: () => clickhouseQuery(args),
}).then(async data => {
if (cache.enabled) {
await cache.storeSession(data);
}
return data;
});
}
async function relationalQuery(data: Prisma.SessionCreateInput) {
export async function createSession(data: Prisma.SessionCreateInput) {
const {
id,
websiteId,
@ -33,71 +18,28 @@ async function relationalQuery(data: Prisma.SessionCreateInput) {
city,
} = data;
return prisma.client.session.create({
data: {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,
subdivision2,
city,
},
});
}
async function clickhouseQuery(data: {
id: string;
websiteId: string;
hostname?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
subdivision1?: string;
subdivision2?: string;
city?: string;
}) {
const {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city,
} = data;
const { getDateFormat, sendMessage } = kafka;
const msg = {
session_id: id,
website_id: websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city,
created_at: getDateFormat(new Date()),
};
await sendMessage(msg, 'event');
return data;
return prisma.client.session
.create({
data: {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,
subdivision2,
city,
},
})
.then(async data => {
if (cache.enabled) {
await cache.storeSession(data);
}
return data;
});
}

39
queries/analytics/session/getSession.ts
View File

@ -1,43 +1,8 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { Prisma } from '@prisma/client';
import prisma from 'lib/prisma';
export async function getSession(args: { id: string }) {
return runQuery({
[PRISMA]: () => relationalQuery(args),
[CLICKHOUSE]: () => clickhouseQuery(args),
});
}
async function relationalQuery(where: Prisma.SessionWhereUniqueInput) {
export async function getSession(where: Prisma.SessionWhereUniqueInput) {
return prisma.client.session.findUnique({
where,
});
}
async function clickhouseQuery({ id: sessionId }: { id: string }) {
const { rawQuery, findFirst } = clickhouse;
const params = { sessionId };
return rawQuery(
`select
session_id,
website_id,
created_at,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city
from website_event
where session_id = {sessionId:UUID}
limit 1`,
params,
).then(result => findFirst(result));
}

43
queries/analytics/session/saveSessionData.ts Normal file
View File

@ -0,0 +1,43 @@
import { DYNAMIC_DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { flattenJSON } from 'lib/dynamicData';
import prisma from 'lib/prisma';
import { DynamicData } from 'lib/types';
export async function saveSessionData(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
const { client, transaction } = prisma;
const { websiteId, sessionId, sessionData } = data;
const jsonKeys = flattenJSON(sessionData);
const flattendData = jsonKeys.map(a => ({
id: uuid(),
websiteId,
sessionId,
key: a.key,
stringValue:
a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||
a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||
a.dynamicDataType === DYNAMIC_DATA_TYPE.array
? a.value
: null,
numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,
dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dynamicDataType,
}));
return transaction([
client.sessionData.deleteMany({
where: {
sessionId,
},
}),
client.sessionData.createMany({
data: flattendData,
}),
]);
}

7
tracker/index.js
View File

@ -173,7 +173,7 @@
}
};
const send = payload => {
const send = (payload, type = 'event') => {
if (trackingDisabled()) return;
const headers = {
'Content-Type': 'application/json',
@ -183,7 +183,7 @@
}
return fetch(endpoint, {
method: 'POST',
body: JSON.stringify({ type: 'event', payload }),
body: JSON.stringify({ type, payload }),
headers,
})
.then(res => res.text())
@ -205,11 +205,14 @@
return send(getPayload());
};
const identify = data => send({ ...getPayload(), data }, 'identify');
/* Start */
if (!window.umami) {
window.umami = {
track,
identify,
};
}