Feat/um 305 unique session ch (#2065)

* Add session_data / session redis to CH.

* Add mysql migration.

components/pages/console/TestConsole.js

@@ -28,22 +28,41 @@ export function TestConsole() {
     window.umami.track({ url: '/page-view', referrer: 'https://www.google.com' });
     window.umami.track('track-event-no-data');
     window.umami.track('track-event-with-data', {
-      data: {
+      test: 'test-data',
+      boolean: true,
+      booleanError: 'true',
+      time: new Date(),
+      number: 1,
+      time2: new Date().toISOString(),
+      nested: {
         test: 'test-data',
-        boolean: true,
-        booleanError: 'true',
-        time: new Date(),
         number: 1,
-        time2: new Date().toISOString(),
+        object: {
-        nested: {
           test: 'test-data',
-          number: 1,
-          object: {
-            test: 'test-data',
-          },
         },
-        array: [1, 2, 3],
       },
+      array: [1, 2, 3],
+    });
+  }
+
+  function handleIdentifyClick() {
+    window.umami.identify({
+      userId: 123,
+      name: 'brian',
+      number: Math.random() * 100,
+      test: 'test-data',
+      boolean: true,
+      booleanError: 'true',
+      time: new Date(),
+      time2: new Date().toISOString(),
+      nested: {
+        test: 'test-data',
+        number: 1,
+        object: {
+          test: 'test-data',
+        },
+      },
+      array: [1, 2, 3],
     });
   }
 
@@ -116,6 +135,10 @@ export function TestConsole() {
               <Button id="manual-button" variant="action" onClick={handleClick}>
                 Run script
               </Button>
+              <p />
+              <Button id="manual-button" variant="action" onClick={handleIdentifyClick}>
+                Run identify
+              </Button>
             </Column>
           </Row>
           <Row>

db/clickhouse/migrations/01_edit_keys.sql

@@ -0,0 +1,4 @@
+ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value";
+ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value";
+ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value";
+ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";

db/clickhouse/schema.sql

@@ -117,10 +117,10 @@ CREATE TABLE umami.event_data
     url_path String,
     event_name String,
     event_key String,
-    event_string_value Nullable(String),
+    string_value Nullable(String),
-    event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
+    numeric_value Nullable(Decimal64(4)), --922337203685477.5625
-    event_date_value Nullable(DateTime('UTC')),
+    date_value Nullable(DateTime('UTC')),
-    event_data_type UInt32,
+    data_type UInt32,
     created_at DateTime('UTC')
 )
     engine = MergeTree
@@ -134,10 +134,10 @@ CREATE TABLE umami.event_data_queue (
     url_path String,
     event_name String,
     event_key String,
-    event_string_value Nullable(String),
+    string_value Nullable(String),
-    event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
+    numeric_value Nullable(Decimal64(4)), --922337203685477.5625
-    event_date_value Nullable(DateTime('UTC')),
+    date_value Nullable(DateTime('UTC')),
-    event_data_type UInt32,
+    data_type UInt32,
     created_at DateTime('UTC'),
     --virtual columns
     _error String,
@@ -158,10 +158,10 @@ SELECT website_id,
     url_path,
     event_name,
     event_key,
-    event_string_value,
+    string_value,
-    event_numeric_value,
+    numeric_value,
-    event_date_value,
+    date_value,
-    event_data_type,
+    data_type,
     created_at
 FROM umami.event_data_queue;
 

db/mysql/migrations/03_session_data/migration.sql

@@ -0,0 +1,37 @@
+/*
+  Warnings:
+
+  - The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint.
+  - You are about to drop the column `event_data_type` on the `event_data` table. All the data in the column will be lost.
+  - You are about to drop the column `event_date_value` on the `event_data` table. All the data in the column will be lost.
+  - You are about to drop the column `event_id` on the `event_data` table. All the data in the column will be lost.
+  - You are about to drop the column `event_numeric_value` on the `event_data` table. All the data in the column will be lost.
+  - You are about to drop the column `event_string_value` on the `event_data` table. All the data in the column will be lost.
+  - Added the required column `data_type` to the `event_data` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `event_data_id` to the `event_data` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE `event_data` RENAME COLUMN `event_data_type` TO `data_type`;
+ALTER TABLE `event_data` RENAME COLUMN `event_date_value` TO `date_value`;
+ALTER TABLE `event_data` RENAME COLUMN `event_id` TO `event_data_id`;
+ALTER TABLE `event_data` RENAME COLUMN `event_numeric_value` TO `numeric_value`;
+ALTER TABLE `event_data` RENAME COLUMN `event_string_value` TO `string_value`;
+
+-- CreateTable
+CREATE TABLE `session_data` (
+    `session_data_id` VARCHAR(36) NOT NULL,
+    `website_id` VARCHAR(36) NOT NULL,
+    `session_id` VARCHAR(36) NOT NULL,
+    `event_key` VARCHAR(500) NOT NULL,
+    `event_string_value` VARCHAR(500) NULL,
+    `event_numeric_value` DECIMAL(19, 4) NULL,
+    `event_date_value` TIMESTAMP(0) NULL,
+    `event_data_type` INTEGER UNSIGNED NOT NULL,
+    `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
+
+    INDEX `session_data_created_at_idx`(`created_at`),
+    INDEX `session_data_website_id_idx`(`website_id`),
+    INDEX `session_data_session_id_idx`(`session_id`),
+    PRIMARY KEY (`session_data_id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

db/mysql/schema.prisma

@@ -14,12 +14,12 @@ model User {
   password  String    @db.VarChar(60)
   role      String    @map("role") @db.VarChar(50)
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
-  updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
+  updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
   deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
 
   website  Website[]
   teamUser TeamUser[]
-  Report      Report[]
+  report   Report[]
 
   @@map("user")
 }
@@ -40,6 +40,7 @@ model Session {
   createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
 
   websiteEvent WebsiteEvent[]
+  sessionData  SessionData[]
 
   @@index([createdAt])
   @@index([websiteId])
@@ -54,13 +55,14 @@ model Website {
   resetAt   DateTime? @map("reset_at") @db.Timestamp(0)
   userId    String?   @map("user_id") @db.VarChar(36)
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
-  updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
+  updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
   deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
 
   user        User?         @relation(fields: [userId], references: [id])
   teamWebsite TeamWebsite[]
   eventData   EventData[]
-  Report      Report[]
+  report      Report[]
+  sessionData SessionData[]
 
   @@index([userId])
   @@index([createdAt])
@@ -94,15 +96,15 @@ model WebsiteEvent {
 }
 
 model EventData {
-  id                String    @id() @map("event_id") @db.VarChar(36)
+  id             String    @id() @map("event_data_id") @db.VarChar(36)
-  websiteEventId    String    @map("website_event_id") @db.VarChar(36)
+  websiteEventId String    @map("website_event_id") @db.VarChar(36)
-  websiteId         String    @map("website_id") @db.VarChar(36)
+  websiteId      String    @map("website_id") @db.VarChar(36)
-  eventKey          String    @map("event_key") @db.VarChar(500)
+  eventKey       String    @map("event_key") @db.VarChar(500)
-  eventStringValue  String?   @map("event_string_value") @db.VarChar(500)
+  stringValue    String?   @map("string_value") @db.VarChar(500)
-  eventNumericValue Decimal?  @map("event_numeric_value") @db.Decimal(19, 4)
+  numericValue   Decimal?  @map("numeric_value") @db.Decimal(19, 4)
-  eventDateValue    DateTime? @map("event_date_value") @db.Timestamp(0)
+  dateValue      DateTime? @map("date_value") @db.Timestamp(0)
-  eventDataType     Int       @map("event_data_type") @db.UnsignedInt
+  dataType       Int       @map("data_type") @db.UnsignedInt
-  createdAt         DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
+  createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
 
   website      Website      @relation(fields: [websiteId], references: [id])
   websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
@@ -114,12 +116,32 @@ model EventData {
   @@map("event_data")
 }
 
+model SessionData {
+  id                String    @id() @map("session_data_id") @db.VarChar(36)
+  websiteId         String    @map("website_id") @db.VarChar(36)
+  sessionId         String    @map("session_id") @db.VarChar(36)
+  eventKey          String    @map("event_key") @db.VarChar(500)
+  eventStringValue  String?   @map("event_string_value") @db.VarChar(500)
+  eventNumericValue Decimal?  @map("event_numeric_value") @db.Decimal(19, 4)
+  eventDateValue    DateTime? @map("event_date_value") @db.Timestamp(0)
+  eventDataType     Int       @map("event_data_type") @db.UnsignedInt
+  createdAt         DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
+
+  website Website @relation(fields: [websiteId], references: [id])
+  session Session @relation(fields: [sessionId], references: [id])
+
+  @@index([createdAt])
+  @@index([websiteId])
+  @@index([sessionId])
+  @@map("session_data")
+}
+
 model Team {
   id         String    @id() @unique() @map("team_id") @db.VarChar(36)
   name       String    @db.VarChar(50)
   accessCode String?   @unique @map("access_code") @db.VarChar(50)
   createdAt  DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
-  updatedAt  DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
+  updatedAt  DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
 
   teamUser    TeamUser[]
   teamWebsite TeamWebsite[]
@@ -134,7 +156,7 @@ model TeamUser {
   userId    String    @map("user_id") @db.VarChar(36)
   role      String    @map("role") @db.VarChar(50)
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
-  updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)
+  updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
 
   team Team @relation(fields: [teamId], references: [id])
   user User @relation(fields: [userId], references: [id])
@@ -177,4 +199,4 @@ model Report {
   @@index([type])
   @@index([name])
   @@map("report")
-}
+}

db/postgresql/migrations/03_session_data/migration.sql

@@ -0,0 +1,31 @@
+-- AlterTable
+ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";
+ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value";
+ALTER TABLE "event_data" RENAME COLUMN "event_id" TO "event_data_id";
+ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value";
+ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value";
+
+-- CreateTable
+CREATE TABLE "session_data" (
+    "session_data_id" UUID NOT NULL,
+    "website_id" UUID NOT NULL,
+    "session_id" UUID NOT NULL,
+    "session_key" VARCHAR(500) NOT NULL,
+    "string_value" VARCHAR(500),
+    "numeric_value" DECIMAL(19,4),
+    "date_value" TIMESTAMPTZ(6),
+    "data_type" INTEGER NOT NULL,
+    "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
+    "deleted_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "session_data_pkey" PRIMARY KEY ("session_data_id")
+);
+
+-- CreateIndex
+CREATE INDEX "session_data_created_at_idx" ON "session_data"("created_at");
+
+-- CreateIndex
+CREATE INDEX "session_data_website_id_idx" ON "session_data"("website_id");
+
+-- CreateIndex
+CREATE INDEX "session_data_session_id_idx" ON "session_data"("session_id");

db/postgresql/schema.prisma

@@ -19,7 +19,7 @@ model User {
 
   website  Website[]
   teamUser TeamUser[]
-  Report   Report[]
+  report   Report[]
 
   @@map("user")
 }
@@ -40,6 +40,7 @@ model Session {
   createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
 
   websiteEvent WebsiteEvent[]
+  sessionData  SessionData[]
 
   @@index([createdAt])
   @@index([websiteId])
@@ -60,7 +61,8 @@ model Website {
   user        User?         @relation(fields: [userId], references: [id])
   teamWebsite TeamWebsite[]
   eventData   EventData[]
-  Report      Report[]
+  report      Report[]
+  sessionData SessionData[]
 
   @@index([userId])
   @@index([createdAt])
@@ -94,15 +96,15 @@ model WebsiteEvent {
 }
 
 model EventData {
-  id                String    @id() @map("event_id") @db.Uuid
+  id             String    @id() @map("event_data_id") @db.Uuid
-  websiteId         String    @map("website_id") @db.Uuid
+  websiteId      String    @map("website_id") @db.Uuid
-  websiteEventId    String    @map("website_event_id") @db.Uuid
+  websiteEventId String    @map("website_event_id") @db.Uuid
-  eventKey          String    @map("event_key") @db.VarChar(500)
+  eventKey       String    @map("event_key") @db.VarChar(500)
-  eventStringValue  String?   @map("event_string_value") @db.VarChar(500)
+  stringValue    String?   @map("string_value") @db.VarChar(500)
-  eventNumericValue Decimal?  @map("event_numeric_value") @db.Decimal(19, 4)
+  numericValue   Decimal?  @map("numeric_value") @db.Decimal(19, 4)
-  eventDateValue    DateTime? @map("event_date_value") @db.Timestamptz(6)
+  dateValue      DateTime? @map("date_value") @db.Timestamptz(6)
-  eventDataType     Int       @map("event_data_type") @db.Integer
+  dataType       Int       @map("data_type") @db.Integer
-  createdAt         DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
+  createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
 
   website      Website      @relation(fields: [websiteId], references: [id])
   websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
@@ -113,6 +115,27 @@ model EventData {
   @@map("event_data")
 }
 
+model SessionData {
+  id           String    @id() @map("session_data_id") @db.Uuid
+  websiteId    String    @map("website_id") @db.Uuid
+  sessionId    String    @map("session_id") @db.Uuid
+  sessionKey   String    @map("session_key") @db.VarChar(500)
+  stringValue  String?   @map("string_value") @db.VarChar(500)
+  numericValue Decimal?  @map("numeric_value") @db.Decimal(19, 4)
+  dateValue    DateTime? @map("date_value") @db.Timestamptz(6)
+  dataType     Int       @map("data_type") @db.Integer
+  createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
+  deletedAt    DateTime? @default(now()) @map("deleted_at") @db.Timestamptz(6)
+
+  website Website @relation(fields: [websiteId], references: [id])
+  session Session @relation(fields: [sessionId], references: [id])
+
+  @@index([createdAt])
+  @@index([websiteId])
+  @@index([sessionId])
+  @@map("session_data")
+}
+
 model Team {
   id         String    @id() @unique() @map("team_id") @db.Uuid
   name       String    @db.VarChar(50)

lib/clickhouse.ts

@@ -2,7 +2,7 @@ import { ClickHouse } from 'clickhouse';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
-import { getEventDataType } from './eventData';
+import { getDynamicDataType } from './dynamicData';
 import { WebsiteMetricFilter } from './types';
 import { FILTER_COLUMNS } from './constants';
 
@@ -74,7 +74,7 @@ function getEventDataFilterQuery(
   params: any,
 ) {
   const query = filters.reduce((ac, cv, i) => {
-    const type = getEventDataType(cv.eventValue);
+    const type = getDynamicDataType(cv.eventValue);
 
     let value = cv.eventValue;
 

lib/constants.ts

@@ -42,6 +42,11 @@ export const SESSION_COLUMNS = [
   'city',
 ];
 
+export const COLLECTION_TYPE = {
+  event: 'event',
+  identify: 'identify',
+};
+
 export const FILTER_COLUMNS = {
   url: 'url_path',
   referrer: 'referrer_domain',
@@ -56,7 +61,7 @@ export const EVENT_TYPE = {
   customEvent: 2,
 } as const;
 
-export const EVENT_DATA_TYPE = {
+export const DYNAMIC_DATA_TYPE = {
   string: 1,
   number: 2,
   boolean: 3,

lib/dynamicData.ts

@@ -1,12 +1,12 @@
 import { isValid, parseISO } from 'date-fns';
-import { EVENT_DATA_TYPE } from './constants';
+import { DYNAMIC_DATA_TYPE } from './constants';
-import { EventDataTypes } from './types';
+import { DynamicDataType } from './types';
 
 export function flattenJSON(
   eventData: { [key: string]: any },
-  keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [],
+  keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],
   parentKey = '',
-): { key: string; value: any; eventDataType: EventDataTypes }[] {
+): { key: string; value: any; dynamicDataType: DynamicDataType }[] {
   return Object.keys(eventData).reduce(
     (acc, key) => {
       const value = eventData[key];
@@ -25,7 +25,7 @@ export function flattenJSON(
   ).keyValues;
 }
 
-export function getEventDataType(value: any): string {
+export function getDynamicDataType(value: any): string {
   let type: string = typeof value;
 
   if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
@@ -36,34 +36,34 @@ export function getEventDataType(value: any): string {
 }
 
 function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
-  const type = getEventDataType(value);
+  const type = getDynamicDataType(value);
 
-  let eventDataType = null;
+  let dynamicDataType = null;
 
   switch (type) {
     case 'number':
-      eventDataType = EVENT_DATA_TYPE.number;
+      dynamicDataType = DYNAMIC_DATA_TYPE.number;
       break;
     case 'string':
-      eventDataType = EVENT_DATA_TYPE.string;
+      dynamicDataType = DYNAMIC_DATA_TYPE.string;
       break;
     case 'boolean':
-      eventDataType = EVENT_DATA_TYPE.boolean;
+      dynamicDataType = DYNAMIC_DATA_TYPE.boolean;
       value = value ? 'true' : 'false';
       break;
     case 'date':
-      eventDataType = EVENT_DATA_TYPE.date;
+      dynamicDataType = DYNAMIC_DATA_TYPE.date;
       break;
     case 'object':
-      eventDataType = EVENT_DATA_TYPE.array;
+      dynamicDataType = DYNAMIC_DATA_TYPE.array;
       value = JSON.stringify(value);
       break;
     default:
-      eventDataType = EVENT_DATA_TYPE.string;
+      dynamicDataType = DYNAMIC_DATA_TYPE.string;
       break;
   }
 
-  acc.keyValues.push({ key, value, eventDataType });
+  acc.keyValues.push({ key, value, dynamicDataType });
 }
 
 function getKeyName(key, parentKey) {

lib/prisma.ts

@@ -1,7 +1,7 @@
 import prisma from '@umami/prisma-client';
 import moment from 'moment-timezone';
 import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
-import { getEventDataType } from './eventData';
+import { getDynamicDataType } from './dynamicData';
 import { FILTER_COLUMNS } from './constants';
 
 const MYSQL_DATE_FORMATS = {
@@ -85,7 +85,7 @@ function getEventDataFilterQuery(
   params: any[],
 ) {
   const query = filters.reduce((ac, cv) => {
-    const type = getEventDataType(cv.eventValue);
+    const type = getDynamicDataType(cv.eventValue);
 
     let value = cv.eventValue;
 

lib/session.ts

@@ -1,12 +1,11 @@
-import clickhouse from 'lib/clickhouse';
 import { secret, uuid } from 'lib/crypto';
 import { getClientInfo, getJsonBody } from 'lib/detect';
 import { parseToken } from 'next-basics';
 import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
 import { createSession } from 'queries';
 import { validate } from 'uuid';
-import { loadSession, loadWebsite } from './query';
 import cache from './cache';
+import { loadSession, loadWebsite } from './query';
 
 export async function findSession(req: NextApiRequestCollect) {
   const { payload } = getJsonBody<CollectRequestBody>(req);
@@ -46,26 +45,8 @@ export async function findSession(req: NextApiRequestCollect) {
 
   const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
     await getClientInfo(req, payload);
-  const sessionId = uuid(websiteId, hostname, ip, userAgent);
 
-  // Clickhouse does not require session lookup
+  const sessionId = uuid(websiteId, hostname, ip, userAgent);
-  if (clickhouse.enabled) {
-    return {
-      id: sessionId,
-      websiteId,
-      hostname,
-      browser,
-      os,
-      device,
-      screen,
-      language,
-      country,
-      subdivision1,
-      subdivision2,
-      city,
-      ownerId: website.userId,
-    };
-  }
 
   // Find session
   let session = await loadSession(sessionId);

lib/types.ts

@@ -1,18 +1,20 @@
 import { NextApiRequest } from 'next';
-import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
+import { COLLECTION_TYPE, DYNAMIC_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
 
 type ObjectValues<T> = T[keyof T];
 
-export type Roles = ObjectValues<typeof ROLES>;
+export type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;
 
-export type EventTypes = ObjectValues<typeof EVENT_TYPE>;
+export type Role = ObjectValues<typeof ROLES>;
 
-export type EventDataTypes = ObjectValues<typeof EVENT_DATA_TYPE>;
+export type EventType = ObjectValues<typeof EVENT_TYPE>;
 
-export type KafkaTopics = ObjectValues<typeof KAFKA_TOPIC>;
+export type DynamicDataType = ObjectValues<typeof DYNAMIC_DATA_TYPE>;
 
-export interface EventData {
+export type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;
-  [key: string]: number | string | EventData | number[] | string[] | EventData[];
+
+export interface DynamicData {
+  [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];
 }
 
 export interface Auth {

pages/api/send.ts

@@ -7,6 +7,9 @@ import { getJsonBody, getIpAddress } from 'lib/detect';
 import { secret } from 'lib/crypto';
 import { NextApiRequest, NextApiResponse } from 'next';
 import { Resolver } from 'dns/promises';
+import { CollectionType } from 'lib/types';
+import { COLLECTION_TYPE } from 'lib/constants';
+import { saveSessionData } from 'queries/analytics/session/saveSessionData';
 
 export interface CollectRequestBody {
   payload: {
@@ -20,7 +23,7 @@ export interface CollectRequestBody {
     website: string;
     name: string;
   };
-  type: string;
+  type: CollectionType;
 }
 
 export interface NextApiRequestCollect extends NextApiRequest {
@@ -52,17 +55,81 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
 
   const { type, payload } = getJsonBody<CollectRequestBody>(req);
 
-  if (type !== 'event') {
+  validateBody(res, { type, payload });
+
+  if (await hasBlockedIp(req)) {
+    return forbidden(res);
+  }
+
+  const { url, referrer, name: eventName, data: dynamicData, title: pageTitle } = payload;
+
+  await useSession(req, res);
+
+  const session = req.session;
+
+  if (type === COLLECTION_TYPE.event) {
+    // eslint-disable-next-line prefer-const
+    let [urlPath, urlQuery] = url?.split('?') || [];
+    let [referrerPath, referrerQuery] = referrer?.split('?') || [];
+    let referrerDomain;
+
+    if (!urlPath) {
+      urlPath = '/';
+    }
+
+    if (referrerPath?.startsWith('http')) {
+      const refUrl = new URL(referrer);
+      referrerPath = refUrl.pathname;
+      referrerQuery = refUrl.search.substring(1);
+      referrerDomain = refUrl.hostname.replace(/www\./, '');
+    }
+
+    if (process.env.REMOVE_TRAILING_SLASH) {
+      urlPath = urlPath.replace(/.+\/$/, '');
+    }
+
+    await saveEvent({
+      urlPath,
+      urlQuery,
+      referrerPath,
+      referrerQuery,
+      referrerDomain,
+      pageTitle,
+      eventName,
+      eventData: dynamicData,
+      ...session,
+      sessionId: session.id,
+    });
+  }
+
+  if (type === COLLECTION_TYPE.identify) {
+    if (!dynamicData) {
+      return badRequest(res, 'Data required.');
+    }
+
+    await saveSessionData({ ...session, sessionData: dynamicData, sessionId: session.id });
+  }
+
+  const token = createToken(session, secret());
+
+  return send(res, token);
+};
+
+function validateBody(res: NextApiResponse, { type, payload }: CollectRequestBody) {
+  const { data } = payload;
+
+  // Validate type
+  if (type !== COLLECTION_TYPE.event && type !== COLLECTION_TYPE.identify) {
     return badRequest(res, 'Wrong payload type.');
   }
 
-  const { url, referrer, name: eventName, data: eventData, title: pageTitle } = payload;
-
   // Validate eventData is JSON
-  if (eventData && !(typeof eventData === 'object' && !Array.isArray(eventData))) {
+  if (data && !(typeof data === 'object' && !Array.isArray(data))) {
     return badRequest(res, 'Invalid event data.');
   }
+}
 
+async function hasBlockedIp(req: NextApiRequestCollect) {
   const ignoreIps = process.env.IGNORE_IP;
   const ignoreHostnames = process.env.IGNORE_HOSTNAME;
 
@@ -100,49 +167,6 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
       return false;
     });
 
-    if (blocked) {
+    return blocked;
-      return forbidden(res);
-    }
   }
-
+}
-  await useSession(req, res);
-
-  const session = req.session;
-
-  // eslint-disable-next-line prefer-const
-  let [urlPath, urlQuery] = url?.split('?') || [];
-  let [referrerPath, referrerQuery] = referrer?.split('?') || [];
-  let referrerDomain;
-
-  if (!urlPath) {
-    urlPath = '/';
-  }
-
-  if (referrerPath?.startsWith('http')) {
-    const refUrl = new URL(referrer);
-    referrerPath = refUrl.pathname;
-    referrerQuery = refUrl.search.substring(1);
-    referrerDomain = refUrl.hostname.replace(/www\./, '');
-  }
-
-  if (process.env.REMOVE_TRAILING_SLASH) {
-    urlPath = urlPath.replace(/.+\/$/, '');
-  }
-
-  await saveEvent({
-    urlPath,
-    urlQuery,
-    referrerPath,
-    referrerQuery,
-    referrerDomain,
-    pageTitle,
-    eventName,
-    eventData,
-    ...session,
-    sessionId: session.id,
-  });
-
-  const token = createToken(session, secret());
-
-  return send(res, token);
-};

pages/api/users/[id]/index.ts

@@ -1,4 +1,4 @@
-import { NextApiRequestQueryBody, Roles, User } from 'lib/types';
+import { NextApiRequestQueryBody, Role, User } from 'lib/types';
 import { canDeleteUser, canUpdateUser, canViewUser } from 'lib/auth';
 import { useAuth } from 'lib/middleware';
 import { NextApiResponse } from 'next';
@@ -12,7 +12,7 @@ export interface UserRequestQuery {
 export interface UserRequestBody {
   username: string;
   password: string;
-  role: Roles;
+  role: Role;
 }
 
 export default async (

pages/api/users/index.ts

@@ -2,7 +2,7 @@ import { canCreateUser, canViewUsers } from 'lib/auth';
 import { ROLES } from 'lib/constants';
 import { uuid } from 'lib/crypto';
 import { useAuth } from 'lib/middleware';
-import { NextApiRequestQueryBody, Roles, User } from 'lib/types';
+import { NextApiRequestQueryBody, Role, User } from 'lib/types';
 import { NextApiResponse } from 'next';
 import { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics';
 import { createUser, getUser, getUsers } from 'queries';
@@ -11,7 +11,7 @@ export interface UsersRequestBody {
   username: string;
   password: string;
   id: string;
-  role?: Roles;
+  role?: Role;
 }
 
 export default async (

queries/admin/user.ts

@@ -3,7 +3,7 @@ import { getRandomChars } from 'next-basics';
 import cache from 'lib/cache';
 import { ROLES } from 'lib/constants';
 import prisma from 'lib/prisma';
-import { Website, User, Roles } from 'lib/types';
+import { Website, User, Role } from 'lib/types';
 
 export async function getUser(
   where: Prisma.UserWhereInput | Prisma.UserWhereUniqueInput,
@@ -91,7 +91,7 @@ export async function createUser(data: {
   id: string;
   username: string;
   password: string;
-  role: Roles;
+  role: Role;
 }): Promise<{
   id: string;
   username: string;

queries/analytics/event/saveEvent.ts

@@ -133,9 +133,10 @@ async function clickhouseQuery(data: {
   const createdAt = getDateFormat(new Date());
 
   const message = {
+    ...args,
     website_id: websiteId,
     session_id: sessionId,
-    event_id: eventId,
+    event_id: uuid(),
     country: country ? country : null,
     subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,
     subdivision2: subdivision2 ? subdivision2 : null,
@@ -149,7 +150,6 @@ async function clickhouseQuery(data: {
     event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
     event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
     created_at: createdAt,
-    ...args,
   };
 
   await sendMessage(message, 'event');

queries/analytics/eventData/saveEventData.ts

@@ -1,11 +1,11 @@
 import { Prisma } from '@prisma/client';
-import { EVENT_DATA_TYPE } from 'lib/constants';
+import { DYNAMIC_DATA_TYPE } from 'lib/constants';
 import { uuid } from 'lib/crypto';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
-import { flattenJSON } from 'lib/eventData';
+import { flattenJSON } from 'lib/dynamicData';
 import kafka from 'lib/kafka';
 import prisma from 'lib/prisma';
-import { EventData } from 'lib/types';
+import { DynamicData } from 'lib/types';
 
 export async function saveEventData(args: {
   websiteId: string;
@@ -13,7 +13,7 @@ export async function saveEventData(args: {
   sessionId?: string;
   urlPath?: string;
   eventName?: string;
-  eventData: EventData;
+  eventData: DynamicData;
   createdAt?: string;
 }) {
   return runQuery({
@@ -25,7 +25,7 @@ export async function saveEventData(args: {
 async function relationalQuery(data: {
   websiteId: string;
   eventId: string;
-  eventData: EventData;
+  eventData: DynamicData;
 }): Promise<Prisma.BatchPayload> {
   const { websiteId, eventId, eventData } = data;
 
@@ -36,16 +36,16 @@ async function relationalQuery(data: {
     id: uuid(),
     websiteEventId: eventId,
     websiteId,
-    eventKey: a.key,
+    key: a.key,
-    eventStringValue:
+    stringValue:
-      a.eventDataType === EVENT_DATA_TYPE.string ||
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||
-      a.eventDataType === EVENT_DATA_TYPE.boolean ||
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||
-      a.eventDataType === EVENT_DATA_TYPE.array
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.array
         ? a.value
         : null,
-    eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
+    numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,
-    eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null,
+    dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null,
-    eventDataType: a.eventDataType,
+    dataType: a.dynamicDataType,
   }));
 
   return prisma.client.eventData.createMany({
@@ -59,7 +59,7 @@ async function clickhouseQuery(data: {
   sessionId?: string;
   urlPath?: string;
   eventName?: string;
-  eventData: EventData;
+  eventData: DynamicData;
   createdAt?: string;
 }) {
   const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
@@ -75,15 +75,15 @@ async function clickhouseQuery(data: {
     url_path: urlPath,
     event_name: eventName,
     event_key: a.key,
-    event_string_value:
+    string_value:
-      a.eventDataType === EVENT_DATA_TYPE.string ||
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||
-      a.eventDataType === EVENT_DATA_TYPE.boolean ||
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||
-      a.eventDataType === EVENT_DATA_TYPE.array
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.array
         ? a.value
         : null,
-    event_numeric_value: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
+    numeric_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,
-    event_date_value: a.eventDataType === EVENT_DATA_TYPE.date ? getDateFormat(a.value) : null,
+    date_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? getDateFormat(a.value) : null,
-    event_data_type: a.eventDataType,
+    data_type: a.dynamicDataType,
     created_at: createdAt,
   }));
 

queries/analytics/session/createSession.ts

@@ -1,23 +1,8 @@
-import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
-import kafka from 'lib/kafka';
-import prisma from 'lib/prisma';
-import cache from 'lib/cache';
 import { Prisma } from '@prisma/client';
+import cache from 'lib/cache';
+import prisma from 'lib/prisma';
 
-export async function createSession(args: Prisma.SessionCreateInput) {
+export async function createSession(data: Prisma.SessionCreateInput) {
-  return runQuery({
-    [PRISMA]: () => relationalQuery(args),
-    [CLICKHOUSE]: () => clickhouseQuery(args),
-  }).then(async data => {
-    if (cache.enabled) {
-      await cache.storeSession(data);
-    }
-
-    return data;
-  });
-}
-
-async function relationalQuery(data: Prisma.SessionCreateInput) {
   const {
     id,
     websiteId,
@@ -33,71 +18,28 @@ async function relationalQuery(data: Prisma.SessionCreateInput) {
     city,
   } = data;
 
-  return prisma.client.session.create({
+  return prisma.client.session
-    data: {
+    .create({
-      id,
+      data: {
-      websiteId,
+        id,
-      hostname,
+        websiteId,
-      browser,
+        hostname,
-      os,
+        browser,
-      device,
+        os,
-      screen,
+        device,
-      language,
+        screen,
-      country,
+        language,
-      subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,
+        country,
-      subdivision2,
+        subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,
-      city,
+        subdivision2,
-    },
+        city,
-  });
+      },
-}
+    })
-
+    .then(async data => {
-async function clickhouseQuery(data: {
+      if (cache.enabled) {
-  id: string;
+        await cache.storeSession(data);
-  websiteId: string;
+      }
-  hostname?: string;
+
-  browser?: string;
+      return data;
-  os?: string;
+    });
-  device?: string;
-  screen?: string;
-  language?: string;
-  country?: string;
-  subdivision1?: string;
-  subdivision2?: string;
-  city?: string;
-}) {
-  const {
-    id,
-    websiteId,
-    hostname,
-    browser,
-    os,
-    device,
-    screen,
-    language,
-    country,
-    subdivision1,
-    subdivision2,
-    city,
-  } = data;
-  const { getDateFormat, sendMessage } = kafka;
-
-  const msg = {
-    session_id: id,
-    website_id: websiteId,
-    hostname,
-    browser,
-    os,
-    device,
-    screen,
-    language,
-    country,
-    subdivision1,
-    subdivision2,
-    city,
-    created_at: getDateFormat(new Date()),
-  };
-
-  await sendMessage(msg, 'event');
-
-  return data;
 }

queries/analytics/session/getSession.ts

@@ -1,43 +1,8 @@
-import clickhouse from 'lib/clickhouse';
-import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
-import prisma from 'lib/prisma';
 import { Prisma } from '@prisma/client';
+import prisma from 'lib/prisma';
 
-export async function getSession(args: { id: string }) {
+export async function getSession(where: Prisma.SessionWhereUniqueInput) {
-  return runQuery({
-    [PRISMA]: () => relationalQuery(args),
-    [CLICKHOUSE]: () => clickhouseQuery(args),
-  });
-}
-
-async function relationalQuery(where: Prisma.SessionWhereUniqueInput) {
   return prisma.client.session.findUnique({
     where,
   });
 }
-
-async function clickhouseQuery({ id: sessionId }: { id: string }) {
-  const { rawQuery, findFirst } = clickhouse;
-  const params = { sessionId };
-
-  return rawQuery(
-    `select
-      session_id, 
-      website_id, 
-      created_at, 
-      hostname, 
-      browser, 
-      os, 
-      device, 
-      screen,
-      language, 
-      country,
-      subdivision1,
-      subdivision2,
-      city
-    from website_event
-      where session_id = {sessionId:UUID}
-    limit 1`,
-    params,
-  ).then(result => findFirst(result));
-}

queries/analytics/session/saveSessionData.ts

@@ -0,0 +1,43 @@
+import { DYNAMIC_DATA_TYPE } from 'lib/constants';
+import { uuid } from 'lib/crypto';
+import { flattenJSON } from 'lib/dynamicData';
+import prisma from 'lib/prisma';
+import { DynamicData } from 'lib/types';
+
+export async function saveSessionData(data: {
+  websiteId: string;
+  sessionId: string;
+  sessionData: DynamicData;
+}) {
+  const { client, transaction } = prisma;
+  const { websiteId, sessionId, sessionData } = data;
+
+  const jsonKeys = flattenJSON(sessionData);
+
+  const flattendData = jsonKeys.map(a => ({
+    id: uuid(),
+    websiteId,
+    sessionId,
+    key: a.key,
+    stringValue:
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||
+      a.dynamicDataType === DYNAMIC_DATA_TYPE.array
+        ? a.value
+        : null,
+    numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,
+    dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null,
+    dataType: a.dynamicDataType,
+  }));
+
+  return transaction([
+    client.sessionData.deleteMany({
+      where: {
+        sessionId,
+      },
+    }),
+    client.sessionData.createMany({
+      data: flattendData,
+    }),
+  ]);
+}

tracker/index.js

@@ -173,7 +173,7 @@
     }
   };
 
-  const send = payload => {
+  const send = (payload, type = 'event') => {
     if (trackingDisabled()) return;
     const headers = {
       'Content-Type': 'application/json',
@@ -183,7 +183,7 @@
     }
     return fetch(endpoint, {
       method: 'POST',
-      body: JSON.stringify({ type: 'event', payload }),
+      body: JSON.stringify({ type, payload }),
       headers,
     })
       .then(res => res.text())
@@ -205,11 +205,14 @@
     return send(getPayload());
   };
 
+  const identify = data => send({ ...getPayload(), data }, 'identify');
+
   /* Start */
 
   if (!window.umami) {
     window.umami = {
       track,
+      identify,
     };
   }