diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index c2fa7135..53fba1fb 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -27,7 +27,7 @@ CREATE TABLE umami.website_event event_type UInt32, event_name String, created_at DateTime('UTC'), - job_id UUID + job_id Nullable(UUID) ) engine = MergeTree ORDER BY (website_id, session_id, created_at) diff --git a/db/mysql/migrations/06_session_data/migration.sql b/db/mysql/migrations/06_session_data/migration.sql new file mode 100644 index 00000000..97ed60ad --- /dev/null +++ b/db/mysql/migrations/06_session_data/migration.sql @@ -0,0 +1,20 @@ +-- DropIndex +DROP INDEX `event_data_website_id_created_at_event_key_idx` ON `event_data`; + +-- DropIndex +DROP INDEX `event_data_website_id_website_event_id_created_at_idx` ON `event_data`; + +-- AlterTable +ALTER TABLE `event_data` RENAME COLUMN `event_key` TO `data_key`; + +-- AlterTable +ALTER TABLE `session_data` RENAME COLUMN `event_key` TO `data_key`; + +-- CreateIndex +CREATE INDEX `event_data_website_id_created_at_data_key_idx` ON `event_data`(`website_id`, `created_at`, `data_key`); + +-- CreateIndex +CREATE INDEX `session_data_session_id_created_at_idx` ON `session_data`(`session_id`, `created_at`); + +-- CreateIndex +CREATE INDEX `session_data_website_id_created_at_data_key_idx` ON `session_data`(`website_id`, `created_at`, `data_key`); diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 152ca265..a7722acc 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -125,7 +125,7 @@ model EventData { id String @id() @map("event_data_id") @db.VarChar(36) websiteId String @map("website_id") @db.VarChar(36) websiteEventId String @map("website_event_id") @db.VarChar(36) - eventKey String @map("event_key") @db.VarChar(500) + dataKey String @map("data_key") @db.VarChar(500) stringValue String? @map("string_value") @db.VarChar(500) numberValue Decimal? @map("number_value") @db.Decimal(19, 4) dateValue DateTime? @map("date_value") @db.Timestamp(0) @@ -138,9 +138,8 @@ model EventData { @@index([createdAt]) @@index([websiteId]) @@index([websiteEventId]) - @@index([websiteId, websiteEventId, createdAt]) @@index([websiteId, createdAt]) - @@index([websiteId, createdAt, eventKey]) + @@index([websiteId, createdAt, dataKey]) @@map("event_data") } @@ -148,7 +147,7 @@ model SessionData { id String @id() @map("session_data_id") @db.VarChar(36) websiteId String @map("website_id") @db.VarChar(36) sessionId String @map("session_id") @db.VarChar(36) - eventKey String @map("event_key") @db.VarChar(500) + dataKey String @map("data_key") @db.VarChar(500) stringValue String? @map("string_value") @db.VarChar(500) numberValue Decimal? @map("number_value") @db.Decimal(19, 4) dateValue DateTime? @map("date_value") @db.Timestamp(0) @@ -161,6 +160,8 @@ model SessionData { @@index([createdAt]) @@index([websiteId]) @@index([sessionId]) + @@index([sessionId, createdAt]) + @@index([websiteId, createdAt, dataKey]) @@map("session_data") } diff --git a/db/postgresql/migrations/06_session_data/migration.sql b/db/postgresql/migrations/06_session_data/migration.sql new file mode 100644 index 00000000..9e4d9716 --- /dev/null +++ b/db/postgresql/migrations/06_session_data/migration.sql @@ -0,0 +1,18 @@ +-- DropIndex +DROP INDEX IF EXISTS "event_data_website_id_created_at_event_key_idx"; + +-- AlterTable +ALTER TABLE "event_data" RENAME COLUMN "event_key" TO "data_key"; + +-- AlterTable +ALTER TABLE "session_data" DROP COLUMN "deleted_at", +ALTER TABLE "session_data" RENAME COLUMN "session_key" TO "data_key"; + +-- CreateIndex +CREATE INDEX "event_data_website_id_created_at_data_key_idx" ON "event_data"("website_id", "created_at", "data_key"); + +-- CreateIndex +CREATE INDEX "session_data_session_id_created_at_idx" ON "session_data"("session_id", "created_at"); + +-- CreateIndex +CREATE INDEX "session_data_website_id_created_at_data_key_idx" ON "session_data"("website_id", "created_at", "data_key"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 0cb8ae8a..b9009e0f 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -125,7 +125,7 @@ model EventData { id String @id() @map("event_data_id") @db.Uuid websiteId String @map("website_id") @db.Uuid websiteEventId String @map("website_event_id") @db.Uuid - eventKey String @map("event_key") @db.VarChar(500) + dataKey String @map("data_key") @db.VarChar(500) stringValue String? @map("string_value") @db.VarChar(500) numberValue Decimal? @map("number_value") @db.Decimal(19, 4) dateValue DateTime? @map("date_value") @db.Timestamptz(6) @@ -139,7 +139,7 @@ model EventData { @@index([websiteId]) @@index([websiteEventId]) @@index([websiteId, createdAt]) - @@index([websiteId, createdAt, eventKey]) + @@index([websiteId, createdAt, dataKey]) @@map("event_data") } @@ -147,13 +147,12 @@ model SessionData { id String @id() @map("session_data_id") @db.Uuid websiteId String @map("website_id") @db.Uuid sessionId String @map("session_id") @db.Uuid - sessionKey String @map("session_key") @db.VarChar(500) + dataKey String @map("data_key") @db.VarChar(500) stringValue String? @map("string_value") @db.VarChar(500) numberValue Decimal? @map("number_value") @db.Decimal(19, 4) dateValue DateTime? @map("date_value") @db.Timestamptz(6) dataType Int @map("data_type") @db.Integer createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) - deletedAt DateTime? @default(now()) @map("deleted_at") @db.Timestamptz(6) website Website @relation(fields: [websiteId], references: [id]) session Session @relation(fields: [sessionId], references: [id]) @@ -161,6 +160,8 @@ model SessionData { @@index([createdAt]) @@index([websiteId]) @@index([sessionId]) + @@index([sessionId, createdAt]) + @@index([websiteId, createdAt, dataKey]) @@map("session_data") } diff --git a/src/app/(main)/reports/event-data/EventDataParameters.tsx b/src/app/(main)/reports/event-data/EventDataParameters.tsx index adc18274..e0afda3f 100644 --- a/src/app/(main)/reports/event-data/EventDataParameters.tsx +++ b/src/app/(main)/reports/event-data/EventDataParameters.tsx @@ -75,8 +75,8 @@ export function EventDataParameters() { {(close: () => void) => { return ( ({ - name: eventKey, + fields={data.map(({ dataKey, eventDataType }) => ({ + name: dataKey, type: DATA_TYPES[eventDataType], }))} group={group} diff --git a/src/queries/analytics/eventData/getEventDataEvents.ts b/src/queries/analytics/eventData/getEventDataEvents.ts index 3eeb3702..f869deae 100644 --- a/src/queries/analytics/eventData/getEventDataEvents.ts +++ b/src/queries/analytics/eventData/getEventDataEvents.ts @@ -22,7 +22,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ` select website_event.event_name as "eventName", - event_data.event_key as "fieldName", + event_data.data_key as "fieldName", event_data.data_type as "dataType", event_data.string_value as "fieldValue", count(*) as "total" @@ -32,7 +32,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { where event_data.website_id = {{websiteId::uuid}} and event_data.created_at between {{startDate}} and {{endDate}} and website_event.event_name = {{event}} - group by website_event.event_name, event_data.event_key, event_data.data_type, event_data.string_value + group by website_event.event_name, event_data.data_key, event_data.data_type, event_data.string_value order by 1 asc, 2 asc, 3 asc, 4 desc `, params, @@ -43,7 +43,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ` select website_event.event_name as "eventName", - event_data.event_key as "fieldName", + event_data.data_key as "fieldName", event_data.data_type as "dataType", count(*) as "total" from event_data @@ -51,7 +51,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { on website_event.event_id = event_data.website_event_id where event_data.website_id = {{websiteId::uuid}} and event_data.created_at between {{startDate}} and {{endDate}} - group by website_event.event_name, event_data.event_key, event_data.data_type + group by website_event.event_name, event_data.data_key, event_data.data_type order by 1 asc, 2 asc limit 500 `, @@ -72,7 +72,7 @@ async function clickhouseQuery( ` select event_name as eventName, - event_key as fieldName, + data_key as fieldName, data_type as dataType, string_value as fieldValue, count(*) as total @@ -80,7 +80,7 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_name = {event:String} - group by event_key, data_type, string_value, event_name + group by data_key, data_type, string_value, event_name order by 1 asc, 2 asc, 3 asc, 4 desc limit 500 `, @@ -102,13 +102,13 @@ async function clickhouseQuery( ` select event_name as eventName, - event_key as fieldName, + data_key as fieldName, data_type as dataType, count(*) as total from event_data where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - group by event_key, data_type, event_name + group by data_key, data_type, event_name order by 1 asc, 2 asc limit 500 `, diff --git a/src/queries/analytics/eventData/getEventDataFields.ts b/src/queries/analytics/eventData/getEventDataFields.ts index 293aed24..6ec3b35a 100644 --- a/src/queries/analytics/eventData/getEventDataFields.ts +++ b/src/queries/analytics/eventData/getEventDataFields.ts @@ -15,13 +15,13 @@ export async function getEventDataFields( async function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) { const { rawQuery, parseFilters } = prisma; const { filterQuery, params } = await parseFilters(websiteId, filters, { - columns: { field: 'event_key' }, + columns: { field: 'data_key' }, }); return rawQuery( ` select - event_key as "fieldName", + data_key as "fieldName", data_type as "dataType", string_value as "fieldValue", count(*) as "total" @@ -29,7 +29,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} ${filterQuery} - group by event_key, data_type, string_value + group by data_key, data_type, string_value order by 3 desc, 2 desc, 1 asc limit 500 `, @@ -43,13 +43,13 @@ async function clickhouseQuery( ): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, filters, { - columns: { field: 'event_key' }, + columns: { field: 'data_key' }, }); return rawQuery( ` select - event_key as fieldName, + data_key as fieldName, data_type as dataType, string_value as fieldValue, count(*) as total @@ -57,7 +57,7 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by event_key, data_type, string_value + group by data_key, data_type, string_value order by 3 desc, 2 desc, 1 asc limit 500 `, diff --git a/src/queries/analytics/eventData/getEventDataStats.ts b/src/queries/analytics/eventData/getEventDataStats.ts index 39afa1ae..978f561b 100644 --- a/src/queries/analytics/eventData/getEventDataStats.ts +++ b/src/queries/analytics/eventData/getEventDataStats.ts @@ -24,18 +24,18 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ` select count(distinct t.website_event_id) as "events", - count(distinct t.event_key) as "fields", + count(distinct t.data_key) as "fields", sum(t.total) as "records" from ( select website_event_id, - event_key, + data_key, count(*) as "total" from event_data where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} ${filterQuery} - group by website_event_id, event_key + group by website_event_id, data_key ) as t `, params, @@ -53,18 +53,18 @@ async function clickhouseQuery( ` select count(distinct t.event_id) as "events", - count(distinct t.event_key) as "fields", + count(distinct t.data_key) as "fields", sum(t.total) as "records" from ( select event_id, - event_key, + data_key, count(*) as "total" from event_data where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by event_id, event_key + group by event_id, data_key ) as t `, params, diff --git a/src/queries/analytics/eventData/saveEventData.ts b/src/queries/analytics/eventData/saveEventData.ts index c37dd896..5bfb5f09 100644 --- a/src/queries/analytics/eventData/saveEventData.ts +++ b/src/queries/analytics/eventData/saveEventData.ts @@ -36,7 +36,7 @@ async function relationalQuery(data: { id: uuid(), websiteEventId: eventId, websiteId, - eventKey: a.key, + dataKey: a.key, stringValue: getStringValue(a.value, a.dataType), numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, @@ -69,7 +69,7 @@ async function clickhouseQuery(data: { event_id: eventId, url_path: urlPath, event_name: eventName, - event_key: a.key, + data_key: a.key, string_value: getStringValue(a.value, a.dataType), number_value: a.dataType === DATA_TYPE.number ? a.value : null, date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,