Merge pull request #2200 from umami-software/feat/um-311-relational-indexing

Feat/um 311 relational indexing
2024-11-16 02:05:04 +01:00 · 2023-08-16 19:15:55 -10:00 · 2023-08-16 19:15:55 -10:00 · bd878587e5
commit bd878587e5
parent 2e2b1411bf 6dba68c823
7 changed files with 155 additions and 9 deletions
--- a/db/mysql/migrations/03_metric_performance_index/migration.sql
+++ b/db/mysql/migrations/03_metric_performance_index/migration.sql
@ -0,0 +1,50 @@
+-- CreateIndex
+CREATE INDEX `event_data_website_id_created_at_idx` ON `event_data`(`website_id`, `created_at`);
+
+-- CreateIndex
+CREATE INDEX `event_data_website_id_created_at_event_key_idx` ON `event_data`(`website_id`, `created_at`, `event_key`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_idx` ON `session`(`website_id`, `created_at`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_hostname_idx` ON `session`(`website_id`, `created_at`, `hostname`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_browser_idx` ON `session`(`website_id`, `created_at`, `browser`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_os_idx` ON `session`(`website_id`, `created_at`, `os`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_device_idx` ON `session`(`website_id`, `created_at`, `device`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_screen_idx` ON `session`(`website_id`, `created_at`, `screen`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_language_idx` ON `session`(`website_id`, `created_at`, `language`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_country_idx` ON `session`(`website_id`, `created_at`, `country`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_subdivision1_idx` ON `session`(`website_id`, `created_at`, `subdivision1`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_city_idx` ON `session`(`website_id`, `created_at`, `city`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_url_path_idx` ON `website_event`(`website_id`, `created_at`, `url_path`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_url_query_idx` ON `website_event`(`website_id`, `created_at`, `url_query`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_referrer_domain_idx` ON `website_event`(`website_id`, `created_at`, `referrer_domain`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_page_title_idx` ON `website_event`(`website_id`, `created_at`, `page_title`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_event_name_idx` ON `website_event`(`website_id`, `created_at`, `event_name`);
--- a/db/mysql/schema.prisma
+++ b/db/mysql/schema.prisma
@ -44,6 +44,16 @@ model Session {

  @@index([createdAt])
  @@index([websiteId])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, hostname])
+  @@index([websiteId, createdAt, browser])
+  @@index([websiteId, createdAt, os])
+  @@index([websiteId, createdAt, device])
+  @@index([websiteId, createdAt, screen])
+  @@index([websiteId, createdAt, language])
+  @@index([websiteId, createdAt, country])
+  @@index([websiteId, createdAt, subdivision1])
+  @@index([websiteId, createdAt, city])
  @@map("session")
 }

@ -91,6 +101,11 @@ model WebsiteEvent {
  @@index([sessionId])
  @@index([websiteId])
  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, urlPath])
+  @@index([websiteId, createdAt, urlQuery])
+  @@index([websiteId, createdAt, referrerDomain])
+  @@index([websiteId, createdAt, pageTitle])
+  @@index([websiteId, createdAt, eventName])
  @@index([websiteId, sessionId, createdAt])
  @@map("website_event")
 }
@ -113,6 +128,8 @@ model EventData {
  @@index([websiteId])
  @@index([websiteEventId])
  @@index([websiteId, websiteEventId, createdAt])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, eventKey])
  @@map("event_data")
 }

--- a/db/postgresql/migrations/03_metric_performance_index/migration.sql
+++ b/db/postgresql/migrations/03_metric_performance_index/migration.sql
@ -0,0 +1,50 @@
+-- CreateIndex
+CREATE INDEX "event_data_website_id_created_at_idx" ON "event_data"("website_id", "created_at");
+
+-- CreateIndex
+CREATE INDEX "event_data_website_id_created_at_event_key_idx" ON "event_data"("website_id", "created_at", "event_key");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_idx" ON "session"("website_id", "created_at");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_hostname_idx" ON "session"("website_id", "created_at", "hostname");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_browser_idx" ON "session"("website_id", "created_at", "browser");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_os_idx" ON "session"("website_id", "created_at", "os");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_device_idx" ON "session"("website_id", "created_at", "device");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_screen_idx" ON "session"("website_id", "created_at", "screen");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_language_idx" ON "session"("website_id", "created_at", "language");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_country_idx" ON "session"("website_id", "created_at", "country");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_subdivision1_idx" ON "session"("website_id", "created_at", "subdivision1");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_city_idx" ON "session"("website_id", "created_at", "city");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_url_path_idx" ON "website_event"("website_id", "created_at", "url_path");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_url_query_idx" ON "website_event"("website_id", "created_at", "url_query");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_referrer_domain_idx" ON "website_event"("website_id", "created_at", "referrer_domain");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_page_title_idx" ON "website_event"("website_id", "created_at", "page_title");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_event_name_idx" ON "website_event"("website_id", "created_at", "event_name");
--- a/db/postgresql/schema.prisma
+++ b/db/postgresql/schema.prisma
@ -44,6 +44,16 @@ model Session {

  @@index([createdAt])
  @@index([websiteId])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, hostname])
+  @@index([websiteId, createdAt, browser])
+  @@index([websiteId, createdAt, os])
+  @@index([websiteId, createdAt, device])
+  @@index([websiteId, createdAt, screen])
+  @@index([websiteId, createdAt, language])
+  @@index([websiteId, createdAt, country])
+  @@index([websiteId, createdAt, subdivision1])
+  @@index([websiteId, createdAt, city])
  @@map("session")
 }

@ -91,6 +101,11 @@ model WebsiteEvent {
  @@index([sessionId])
  @@index([websiteId])
  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, urlPath])
+  @@index([websiteId, createdAt, urlQuery])
+  @@index([websiteId, createdAt, referrerDomain])
+  @@index([websiteId, createdAt, pageTitle])
+  @@index([websiteId, createdAt, eventName])
  @@index([websiteId, sessionId, createdAt])
  @@map("website_event")
 }
@ -112,6 +127,8 @@ model EventData {
  @@index([createdAt])
  @@index([websiteId])
  @@index([websiteEventId])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, eventKey])
  @@map("event_data")
 }

--- a/lib/clickhouse.ts
+++ b/lib/clickhouse.ts
@ -66,7 +66,7 @@ function getDateFormat(date) {
 function mapFilter(column, operator, name, type = 'String') {
  switch (operator) {
    case OPERATORS.equals:
-      return `${column} = {${name}:${type}`;
+      return `${column} = {${name}:${type}}`;
    case OPERATORS.notEquals:
      return `${column} != {${name}:${type}}`;
    default:
@ -94,13 +94,23 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
  return query.join('\n');
 }

+function normalizeFilters(filters = {}) {
+  return Object.keys(filters).reduce((obj, key) => {
+    const value = filters[key];
+
+    obj[key] = value?.value ?? value;
+
+    return obj;
+  }, {});
+}
+
 async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) {
  const website = await loadWebsite(websiteId);

  return {
    filterQuery: getFilterQuery(filters, options),
    params: {
-      ...filters,
+      ...normalizeFilters(filters),
      websiteId,
      startDate: maxDate(filters.startDate, website.resetAt),
      websiteDomain: website.domain,
--- a/lib/prisma.ts
+++ b/lib/prisma.ts
@ -92,12 +92,12 @@ function getTimestampIntervalQuery(field: string): string {
  }
 }

-function mapFilter(column, operator, name, type = 'String') {
+function mapFilter(column, operator, name, type = 'varchar') {
  switch (operator) {
    case OPERATORS.equals:
-      return `${column} = {${name}:${type}`;
+      return `${column} = {{${name}::${type}}}`;
    case OPERATORS.notEquals:
-      return `${column} != {${name}:${type}}`;
+      return `${column} != {{${name}::${type}}}`;
    default:
      return '';
  }
@ -161,7 +161,7 @@ async function rawQuery(sql: string, data: object): Promise<any> {
    return Promise.reject(new Error('Unknown database.'));
  }

-  const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
+  const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*\}\}/g, (...args) => {
    const [, name, type] = args;
    params.push(data[name]);

--- a/queries/analytics/reports/getFunnel.ts
+++ b/queries/analytics/reports/getFunnel.ts
@ -57,12 +57,14 @@ async function relationalQuery(
            from level${i} l
            join website_event we
                on l.session_id = we.session_id
-            where we.created_at between l.created_at 
-                and ${getAddMinutesQuery(`l.created_at `, windowMinutes)}
+            where we.website_id = {{websiteId::uuid}}
+                and we.created_at between l.created_at and ${getAddMinutesQuery(
+                  `l.created_at `,
+                  windowMinutes,
+                )}
                and we.referrer_path = {{${i - 1}}}
                and we.url_path = {{${i}}}
                and we.created_at <= {{endDate}}
-                and we.website_id = {{websiteId::uuid}}
          )`;
        }