From 9979672de52a661714057af6ef7768b47374db7a Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 23 Mar 2023 14:01:15 -0700 Subject: [PATCH] Feat/um 202 event data new (#1841) * Add event_data base. * Add url_path. * Add eventData back. * Finish event_data relational. * resolve comments. --- .eslintrc.json | 3 +- components/pages/console/TestConsole.js | 27 ++-- db/clickhouse/schema.sql | 7 +- db/mysql/migrations/01_init/migration.sql | 10 +- db/mysql/schema.prisma | 42 +++---- .../09_add_new_event_data/migration.sql | 38 ------ .../migrations/09_event_data/migration.sql | 23 ++++ db/postgresql/schema.prisma | 17 +-- lib/clickhouse.ts | 41 ++++++ lib/constants.ts | 13 ++ lib/eventData.ts | 74 +++++++++++ lib/{kafka.js => kafka.ts} | 45 ++++--- lib/prisma.ts | 43 +++++++ lib/types.ts | 18 ++- middleware.js | 2 +- package.json | 1 + pages/api/send.ts | 13 +- pages/api/websites/[id]/eventData.ts | 60 +++++++++ queries/analytics/event/saveEvent.ts | 46 ++++++- queries/analytics/eventData/getEventData.ts | 117 ++++++++++++++++++ queries/analytics/eventData/saveEventData.ts | 96 ++++++++++++++ queries/analytics/pageview/savePageView.ts | 1 + queries/index.js | 1 + scripts/telemetry.js | 2 +- scripts/update-tracker.js | 2 +- tracker/index.js | 6 +- yarn.lock | 101 ++++++++++++++- 27 files changed, 719 insertions(+), 130 deletions(-) delete mode 100644 db/postgresql/migrations/09_add_new_event_data/migration.sql create mode 100644 db/postgresql/migrations/09_event_data/migration.sql create mode 100644 lib/eventData.ts rename lib/{kafka.js => kafka.ts} (66%) create mode 100644 pages/api/websites/[id]/eventData.ts create mode 100644 queries/analytics/eventData/getEventData.ts create mode 100644 queries/analytics/eventData/saveEventData.ts diff --git a/.eslintrc.json b/.eslintrc.json index de639b85..bcc38cf8 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -48,7 +48,8 @@ "import/no-anonymous-default-export": "off", "@next/next/no-img-element": "off", "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-explicit-any": "off" + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-var-requires": "off" }, "globals": { "React": "writable" diff --git a/components/pages/console/TestConsole.js b/components/pages/console/TestConsole.js index 88e69d7d..c7e41c62 100644 --- a/components/pages/console/TestConsole.js +++ b/components/pages/console/TestConsole.js @@ -1,13 +1,13 @@ -import { Button, Column, Row, Dropdown, Item } from 'react-basics'; -import Head from 'next/head'; -import Link from 'next/link'; -import { useRouter } from 'next/router'; +import WebsiteSelect from 'components/input/WebsiteSelect'; import Page from 'components/layout/Page'; import PageHeader from 'components/layout/PageHeader'; import EventsChart from 'components/metrics/EventsChart'; import WebsiteChart from 'components/metrics/WebsiteChart'; -import WebsiteSelect from 'components/input/WebsiteSelect'; import useApi from 'hooks/useApi'; +import Head from 'next/head'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; +import { Button, Column, Row } from 'react-basics'; import styles from './TestConsole.module.css'; export default function TestConsole() { @@ -27,7 +27,20 @@ export default function TestConsole() { window.umami('umami-default'); window.umami.trackView('/page-view', 'https://www.google.com'); window.umami.trackEvent('track-event-no-data'); - window.umami.trackEvent('track-event-with-data', { test: 'test-data', time: Date.now() }); + window.umami.trackEvent('track-event-with-data', { + test: 'test-data', + time: new Date(), + number: 1, + time2: new Date().toISOString(), + nested: { + test: 'test-data', + number: 1, + object: { + test: 'test-data', + }, + }, + array: [1, 2, 3], + }); } if (!data) { @@ -45,7 +58,7 @@ export default function TestConsole() { async defer data-website-id={website.id} - src={`${basePath}/umami.js`} + src={`${basePath}/script.js`} data-cache="true" /> )} diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index acae2ac9..b45f8aa0 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -106,16 +106,15 @@ CREATE TABLE umami.event_data event_name String, event_key String, event_string_value Nullable(String), - event_numeric_value Nullable(Decimal64(4)), + event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 event_date_value Nullable(DateTime('UTC')), event_data_type UInt32, created_at DateTime('UTC') ) engine = MergeTree - ORDER BY (website_id, session_id, event_id, event_key, created_at) + ORDER BY (website_id, event_id, event_key, created_at) SETTINGS index_granularity = 8192; -CREATE TABLE umami.event_data_queue ( website_id UUID, session_id UUID, event_id UUID, @@ -124,7 +123,7 @@ CREATE TABLE umami.event_data_queue ( event_name String, event_key String, event_string_value Nullable(String), - event_numeric_value Nullable(Decimal64(4)), + event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 event_date_value Nullable(DateTime('UTC')), event_data_type UInt32, created_at DateTime('UTC') diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql index 586c604f..58b7e698 100644 --- a/db/mysql/migrations/01_init/migration.sql +++ b/db/mysql/migrations/01_init/migration.sql @@ -83,23 +83,17 @@ CREATE TABLE `event_data` ( `event_id` VARCHAR(36) NOT NULL, `website_event_id` VARCHAR(36) NOT NULL, `website_id` VARCHAR(36) NOT NULL, - `session_id` VARCHAR(36) NOT NULL, - `url_path` VARCHAR(500) NOT NULL, - `event_name` VARCHAR(500) NOT NULL, `event_key` VARCHAR(500) NOT NULL, - `event_string_value` VARCHAR(500) NOT NULL, - `event_numeric_value` DECIMAL(19, 4) NOT NULL, + `event_string_value` VARCHAR(500) NULL, + `event_numeric_value` DECIMAL(19, 4) NULL, `event_date_value` TIMESTAMP(0) NULL, `event_data_type` INTEGER UNSIGNED NOT NULL, `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), INDEX `event_data_created_at_idx`(`created_at`), - INDEX `event_data_session_id_idx`(`session_id`), INDEX `event_data_website_id_idx`(`website_id`), INDEX `event_data_website_event_id_idx`(`website_event_id`), INDEX `event_data_website_id_website_event_id_created_at_idx`(`website_id`, `website_event_id`, `created_at`), - INDEX `event_data_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`), - INDEX `event_data_website_id_session_id_website_event_id_created_at_idx`(`website_id`, `session_id`, `website_event_id`, `created_at`), PRIMARY KEY (`event_id`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 7a7a1b15..13dcdcbc 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -17,29 +17,28 @@ model User { updatedAt DateTime? @map("updated_at") @db.Timestamp(0) deletedAt DateTime? @map("deleted_at") @db.Timestamp(0) - website Website[] - teamUser TeamUser[] + website Website[] + teamUser TeamUser[] @@map("user") } model Session { - id String @id @unique @map("session_id") @db.VarChar(36) - websiteId String @map("website_id") @db.VarChar(36) - hostname String? @db.VarChar(100) - browser String? @db.VarChar(20) - os String? @db.VarChar(20) - device String? @db.VarChar(20) - screen String? @db.VarChar(11) - language String? @db.VarChar(35) - country String? @db.Char(2) - subdivision1 String? @db.Char(3) - subdivision2 String? @db.VarChar(50) - city String? @db.VarChar(50) + id String @id @unique @map("session_id") @db.VarChar(36) + websiteId String @map("website_id") @db.VarChar(36) + hostname String? @db.VarChar(100) + browser String? @db.VarChar(20) + os String? @db.VarChar(20) + device String? @db.VarChar(20) + screen String? @db.VarChar(11) + language String? @db.VarChar(35) + country String? @db.Char(2) + subdivision1 String? @db.Char(3) + subdivision2 String? @db.VarChar(50) + city String? @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) websiteEvent WebsiteEvent[] - eventData EventData[] @@index([createdAt]) @@index([websiteId]) @@ -96,27 +95,20 @@ model EventData { id String @id() @map("event_id") @db.VarChar(36) websiteEventId String @map("website_event_id") @db.VarChar(36) websiteId String @map("website_id") @db.VarChar(36) - sessionId String @map("session_id") @db.VarChar(36) - urlPath String @map("url_path") @db.VarChar(500) - eventName String @map("event_name") @db.VarChar(500) eventKey String @map("event_key") @db.VarChar(500) - eventStringValue String @map("event_string_value") @db.VarChar(500) - eventNumericValue Decimal @map("event_numeric_value") @db.Decimal(19,4) + eventStringValue String? @map("event_string_value") @db.VarChar(500) + eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0) eventDataType Int @map("event_data_type") @db.UnsignedInt - createdAt DateTime? @default(now()) @map("created_at")@db.Timestamp(0) + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) website Website @relation(fields: [websiteId], references: [id]) websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) - session Session @relation(fields: [sessionId], references: [id]) @@index([createdAt]) - @@index([sessionId]) @@index([websiteId]) @@index([websiteEventId]) @@index([websiteId, websiteEventId, createdAt]) - @@index([websiteId, sessionId, createdAt]) - @@index([websiteId, sessionId, websiteEventId, createdAt]) @@map("event_data") } diff --git a/db/postgresql/migrations/09_add_new_event_data/migration.sql b/db/postgresql/migrations/09_add_new_event_data/migration.sql deleted file mode 100644 index 932d4f19..00000000 --- a/db/postgresql/migrations/09_add_new_event_data/migration.sql +++ /dev/null @@ -1,38 +0,0 @@ --- CreateTable -CREATE TABLE "event_data" ( - "event_id" UUID NOT NULL, - "website_event_id" UUID NOT NULL, - "website_id" UUID NOT NULL, - "session_id" UUID NOT NULL, - "url_path" VARCHAR(500) NOT NULL, - "event_name" VARCHAR(500) NOT NULL, - "event_key" VARCHAR(500) NOT NULL, - "event_string_value" VARCHAR(500) NOT NULL, - "event_numeric_value" DECIMAL(19,4) NOT NULL, - "event_date_value" TIMESTAMPTZ(6), - "event_data_type" INTEGER NOT NULL, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id") -); - --- CreateIndex -CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at"); - --- CreateIndex -CREATE INDEX "event_data_session_id_idx" ON "event_data"("session_id"); - --- CreateIndex -CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id"); - --- CreateIndex -CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id"); - --- CreateIndex -CREATE INDEX "event_data_website_id_website_event_id_created_at_idx" ON "event_data"("website_id", "website_event_id", "created_at"); - --- CreateIndex -CREATE INDEX "event_data_website_id_session_id_created_at_idx" ON "event_data"("website_id", "session_id", "created_at"); - --- CreateIndex -CREATE INDEX "event_data_website_id_session_id_website_event_id_created_a_idx" ON "event_data"("website_id", "session_id", "website_event_id", "created_at"); diff --git a/db/postgresql/migrations/09_event_data/migration.sql b/db/postgresql/migrations/09_event_data/migration.sql new file mode 100644 index 00000000..1196e1c7 --- /dev/null +++ b/db/postgresql/migrations/09_event_data/migration.sql @@ -0,0 +1,23 @@ +-- CreateTable +CREATE TABLE "event_data" ( + "event_id" UUID NOT NULL, + "website_id" UUID NOT NULL, + "website_event_id" UUID NOT NULL, + "event_key" VARCHAR(500) NOT NULL, + "event_string_value" VARCHAR(500), + "event_numeric_value" DECIMAL(19,4), + "event_date_value" TIMESTAMPTZ(6), + "event_data_type" INTEGER NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id") +); + +-- CreateIndex +CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at"); + +-- CreateIndex +CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id"); + +-- CreateIndex +CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index a29b96b2..bb26a807 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -39,7 +39,6 @@ model Session { createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) websiteEvent WebsiteEvent[] - eventData EventData[] @@index([createdAt]) @@index([websiteId]) @@ -94,29 +93,21 @@ model WebsiteEvent { model EventData { id String @id() @map("event_id") @db.Uuid - websiteEventId String @map("website_event_id") @db.Uuid websiteId String @map("website_id") @db.Uuid - sessionId String @map("session_id") @db.Uuid - urlPath String @map("url_path") @db.VarChar(500) - eventName String @map("event_name") @db.VarChar(500) + websiteEventId String @map("website_event_id") @db.Uuid eventKey String @map("event_key") @db.VarChar(500) - eventStringValue String @map("event_string_value") @db.VarChar(500) - eventNumericValue Decimal @map("event_numeric_value") @db.Decimal(19,4) + eventStringValue String? @map("event_string_value") @db.VarChar(500) + eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6) eventDataType Int @map("event_data_type") @db.Integer createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) website Website @relation(fields: [websiteId], references: [id]) websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) - session Session @relation(fields: [sessionId], references: [id]) @@index([createdAt]) - @@index([sessionId]) @@index([websiteId]) @@index([websiteEventId]) - @@index([websiteId, websiteEventId, createdAt]) - @@index([websiteId, sessionId, createdAt]) - @@index([websiteId, sessionId, websiteEventId, createdAt]) @@map("event_data") } @@ -162,4 +153,4 @@ model TeamWebsite { @@index([teamId]) @@index([websiteId]) @@map("team_website") -} \ No newline at end of file +} diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index 073a92e7..d9e6a02b 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -3,6 +3,7 @@ import dateFormat from 'dateformat'; import debug from 'debug'; import { FILTER_IGNORED } from 'lib/constants'; import { CLICKHOUSE } from 'lib/db'; +import { getEventDataType } from './eventData'; export const CLICKHOUSE_DATE_FORMATS = { minute: '%Y-%m-%d %H:%M:00', @@ -64,6 +65,45 @@ function getBetweenDates(field, startAt, endAt) { return `${field} between ${getDateFormat(startAt)} and ${getDateFormat(endAt)}`; } +function getEventDataFilterQuery( + filters: { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }[] = [], + params: any, +) { + const query = filters.reduce((ac, cv, i) => { + const type = getEventDataType(cv.eventValue); + + let value = cv.eventValue; + + ac.push(`and (event_key = {eventKey${i}:String}`); + + switch (type) { + case 'number': + ac.push(`and event_numeric_value = {eventValue${i}:UInt64})`); + break; + case 'string': + ac.push(`and event_string_value = {eventValue${i}:String})`); + break; + case 'boolean': + ac.push(`and event_string_value = {eventValue${i}:String})`); + value = cv ? 'true' : 'false'; + break; + case 'date': + ac.push(`and event_date_value = {eventValue${i}:DateTime('UTC')})`); + break; + } + + params[`eventKey${i}`] = cv.eventKey; + params[`eventValue${i}`] = value; + + return ac; + }, []); + + return query.join('\n'); +} + function getFilterQuery(filters = {}, params = {}) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -189,6 +229,7 @@ export default { getDateFormat, getBetweenDates, getFilterQuery, + getEventDataFilterQuery, parseFilters, findUnique, findFirst, diff --git a/lib/constants.ts b/lib/constants.ts index f7bce52e..1c5aa513 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -38,6 +38,19 @@ export const EVENT_TYPE = { customEvent: 2, } as const; +export const EVENT_DATA_TYPE = { + string: 1, + number: 2, + boolean: 3, + date: 4, + array: 5, +} as const; + +export const KAFKA_TOPIC = { + event: 'event', + eventData: 'event_data', +} as const; + export const ROLES = { admin: 'admin', user: 'user', diff --git a/lib/eventData.ts b/lib/eventData.ts new file mode 100644 index 00000000..4588d081 --- /dev/null +++ b/lib/eventData.ts @@ -0,0 +1,74 @@ +import { isValid, parseISO } from 'date-fns'; +import { EVENT_DATA_TYPE } from './constants'; +import { EventDataTypes } from './types'; + +export function flattenJSON( + eventData: { [key: string]: any }, + keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [], + parentKey = '', +): { key: string; value: any; eventDataType: EventDataTypes }[] { + return Object.keys(eventData).reduce( + (acc, key) => { + const value = eventData[key]; + const type = typeof eventData[key]; + + // nested object + if (value && type === 'object' && !Array.isArray(value) && !isValid(value)) { + flattenJSON(value, acc.keyValues, getKeyName(key, parentKey)); + } else { + createKey(getKeyName(key, parentKey), value, acc); + } + + return acc; + }, + { keyValues, parentKey }, + ).keyValues; +} + +export function getEventDataType(value: any): string { + let type: string = typeof value; + + if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) { + type = 'date'; + } + + return type; +} + +function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) { + const type = getEventDataType(value); + + let eventDataType = null; + + switch (type) { + case 'number': + eventDataType = EVENT_DATA_TYPE.number; + break; + case 'string': + eventDataType = EVENT_DATA_TYPE.string; + break; + case 'boolean': + eventDataType = EVENT_DATA_TYPE.boolean; + break; + case 'date': + eventDataType = EVENT_DATA_TYPE.date; + break; + case 'object': + eventDataType = EVENT_DATA_TYPE.array; + value = JSON.stringify(value); + break; + default: + eventDataType = EVENT_DATA_TYPE.string; + break; + } + + acc.keyValues.push({ key, value, eventDataType }); +} + +function getKeyName(key, parentKey) { + if (!parentKey) { + return key; + } + + return `${parentKey}.${key}`; +} diff --git a/lib/kafka.js b/lib/kafka.ts similarity index 66% rename from lib/kafka.js rename to lib/kafka.ts index 782f2803..3d3e281c 100644 --- a/lib/kafka.js +++ b/lib/kafka.ts @@ -1,19 +1,20 @@ -import { Kafka, logLevel } from 'kafkajs'; import dateFormat from 'dateformat'; import debug from 'debug'; +import { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs'; import { KAFKA, KAFKA_PRODUCER } from 'lib/db'; +import * as tls from 'tls'; const log = debug('umami:kafka'); -let kafka; -let producer; +let kafka: Kafka; +let producer: Producer; const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER); function getClient() { const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); - const ssl = + const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } = username && password ? { ssl: { @@ -30,7 +31,7 @@ function getClient() { } : {}; - const client = new Kafka({ + const client: Kafka = new Kafka({ clientId: 'umami', brokers: brokers, connectionTimeout: 3000, @@ -47,7 +48,7 @@ function getClient() { return client; } -async function getProducer() { +async function getProducer(): Promise { const producer = kafka.producer(); await producer.connect(); @@ -60,25 +61,40 @@ async function getProducer() { return producer; } -function getDateFormat(date) { +function getDateFormat(date): string { return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss'); } -async function sendMessage(params, topic) { +async function sendMessage( + message: { [key: string]: string | number }, + topic: string, +): Promise { + await connect(); + + return producer.send({ + topic, + messages: [ + { + value: JSON.stringify(message), + }, + ], + acks: -1, + }); +} + +async function sendMessages(messages: { [key: string]: string | number }[], topic: string) { await connect(); await producer.send({ topic, - messages: [ - { - value: JSON.stringify(params), - }, - ], + messages: messages.map(a => { + return { value: JSON.stringify(a) }; + }), acks: 1, }); } -async function connect() { +async function connect(): Promise { if (!kafka) { kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient()); @@ -98,4 +114,5 @@ export default { connect, getDateFormat, sendMessage, + sendMessages, }; diff --git a/lib/prisma.ts b/lib/prisma.ts index 20a5e4a6..98bf921e 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -2,6 +2,7 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; import { FILTER_IGNORED } from 'lib/constants'; +import { getEventDataType } from './eventData'; const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -64,6 +65,47 @@ function getTimestampInterval(field: string): string { } } +function getEventDataFilterQuery( + filters: { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }[], + params: any[], +) { + const query = filters.reduce((ac, cv) => { + const type = getEventDataType(cv.eventValue); + + let value = cv.eventValue; + + ac.push(`and (event_key = $${params.length + 1}`); + params.push(cv.eventKey); + + switch (type) { + case 'number': + ac.push(`and event_numeric_value = $${params.length + 1})`); + params.push(value); + break; + case 'string': + ac.push(`and event_string_value = $${params.length + 1})`); + params.push(decodeURIComponent(cv.eventValue as string)); + break; + case 'boolean': + ac.push(`and event_string_value = $${params.length + 1})`); + params.push(decodeURIComponent(cv.eventValue as string)); + value = cv ? 'true' : 'false'; + break; + case 'date': + ac.push(`and event_date_value = $${params.length + 1})`); + params.push(cv.eventValue); + break; + } + + return ac; + }, []); + + return query.join('\n'); +} + function getFilterQuery(filters = {}, params = []): string { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -173,6 +215,7 @@ export default { getDateQuery, getTimestampInterval, getFilterQuery, + getEventDataFilterQuery, toUuid, parseFilters, rawQuery, diff --git a/lib/types.ts b/lib/types.ts index 68aa144e..c818d6ff 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1,10 +1,19 @@ import { NextApiRequest } from 'next'; -import { ROLES } from './constants'; +import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants'; type ObjectValues = T[keyof T]; export type Roles = ObjectValues; +export type EventTypes = ObjectValues; + +export type EventDataTypes = ObjectValues; + +export type KafkaTopics = ObjectValues; + +export interface EventData { + [key: string]: number | string | EventData | number[] | string[] | EventData[]; +} export interface Auth { user?: { id: string; @@ -66,6 +75,13 @@ export interface WebsiteEventMetric { y: number; } +export interface WebsiteEventDataMetric { + x: string; + t: string; + eventName?: string; + urlPath?: string; +} + export interface WebsitePageviews { pageviews: { t: string; diff --git a/middleware.js b/middleware.js index fb7fe7f5..5afc4f90 100644 --- a/middleware.js +++ b/middleware.js @@ -12,7 +12,7 @@ function customCollectEndpoint(req) { const { pathname } = url; if (pathname.endsWith(collectEndpoint)) { - url.pathname = '/api/collect'; + url.pathname = '/api/send'; return NextResponse.rewrite(url); } } diff --git a/package.json b/package.json index 71452dca..0f7eeccf 100644 --- a/package.json +++ b/package.json @@ -143,6 +143,7 @@ "stylelint-config-prettier": "^9.0.3", "stylelint-config-recommended": "^9.0.0", "tar": "^6.1.2", + "ts-node": "^10.9.1", "typescript": "^4.9.5" } } diff --git a/pages/api/send.ts b/pages/api/send.ts index 86632bf3..1a880a46 100644 --- a/pages/api/send.ts +++ b/pages/api/send.ts @@ -34,7 +34,13 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { const { type, payload } = getJsonBody(req); - const { url, referrer, eventName, pageTitle } = payload; + const { url, referrer, eventName, eventData, pageTitle } = payload; + + // Validate eventData is JSON + if (eventData && !(typeof eventData === 'object' && !Array.isArray(eventData))) { + return badRequest(res, 'Event Data must be in the form of a JSON Object.'); + } + const ignoreIps = process.env.IGNORE_IP; const ignoreHostnames = process.env.IGNORE_HOSTNAME; @@ -93,8 +99,8 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { referrerDomain = newRef.hostname; referrerQuery = newRef.search.substring(1); } catch { - referrerPath = referrer.split('?')[0]; - referrerQuery = referrer.split('?')[1]; + referrerPath = referrer?.split('?')[0]; + referrerQuery = referrer?.split('?')[1]; } if (process.env.REMOVE_TRAILING_SLASH) { @@ -118,6 +124,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { urlQuery, pageTitle, eventName, + eventData, }); } else { return badRequest(res); diff --git a/pages/api/websites/[id]/eventData.ts b/pages/api/websites/[id]/eventData.ts new file mode 100644 index 00000000..65c4d687 --- /dev/null +++ b/pages/api/websites/[id]/eventData.ts @@ -0,0 +1,60 @@ +import { canViewWebsite } from 'lib/auth'; +import { useAuth, useCors } from 'lib/middleware'; +import { NextApiRequestQueryBody, WebsiteEventDataMetric } from 'lib/types'; +import { NextApiResponse } from 'next'; +import { methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { getEventData } from 'queries'; + +export interface WebsiteEventDataRequestQuery { + id: string; +} + +export interface WebsiteEventDataRequestBody { + startAt: string; + endAt: string; + eventName?: string; + urlPath?: string; + timeSeries?: { + unit: string; + timezone: string; + }; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; +} + +export default async ( + req: NextApiRequestQueryBody, + res: NextApiResponse, +) => { + await useCors(req, res); + await useAuth(req, res); + + const { id: websiteId } = req.query; + + if (req.method === 'POST') { + if (!(await canViewWebsite(req.auth, websiteId))) { + return unauthorized(res); + } + + const { startAt, endAt, eventName, urlPath, filters } = req.body; + + const startDate = new Date(+startAt); + const endDate = new Date(+endAt); + + const events = await getEventData(websiteId, { + startDate, + endDate, + eventName, + urlPath, + filters, + }); + + return ok(res, events); + } + + return methodNotAllowed(res); +}; diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts index 60195960..d1f124ea 100644 --- a/queries/analytics/event/saveEvent.ts +++ b/queries/analytics/event/saveEvent.ts @@ -4,6 +4,7 @@ import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; import { uuid } from 'lib/crypto'; import cache from 'lib/cache'; +import { saveEventData } from '../eventData/saveEventData'; export async function saveEvent(args: { id: string; @@ -12,6 +13,7 @@ export async function saveEvent(args: { urlQuery?: string; pageTitle?: string; eventName?: string; + eventData?: any; hostname?: string; browser?: string; os?: string; @@ -36,12 +38,15 @@ async function relationalQuery(data: { urlQuery?: string; pageTitle?: string; eventName?: string; + eventData?: any; }) { - const { websiteId, id: sessionId, urlPath, urlQuery, eventName, pageTitle } = data; + const { websiteId, id: sessionId, urlPath, urlQuery, eventName, eventData, pageTitle } = data; + const website = await cache.fetchWebsite(websiteId); + const websiteEventId = uuid(); - return prisma.client.websiteEvent.create({ + const websiteEvent = prisma.client.websiteEvent.create({ data: { - id: uuid(), + id: websiteEventId, websiteId, sessionId, urlPath: urlPath?.substring(0, URL_LENGTH), @@ -51,6 +56,20 @@ async function relationalQuery(data: { eventName: eventName?.substring(0, EVENT_NAME_LENGTH), }, }); + + if (eventData) { + await saveEventData({ + websiteId, + sessionId, + eventId: websiteEventId, + revId: website?.revId, + urlPath: urlPath?.substring(0, URL_LENGTH), + eventName: eventName?.substring(0, EVENT_NAME_LENGTH), + eventData, + }); + } + + return websiteEvent; } async function clickhouseQuery(data: { @@ -60,6 +79,7 @@ async function clickhouseQuery(data: { urlQuery?: string; pageTitle?: string; eventName?: string; + eventData?: any; hostname?: string; browser?: string; os?: string; @@ -78,6 +98,7 @@ async function clickhouseQuery(data: { urlQuery, pageTitle, eventName, + eventData, country, subdivision1, subdivision2, @@ -86,11 +107,13 @@ async function clickhouseQuery(data: { } = data; const { getDateFormat, sendMessage } = kafka; const website = await cache.fetchWebsite(websiteId); + const eventId = uuid(); + const createdAt = getDateFormat(new Date()); const message = { website_id: websiteId, session_id: sessionId, - event_id: uuid(), + event_id: eventId, country: country ? country : null, subdivision1: subdivision1 ? subdivision1 : null, subdivision2: subdivision2 ? subdivision2 : null, @@ -101,11 +124,24 @@ async function clickhouseQuery(data: { event_type: EVENT_TYPE.customEvent, event_name: eventName?.substring(0, EVENT_NAME_LENGTH), rev_id: website?.revId || 0, - created_at: getDateFormat(new Date()), + created_at: createdAt, ...args, }; await sendMessage(message, 'event'); + if (eventData) { + await saveEventData({ + websiteId, + sessionId, + eventId, + revId: website?.revId, + urlPath: urlPath?.substring(0, URL_LENGTH), + eventName: eventName?.substring(0, EVENT_NAME_LENGTH), + eventData, + createdAt, + }); + } + return data; } diff --git a/queries/analytics/eventData/getEventData.ts b/queries/analytics/eventData/getEventData.ts new file mode 100644 index 00000000..1c34a733 --- /dev/null +++ b/queries/analytics/eventData/getEventData.ts @@ -0,0 +1,117 @@ +import cache from 'lib/cache'; +import clickhouse from 'lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; +import { WebsiteEventDataMetric } from 'lib/types'; + +export async function getEventData( + ...args: [ + websiteId: string, + data: { + startDate: Date; + endDate: Date; + eventName: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, + ] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + data: { + startDate: Date; + endDate: Date; + timeSeries?: { + unit: string; + timezone: string; + }; + eventName: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, +) { + const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; + const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma; + const params: any = [websiteId, startDate, endDate, eventName || '']; + + return rawQuery( + `select + count(*) x + ${eventName ? `,event_name eventName` : ''} + ${urlPath ? `,url_path urlPath` : ''} + ${ + timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : '' + } + from event_data + ${ + eventName || urlPath + ? 'join website_event on event_data.id = website_event.website_event_id' + : '' + } + where website_id = $1${toUuid()} + and created_at between $2 and $3 + ${eventName ? `and eventName = $4` : ''} + ${getEventDataFilterQuery(filters, params)} + ${timeSeries ? 'group by t' : ''}`, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + data: { + startDate: Date; + endDate: Date; + timeSeries?: { + unit: string; + timezone: string; + }; + eventName?: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, +) { + const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; + const { rawQuery, getBetweenDates, getDateQuery, getEventDataFilterQuery } = clickhouse; + const website = await cache.fetchWebsite(websiteId); + const params = { websiteId, revId: website?.revId || 0 }; + + return rawQuery( + `select + count(*) x + ${eventName ? `,event_name eventName` : ''} + ${urlPath ? `,url_path urlPath` : ''} + ${ + timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : '' + } + from event_data + where website_id = {websiteId:UUID} + and rev_id = {revId:UInt32} + ${eventName ? `and eventName = ${eventName}` : ''} + and ${getBetweenDates('created_at', startDate, endDate)} + ${getEventDataFilterQuery(filters, params)} + ${timeSeries ? 'group by t' : ''}`, + params, + ); +} diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts new file mode 100644 index 00000000..c7ccb1b1 --- /dev/null +++ b/queries/analytics/eventData/saveEventData.ts @@ -0,0 +1,96 @@ +import { Prisma } from '@prisma/client'; +import { EVENT_DATA_TYPE } from 'lib/constants'; +import { uuid } from 'lib/crypto'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { flattenJSON } from 'lib/eventData'; +import kafka from 'lib/kafka'; +import prisma from 'lib/prisma'; +import { EventData } from 'lib/types'; + +export async function saveEventData(args: { + websiteId: string; + eventId: string; + sessionId?: string; + revId?: number; + urlPath?: string; + eventName?: string; + eventData: EventData; + createdAt?: string; +}) { + return runQuery({ + [PRISMA]: () => relationalQuery(args), + [CLICKHOUSE]: () => clickhouseQuery(args), + }); +} + +async function relationalQuery(data: { + websiteId: string; + eventId: string; + eventData: EventData; +}): Promise { + const { websiteId, eventId, eventData } = data; + + const jsonKeys = flattenJSON(eventData); + + //id, websiteEventId, eventStringValue + const flattendData = jsonKeys.map(a => ({ + id: uuid(), + websiteEventId: eventId, + websiteId, + eventKey: a.key, + eventStringValue: + a.eventDataType === EVENT_DATA_TYPE.string || + a.eventDataType === EVENT_DATA_TYPE.boolean || + a.eventDataType === EVENT_DATA_TYPE.array + ? a.value + : null, + eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null, + eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null, + eventDataType: a.eventDataType, + })); + + return prisma.client.eventData.createMany({ + data: flattendData, + }); +} + +async function clickhouseQuery(data: { + websiteId: string; + eventId: string; + sessionId?: string; + revId?: number; + urlPath?: string; + eventName?: string; + eventData: EventData; + createdAt?: string; +}) { + const { websiteId, sessionId, eventId, revId, urlPath, eventName, eventData, createdAt } = data; + + const { getDateFormat, sendMessages } = kafka; + + const jsonKeys = flattenJSON(eventData); + + const messages = jsonKeys.map(a => ({ + website_id: websiteId, + session_id: sessionId, + event_id: eventId, + rev_id: revId, + url_path: urlPath, + event_name: eventName, + event_key: a.key, + event_string_value: + a.eventDataType === EVENT_DATA_TYPE.string || + a.eventDataType === EVENT_DATA_TYPE.boolean || + a.eventDataType === EVENT_DATA_TYPE.array + ? a.value + : null, + event_numeric_value: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null, + event_date_value: a.eventDataType === EVENT_DATA_TYPE.date ? getDateFormat(a.value) : null, + event_data_type: a.eventDataType, + created_at: createdAt, + })); + + await sendMessages(messages, 'event_data'); + + return data; +} diff --git a/queries/analytics/pageview/savePageView.ts b/queries/analytics/pageview/savePageView.ts index e5c39ea7..1b0239ca 100644 --- a/queries/analytics/pageview/savePageView.ts +++ b/queries/analytics/pageview/savePageView.ts @@ -109,6 +109,7 @@ async function clickhouseQuery(data: { const message = { website_id: websiteId, session_id: sessionId, + event_id: uuid(), rev_id: website?.revId || 0, country: country ? country : null, subdivision1: subdivision1 ? subdivision1 : null, diff --git a/queries/index.js b/queries/index.js index e59413b9..d98f2609 100644 --- a/queries/index.js +++ b/queries/index.js @@ -4,6 +4,7 @@ export * from './admin/user'; export * from './admin/website'; export * from './analytics/event/getEventMetrics'; export * from './analytics/event/getEvents'; +export * from './analytics/eventData/getEventData'; export * from './analytics/event/saveEvent'; export * from './analytics/pageview/getPageviewMetrics'; export * from './analytics/pageview/getPageviews'; diff --git a/scripts/telemetry.js b/scripts/telemetry.js index 100e6864..67fe3202 100644 --- a/scripts/telemetry.js +++ b/scripts/telemetry.js @@ -5,7 +5,7 @@ const isCI = require('is-ci'); const pkg = require('../package.json'); const dest = path.resolve(__dirname, '../.next/cache/umami.json'); -const url = 'https://telemetry.umami.is/api/collect'; +const url = 'https://telemetry.umami.is/api/send'; async function sendTelemetry(action) { let json = {}; diff --git a/scripts/update-tracker.js b/scripts/update-tracker.js index fc8ad8b8..fdf920f0 100644 --- a/scripts/update-tracker.js +++ b/scripts/update-tracker.js @@ -12,7 +12,7 @@ if (endPoint) { fs.writeFileSync( path.resolve(file), - tracker.toString().replace(/"\/api\/collect"/g, `"${endPoint}"`), + tracker.toString().replace(/"\/api\/send"/g, `"${endPoint}"`), ); console.log(`Updated tracker endpoint: ${endPoint}.`); diff --git a/tracker/index.js b/tracker/index.js index 8f27ab36..f3c05f0e 100644 --- a/tracker/index.js +++ b/tracker/index.js @@ -73,7 +73,7 @@ let cache; if (currentRef.substring(0, 4) === 'http') { - if ((currentRef = currentRef.split('/')[2].split(':')[0] === hostname)) { + if (currentRef.split('/')[2].split(':')[0] === hostname) { currentRef = '/' + currentRef.split('/').splice(3).join('/'); } } @@ -118,6 +118,7 @@ const trackEvent = ( eventName, + eventData, url = currentUrl, websiteId = website, pageTitle = currentPageTitle, @@ -128,7 +129,8 @@ website: websiteId, url, pageTitle, - eventName: eventName, + eventName, + eventData, }), ); diff --git a/yarn.lock b/yarn.lock index 28d0d787..9616b887 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1403,6 +1403,13 @@ "@babel/helper-validator-identifier" "^7.18.6" to-fast-properties "^2.0.0" +"@cspotcode/source-map-support@^0.8.0": + version "0.8.1" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" + integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== + dependencies: + "@jridgewell/trace-mapping" "0.3.9" + "@csstools/postcss-cascade-layers@^1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@csstools/postcss-cascade-layers/-/postcss-cascade-layers-1.1.1.tgz#8a997edf97d34071dd2e37ea6022447dd9e795ad" @@ -1714,7 +1721,7 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.9" -"@jridgewell/resolve-uri@3.1.0": +"@jridgewell/resolve-uri@3.1.0", "@jridgewell/resolve-uri@^3.0.3": version "3.1.0" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== @@ -1737,6 +1744,14 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/trace-mapping@0.3.9": + version "0.3.9" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" + integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== + dependencies: + "@jridgewell/resolve-uri" "^3.0.3" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping@^0.3.9": version "0.3.17" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" @@ -2425,6 +2440,26 @@ resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== +"@tsconfig/node10@^1.0.7": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" + integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA== + +"@tsconfig/node12@^1.0.7": + version "1.0.11" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d" + integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag== + +"@tsconfig/node14@^1.0.0": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1" + integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== + +"@tsconfig/node16@^1.0.2": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.3.tgz#472eaab5f15c1ffdd7f8628bd4c4f753995ec79e" + integrity sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ== + "@types/babel__core@^7.1.7": version "7.1.18" resolved "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.18.tgz" @@ -2861,21 +2896,26 @@ acorn-jsx@^5.2.0, acorn-jsx@^5.3.2: resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== +acorn-walk@^8.1.1: + version "8.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + acorn@^6.4.1: version "6.4.2" resolved "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz" integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ== +acorn@^8.4.1, acorn@^8.8.0: + version "8.8.2" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" + integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== + acorn@^8.5.0: version "8.7.1" resolved "https://registry.npmjs.org/acorn/-/acorn-8.7.1.tgz" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== -acorn@^8.8.0: - version "8.8.2" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" - integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== - agent-base@6: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" @@ -2984,6 +3024,11 @@ arch@^2.2.0: resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== + argparse@^1.0.7: version "1.0.10" resolved "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz" @@ -3601,6 +3646,11 @@ cosmiconfig@^7.0.1, cosmiconfig@^7.1.0: path-type "^4.0.0" yaml "^1.10.0" +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + cross-env@^7.0.3: version "7.0.3" resolved "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz" @@ -3951,6 +4001,11 @@ detect-libc@^2.0.0, detect-libc@^2.0.1: resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.1.tgz#e1897aa88fa6ad197862937fbc0441ef352ee0cd" integrity sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w== +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -5898,6 +5953,11 @@ make-dir@^3.0.0: dependencies: semver "^6.0.0" +make-error@^1.1.1: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + map-obj@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" @@ -8260,6 +8320,25 @@ trim-newlines@^3.0.0: resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw== +ts-node@^10.9.1: + version "10.9.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" + integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== + dependencies: + "@cspotcode/source-map-support" "^0.8.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + v8-compile-cache-lib "^3.0.1" + yn "3.1.1" + tsconfig-paths@^3.14.1: version "3.14.1" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a" @@ -8553,6 +8632,11 @@ uuid@^8.3.0, uuid@^8.3.2: resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +v8-compile-cache-lib@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" + integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== + v8-compile-cache@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz" @@ -8748,6 +8832,11 @@ yargs-parser@^20.2.3: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== +yn@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" + integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"