From 45f93eb4fbdb991735f88240dd1a0e702dc4b469 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 9 Sep 2024 13:09:09 -0700 Subject: [PATCH 1/3] move event stats to event-data folder --- .../api/websites/[websiteId]/{events => event-data}/stats.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/pages/api/websites/[websiteId]/{events => event-data}/stats.ts (100%) diff --git a/src/pages/api/websites/[websiteId]/events/stats.ts b/src/pages/api/websites/[websiteId]/event-data/stats.ts similarity index 100% rename from src/pages/api/websites/[websiteId]/events/stats.ts rename to src/pages/api/websites/[websiteId]/event-data/stats.ts From fa3710badcbc4ffb9fc388909034e45a8ecc4dcf Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Fri, 13 Sep 2024 08:31:52 -0700 Subject: [PATCH 2/3] add getEventDataFields --- .../websites/[websiteId]/event-data/fields.ts | 51 ++++++++++++++ .../analytics/events/getEventDataFields.ts | 69 +++++++++++++++++++ .../analytics/events/getEventDataValues.ts | 4 +- src/queries/index.ts | 1 + 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 src/pages/api/websites/[websiteId]/event-data/fields.ts create mode 100644 src/queries/analytics/events/getEventDataFields.ts diff --git a/src/pages/api/websites/[websiteId]/event-data/fields.ts b/src/pages/api/websites/[websiteId]/event-data/fields.ts new file mode 100644 index 00000000..c5075c5e --- /dev/null +++ b/src/pages/api/websites/[websiteId]/event-data/fields.ts @@ -0,0 +1,51 @@ +import { canViewWebsite } from 'lib/auth'; +import { useAuth, useCors, useValidate } from 'lib/middleware'; +import { NextApiRequestQueryBody } from 'lib/types'; +import { NextApiResponse } from 'next'; +import { methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { getEventDataFields } from 'queries'; + +import * as yup from 'yup'; + +export interface EventDataFieldsRequestQuery { + websiteId: string; + startAt: string; + endAt: string; +} + +const schema = { + GET: yup.object().shape({ + websiteId: yup.string().uuid().required(), + startAt: yup.number().integer().required(), + endAt: yup.number().integer().min(yup.ref('startAt')).required(), + }), +}; + +export default async ( + req: NextApiRequestQueryBody, + res: NextApiResponse, +) => { + await useCors(req, res); + await useAuth(req, res); + await useValidate(schema, req, res); + + if (req.method === 'GET') { + const { websiteId, startAt, endAt } = req.query; + + if (!(await canViewWebsite(req.auth, websiteId))) { + return unauthorized(res); + } + + const startDate = new Date(+startAt); + const endDate = new Date(+endAt); + + const data = await getEventDataFields(websiteId, { + startDate, + endDate, + }); + + return ok(res, data); + } + + return methodNotAllowed(res); +}; diff --git a/src/queries/analytics/events/getEventDataFields.ts b/src/queries/analytics/events/getEventDataFields.ts new file mode 100644 index 00000000..05fee072 --- /dev/null +++ b/src/queries/analytics/events/getEventDataFields.ts @@ -0,0 +1,69 @@ +import prisma from 'lib/prisma'; +import clickhouse from 'lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { QueryFilters, WebsiteEventData } from 'lib/types'; + +export async function getEventDataFields( + ...args: [websiteId: string, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { rawQuery, parseFilters, getDateSQL } = prisma; + const { filterQuery, params } = await parseFilters(websiteId, filters); + + return rawQuery( + ` + select + data_key as "propertyName", + data_type as "dataType", + case + when data_type = 2 then replace(string_value, '.0000', '') + when data_type = 4 then ${getDateSQL('date_value', 'hour')} + else string_value + end as "value", + count(*) as "total" + from event_data + join website_event on website_event.event_id = event_data.website_event_id + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + ${filterQuery} + group by data_key, data_type, value + order by 2 desc + limit 100 + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> { + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, filters); + + return rawQuery( + ` + select + data_key as propertyName, + data_type as dataType, + multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), + data_type = 4, toString(date_trunc('hour', date_value)), + string_value) as "value", + count(*) as "total" + from event_data + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + ${filterQuery} + group by data_key, data_type, value + order by 2 desc + limit 100 + `, + params, + ); +} diff --git a/src/queries/analytics/events/getEventDataValues.ts b/src/queries/analytics/events/getEventDataValues.ts index 07e42307..63101824 100644 --- a/src/queries/analytics/events/getEventDataValues.ts +++ b/src/queries/analytics/events/getEventDataValues.ts @@ -49,7 +49,7 @@ async function relationalQuery( async function clickhouseQuery( websiteId: string, filters: QueryFilters & { eventName?: string; propertyName?: string }, -): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> { +): Promise<{ value: string; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, filters); @@ -66,7 +66,7 @@ async function clickhouseQuery( and data_key = {propertyName:String} and event_name = {eventName:String} ${filterQuery} - group by event_name, value + group by value order by 2 desc limit 100 `, diff --git a/src/queries/index.ts b/src/queries/index.ts index a2697ced..8c7e564a 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -4,6 +4,7 @@ export * from 'queries/prisma/teamUser'; export * from 'queries/prisma/user'; export * from 'queries/prisma/website'; export * from './analytics/events/getEventDataEvents'; +export * from './analytics/events/getEventDataFields'; export * from './analytics/events/getEventDataProperties'; export * from './analytics/events/getEventDataValues'; export * from './analytics/events/getEventDataStats'; From fcb7fa2264663f8111129bb361043a11b7ad8a00 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 17 Sep 2024 19:59:58 -0700 Subject: [PATCH 3/3] fix reset website bug --- src/lib/clickhouse.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 5502b5ba..b588ec84 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -2,9 +2,9 @@ import { ClickHouseClient, createClient } from '@clickhouse/client'; import { formatInTimeZone } from 'date-fns-tz'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; +import { getWebsite } from 'queries/index'; import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants'; import { maxDate } from './date'; -import { fetchWebsite } from './load'; import { filtersToArray } from './params'; import { PageParams, QueryFilters, QueryOptions } from './types'; @@ -132,7 +132,7 @@ function getFilterParams(filters: QueryFilters = {}) { } async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) { - const website = await fetchWebsite(websiteId); + const website = await getWebsite(websiteId); return { filterQuery: getFilterQuery(filters, options),