From 7723576212683a8cfb2193f84594ba2996554d6d Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sun, 25 Feb 2024 23:14:45 -0800 Subject: [PATCH] Updated handling of date values for event data. --- src/lib/constants.ts | 2 + src/lib/data.ts | 47 ++++++++++++------- .../analytics/eventData/saveEventData.ts | 26 ++++------ .../analytics/sessions/saveSessionData.ts | 15 ++---- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 0773f439..f728837a 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -216,6 +216,8 @@ export const UUID_REGEX = export const HOSTNAME_REGEX = /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/; export const IP_REGEX = /^((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])(\.(?!$)|$)){4}$/; +export const DATETIME_REGEX = + /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3}(Z|\+[0-9]{2}:[0-9]{2})?)?$/; export const DESKTOP_SCREEN_WIDTH = 1920; export const LAPTOP_SCREEN_WIDTH = 1024; diff --git a/src/lib/data.ts b/src/lib/data.ts index 5b502c5d..24c76dd0 100644 --- a/src/lib/data.ts +++ b/src/lib/data.ts @@ -1,19 +1,18 @@ -import { isValid } from 'date-fns'; -import { DATA_TYPE } from './constants'; +import { DATA_TYPE, DATETIME_REGEX } from './constants'; import { DynamicDataType } from './types'; export function flattenJSON( eventData: { [key: string]: any }, - keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [], + keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [], parentKey = '', -): { key: string; value: any; dynamicDataType: DynamicDataType }[] { +): { key: string; value: any; dataType: DynamicDataType }[] { return Object.keys(eventData).reduce( (acc, key) => { const value = eventData[key]; const type = typeof eventData[key]; // nested object - if (value && type === 'object' && !Array.isArray(value) && !isValid(value)) { + if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) { flattenJSON(value, acc.keyValues, getKeyName(key, parentKey)); } else { createKey(getKeyName(key, parentKey), value, acc); @@ -25,48 +24,64 @@ export function flattenJSON( ).keyValues; } +export function isValidDateValue(value: string) { + return typeof value === 'string' && DATETIME_REGEX.test(value); +} + export function getDataType(value: any): string { let type: string = typeof value; - if ((type === 'string' && isValid(value)) || isValid(new Date(value))) { + if (isValidDateValue(value)) { type = 'date'; } return type; } -function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) { +export function getStringValue(value: string, dataType: number) { + if (dataType === DATA_TYPE.number) { + return parseFloat(value).toFixed(4); + } + + if (dataType === DATA_TYPE.date) { + return new Date(value).toISOString(); + } + + return value; +} + +function createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) { const type = getDataType(value); - let dynamicDataType = null; + let dataType = null; switch (type) { case 'number': - dynamicDataType = DATA_TYPE.number; + dataType = DATA_TYPE.number; break; case 'string': - dynamicDataType = DATA_TYPE.string; + dataType = DATA_TYPE.string; break; case 'boolean': - dynamicDataType = DATA_TYPE.boolean; + dataType = DATA_TYPE.boolean; value = value ? 'true' : 'false'; break; case 'date': - dynamicDataType = DATA_TYPE.date; + dataType = DATA_TYPE.date; break; case 'object': - dynamicDataType = DATA_TYPE.array; + dataType = DATA_TYPE.array; value = JSON.stringify(value); break; default: - dynamicDataType = DATA_TYPE.string; + dataType = DATA_TYPE.string; break; } - acc.keyValues.push({ key, value, dynamicDataType }); + acc.keyValues.push({ key, value, dataType }); } -function getKeyName(key, parentKey) { +function getKeyName(key: string, parentKey: string) { if (!parentKey) { return key; } diff --git a/src/queries/analytics/eventData/saveEventData.ts b/src/queries/analytics/eventData/saveEventData.ts index 0f1ddb37..345409ce 100644 --- a/src/queries/analytics/eventData/saveEventData.ts +++ b/src/queries/analytics/eventData/saveEventData.ts @@ -2,7 +2,7 @@ import { Prisma } from '@prisma/client'; import { DATA_TYPE } from 'lib/constants'; import { uuid } from 'lib/crypto'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import { flattenJSON } from 'lib/data'; +import { flattenJSON, getStringValue } from 'lib/data'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; import { DynamicData } from 'lib/types'; @@ -37,15 +37,10 @@ async function relationalQuery(data: { websiteEventId: eventId, websiteId, eventKey: a.key, - stringValue: - a.dynamicDataType === DATA_TYPE.number - ? parseFloat(a.value).toFixed(4) - : a.dynamicDataType === DATA_TYPE.date - ? a.value.split('.')[0] + 'Z' - : a.value.toString(), - numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, - dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, - dataType: a.dynamicDataType, + stringValue: getStringValue(a.value, a.dataType), + numberValue: a.dataType === DATA_TYPE.number ? a.value : null, + dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dataType, })); return prisma.client.eventData.createMany({ @@ -75,13 +70,10 @@ async function clickhouseQuery(data: { url_path: urlPath, event_name: eventName, event_key: a.key, - string_value: - a.dynamicDataType === DATA_TYPE.date - ? getDateFormat(a.value, 'isoUtcDateTime') - : a.value.toString(), - number_value: a.dynamicDataType === DATA_TYPE.number ? a.value : null, - date_value: a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value) : null, - data_type: a.dynamicDataType, + string_value: getStringValue(a.value, a.dataType), + number_value: a.dataType === DATA_TYPE.number ? a.value : null, + date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null, + data_type: a.dataType, created_at: createdAt, })); diff --git a/src/queries/analytics/sessions/saveSessionData.ts b/src/queries/analytics/sessions/saveSessionData.ts index ef32bcfb..b1c5759b 100644 --- a/src/queries/analytics/sessions/saveSessionData.ts +++ b/src/queries/analytics/sessions/saveSessionData.ts @@ -1,6 +1,6 @@ import { DATA_TYPE } from 'lib/constants'; import { uuid } from 'lib/crypto'; -import { flattenJSON } from 'lib/data'; +import { flattenJSON, getStringValue } from 'lib/data'; import prisma from 'lib/prisma'; import { DynamicData } from 'lib/types'; @@ -19,15 +19,10 @@ export async function saveSessionData(data: { websiteId, sessionId, key: a.key, - stringValue: - a.dynamicDataType === DATA_TYPE.number - ? parseFloat(a.value).toFixed(4) - : a.dynamicDataType === DATA_TYPE.date - ? a.value.split('.')[0] + 'Z' - : a.value.toString(), - numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, - dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, - dataType: a.dynamicDataType, + stringValue: getStringValue(a.value, a.dataType), + numberValue: a.dataType === DATA_TYPE.number ? a.value : null, + dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dataType, })); return transaction([