From e92d958b24112ff746b6eababc37a4b90ae9b09b Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 5 May 2023 21:16:13 -0700 Subject: [PATCH 1/2] Updated redis client. Soft deletes reset username. --- lib/cache.ts | 30 +----------------------------- package.json | 2 +- pages/api/users/[id]/index.ts | 2 +- pages/api/websites/[id]/index.ts | 4 ++-- queries/admin/user.ts | 2 ++ yarn.lock | 8 ++++---- 6 files changed, 11 insertions(+), 37 deletions(-) diff --git a/lib/cache.ts b/lib/cache.ts index 2aad7ed8..528dd012 100644 --- a/lib/cache.ts +++ b/lib/cache.ts @@ -2,35 +2,7 @@ import { User, Website } from '@prisma/client'; import redis from '@umami/redis-client'; import { getSession, getUser, getWebsite } from '../queries'; -const DELETED = 'DELETED'; - -async function fetchObject(key, query) { - const obj = await redis.get(key); - - if (obj === DELETED) { - return null; - } - - if (!obj) { - return query().then(async data => { - if (data) { - await redis.set(key, data); - } - - return data; - }); - } - - return obj; -} - -async function storeObject(key, data) { - return redis.set(key, data); -} - -async function deleteObject(key, soft = false) { - return soft ? redis.set(key, DELETED) : redis.del(key); -} +const { fetchObject, storeObject, deleteObject } = redis; async function fetchWebsite(id): Promise { return fetchObject(`website:${id}`, () => getWebsite({ id })); diff --git a/package.json b/package.json index 736ec659..0db07f49 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "@prisma/client": "4.13.0", "@tanstack/react-query": "^4.16.1", "@umami/prisma-client": "^0.2.0", - "@umami/redis-client": "^0.2.0", + "@umami/redis-client": "^0.5.0", "chalk": "^4.1.1", "chart.js": "^4.2.1", "chartjs-adapter-date-fns": "^3.0.0", diff --git a/pages/api/users/[id]/index.ts b/pages/api/users/[id]/index.ts index 8219c4a7..de4642cb 100644 --- a/pages/api/users/[id]/index.ts +++ b/pages/api/users/[id]/index.ts @@ -51,11 +51,11 @@ export default async ( data.password = hashPassword(password); } + // Only admin can change these fields if (role && isAdmin) { data.role = role; } - // Only admin can change these fields if (username && isAdmin) { data.username = username; } diff --git a/pages/api/websites/[id]/index.ts b/pages/api/websites/[id]/index.ts index c1907fce..3f660a91 100644 --- a/pages/api/websites/[id]/index.ts +++ b/pages/api/websites/[id]/index.ts @@ -1,8 +1,8 @@ +import { NextApiResponse } from 'next'; +import { methodNotAllowed, ok, serverError, unauthorized } from 'next-basics'; import { Website, NextApiRequestQueryBody } from 'lib/types'; import { canViewWebsite, canUpdateWebsite, canDeleteWebsite } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; -import { NextApiResponse } from 'next'; -import { methodNotAllowed, ok, serverError, unauthorized } from 'next-basics'; import { deleteWebsite, getWebsite, updateWebsite } from 'queries'; export interface WebsiteRequestQuery { diff --git a/queries/admin/user.ts b/queries/admin/user.ts index 412c7785..a81a76ef 100644 --- a/queries/admin/user.ts +++ b/queries/admin/user.ts @@ -1,4 +1,5 @@ import { Prisma, Team, TeamUser } from '@prisma/client'; +import { getRandomChars } from 'next-basics'; import cache from 'lib/cache'; import { ROLES } from 'lib/constants'; import prisma from 'lib/prisma'; @@ -222,6 +223,7 @@ export async function deleteUser( cloudMode ? client.user.update({ data: { + username: getRandomChars(32), deletedAt: new Date(), }, where: { diff --git a/yarn.lock b/yarn.lock index d98ada11..41cca434 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3272,10 +3272,10 @@ dependencies: debug "^4.3.4" -"@umami/redis-client@^0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@umami/redis-client/-/redis-client-0.2.0.tgz#bdb1cd8b5c99afc5230621f19296c6d3559d68af" - integrity sha512-TONWhkuC//K2hRo3Psk7FHsuvu3XkQIYMY62/CERPtlIJz4Ac7DqsmYw4jO9/RkljA9XLl/5u+OggD4ARhMV8A== +"@umami/redis-client@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@umami/redis-client/-/redis-client-0.5.0.tgz#09b15458001bc172fc856d65316efbe5ff749461" + integrity sha512-x7wx/pMjyg3AAYzgjGOw031bNhyZ81h6tRMAl60RQQI9xlJaJEA1r0TEUrWfFi21gHAvdBLJGYCsvHzpix4LKQ== dependencies: debug "^4.3.4" redis "^4.5.1" From d827bf1417dd989c314f0253b1bafe55c0ccf6d6 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Sun, 7 May 2023 23:13:40 -0700 Subject: [PATCH 2/2] update CH kafka engine tables for error handling --- db/clickhouse/schema.sql | 42 +++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 77176413..8f48b434 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -58,7 +58,10 @@ CREATE TABLE umami.website_event_queue ( --event event_type UInt32, event_name String, - created_at DateTime('UTC') + created_at DateTime('UTC'), + --virtual columns + _error String, + _raw_message String ) ENGINE = Kafka SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list @@ -66,7 +69,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro kafka_group_name = 'event_consumer_group', kafka_format = 'JSONEachRow', kafka_max_block_size = 1048576, - kafka_skip_broken_messages = 100; + kafka_handle_error_mode = 'stream' CREATE MATERIALIZED VIEW umami.website_event_queue_mv TO umami.website_event AS SELECT website_id, @@ -93,6 +96,19 @@ SELECT website_id, created_at FROM umami.website_event_queue; +CREATE MATERIALIZED VIEW umami.website_event_errors_mv +( + error String, + raw String +) +ENGINE = MergeTree +ORDER BY (error, raw) +SETTINGS index_granularity = 8192 AS +SELECT _error AS error, + _raw_message AS raw +FROM umami.website_event_queue +WHERE length(_error) > 0 + CREATE TABLE umami.event_data ( website_id UUID, @@ -122,7 +138,10 @@ CREATE TABLE umami.event_data_queue ( event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 event_date_value Nullable(DateTime('UTC')), event_data_type UInt32, - created_at DateTime('UTC') + created_at DateTime('UTC'), + --virtual columns + _error String, + _raw_message String ) ENGINE = Kafka SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list @@ -130,7 +149,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro kafka_group_name = 'event_data_consumer_group', kafka_format = 'JSONEachRow', kafka_max_block_size = 1048576, - kafka_skip_broken_messages = 100; + kafka_handle_error_mode = 'stream' CREATE MATERIALIZED VIEW umami.event_data_queue_mv TO umami.event_data AS SELECT website_id, @@ -144,4 +163,17 @@ SELECT website_id, event_date_value, event_data_type, created_at -FROM umami.event_data_queue; \ No newline at end of file +FROM umami.event_data_queue; + +CREATE MATERIALIZED VIEW umami.event_data_errors_mv +( + error String, + raw String +) +ENGINE = MergeTree +ORDER BY (error, raw) +SETTINGS index_granularity = 8192 AS +SELECT _error AS error, + _raw_message AS raw +FROM umami.event_data_queue +WHERE length(_error) > 0 \ No newline at end of file