Merge branch 'dev' into feat/um-285-report-schema

db/clickhouse/schema.sql

@@ -58,7 +58,10 @@ CREATE TABLE umami.website_event_queue (
     --event
     event_type UInt32,
     event_name String,
-    created_at DateTime('UTC')
+    created_at DateTime('UTC'),
+    --virtual columns
+    _error String,
+    _raw_message String
 )
 ENGINE = Kafka
 SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list
@@ -66,7 +69,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro
        kafka_group_name = 'event_consumer_group',
        kafka_format = 'JSONEachRow',
        kafka_max_block_size = 1048576,
-       kafka_skip_broken_messages = 100;
+       kafka_handle_error_mode = 'stream'
 
 CREATE MATERIALIZED VIEW umami.website_event_queue_mv TO umami.website_event AS
 SELECT website_id,
@@ -93,6 +96,19 @@ SELECT website_id,
     created_at
 FROM umami.website_event_queue;
 
+CREATE MATERIALIZED VIEW umami.website_event_errors_mv
+(
+    error String,
+    raw String
+)
+ENGINE = MergeTree
+ORDER BY (error, raw)
+SETTINGS index_granularity = 8192 AS
+SELECT _error AS error,
+    _raw_message AS raw
+FROM umami.website_event_queue
+WHERE length(_error) > 0
+
 CREATE TABLE umami.event_data
 (
     website_id UUID,
@@ -122,7 +138,10 @@ CREATE TABLE umami.event_data_queue (
     event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
     event_date_value Nullable(DateTime('UTC')),
     event_data_type UInt32,
-    created_at DateTime('UTC')
+    created_at DateTime('UTC'),
+    --virtual columns
+    _error String,
+    _raw_message String
 )
 ENGINE = Kafka
 SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list
@@ -130,7 +149,7 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro
        kafka_group_name = 'event_data_consumer_group',
        kafka_format = 'JSONEachRow',
        kafka_max_block_size = 1048576,
-       kafka_skip_broken_messages = 100;
+       kafka_handle_error_mode = 'stream'
 
 CREATE MATERIALIZED VIEW umami.event_data_queue_mv TO umami.event_data AS
 SELECT website_id,
@@ -144,4 +163,17 @@ SELECT website_id,
     event_date_value,
     event_data_type,
     created_at
-FROM umami.event_data_queue;
+FROM umami.event_data_queue;
+
+CREATE MATERIALIZED VIEW umami.event_data_errors_mv
+(
+    error String,
+    raw String
+)
+ENGINE = MergeTree
+ORDER BY (error, raw)
+SETTINGS index_granularity = 8192 AS
+SELECT _error AS error,
+    _raw_message AS raw
+FROM umami.event_data_queue
+WHERE length(_error) > 0

lib/cache.ts

@@ -2,35 +2,7 @@ import { User, Website } from '@prisma/client';
 import redis from '@umami/redis-client';
 import { getSession, getUser, getWebsite } from '../queries';
 
-const DELETED = 'DELETED';
-
-async function fetchObject(key, query) {
-  const obj = await redis.get(key);
-
-  if (obj === DELETED) {
-    return null;
-  }
-
-  if (!obj) {
-    return query().then(async data => {
-      if (data) {
-        await redis.set(key, data);
-      }
-
-      return data;
-    });
-  }
-
-  return obj;
-}
-
-async function storeObject(key, data) {
-  return redis.set(key, data);
-}
-
-async function deleteObject(key, soft = false) {
-  return soft ? redis.set(key, DELETED) : redis.del(key);
-}
+const { fetchObject, storeObject, deleteObject } = redis;
 
 async function fetchWebsite(id): Promise<Website> {
   return fetchObject(`website:${id}`, () => getWebsite({ id }));

package.json

@@ -62,7 +62,7 @@
     "@prisma/client": "4.13.0",
     "@tanstack/react-query": "^4.16.1",
     "@umami/prisma-client": "^0.2.0",
-    "@umami/redis-client": "^0.2.0",
+    "@umami/redis-client": "^0.5.0",
     "chalk": "^4.1.1",
     "chart.js": "^4.2.1",
     "chartjs-adapter-date-fns": "^3.0.0",

pages/api/users/[id]/index.ts

@@ -51,11 +51,11 @@ export default async (
       data.password = hashPassword(password);
     }
 
+    // Only admin can change these fields
     if (role && isAdmin) {
       data.role = role;
     }
 
-    // Only admin can change these fields
     if (username && isAdmin) {
       data.username = username;
     }

pages/api/websites/[id]/index.ts

@@ -1,8 +1,8 @@
+import { NextApiResponse } from 'next';
+import { methodNotAllowed, ok, serverError, unauthorized } from 'next-basics';
 import { Website, NextApiRequestQueryBody } from 'lib/types';
 import { canViewWebsite, canUpdateWebsite, canDeleteWebsite } from 'lib/auth';
 import { useAuth, useCors } from 'lib/middleware';
-import { NextApiResponse } from 'next';
-import { methodNotAllowed, ok, serverError, unauthorized } from 'next-basics';
 import { deleteWebsite, getWebsite, updateWebsite } from 'queries';
 
 export interface WebsiteRequestQuery {

queries/admin/user.ts

@@ -1,4 +1,5 @@
 import { Prisma, Team, TeamUser } from '@prisma/client';
+import { getRandomChars } from 'next-basics';
 import cache from 'lib/cache';
 import { ROLES } from 'lib/constants';
 import prisma from 'lib/prisma';
@@ -222,6 +223,7 @@ export async function deleteUser(
       cloudMode
         ? client.user.update({
             data: {
+              username: getRandomChars(32),
               deletedAt: new Date(),
             },
             where: {

yarn.lock

@@ -3272,10 +3272,10 @@
   dependencies:
     debug "^4.3.4"
 
-"@umami/redis-client@^0.2.0":
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/@umami/redis-client/-/redis-client-0.2.0.tgz#bdb1cd8b5c99afc5230621f19296c6d3559d68af"
-  integrity sha512-TONWhkuC//K2hRo3Psk7FHsuvu3XkQIYMY62/CERPtlIJz4Ac7DqsmYw4jO9/RkljA9XLl/5u+OggD4ARhMV8A==
+"@umami/redis-client@^0.5.0":
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/@umami/redis-client/-/redis-client-0.5.0.tgz#09b15458001bc172fc856d65316efbe5ff749461"
+  integrity sha512-x7wx/pMjyg3AAYzgjGOw031bNhyZ81h6tRMAl60RQQI9xlJaJEA1r0TEUrWfFi21gHAvdBLJGYCsvHzpix4LKQ==
   dependencies:
     debug "^4.3.4"
     redis "^4.5.1"