redis checkpoint

lib/redis.js

@@ -0,0 +1,59 @@
+import { createClient } from 'redis';
+import { startOfMonth } from 'date-fns';
+import { getSessions, getAllWebsites } from '/queries';
+
+async function getClient() {
+  const redis = new createClient({
+    url: process.env.REDIS_URL,
+  });
+
+  await redis.connect();
+
+  if (process.env.LOG_QUERY) {
+    redis.on('error', err => console.log('Redis Client Error', err));
+  }
+
+  return redis;
+}
+
+let redis = null;
+
+(async () => {
+  redis = global.redis || (await getClient());
+
+  if (process.env.NODE_ENV !== 'production') {
+    global.redis = redis;
+  }
+
+  const value = await redis.get('initialized');
+
+  if (!value) {
+    await stageData();
+  }
+})();
+
+export default redis;
+
+async function stageData() {
+  const sessions = await getSessions([], startOfMonth(new Date()).toUTCString());
+  const websites = await getAllWebsites();
+
+  const sessionUuids = sessions.map(a => {
+    return { key: `session:${a.session_uuid}`, value: '' };
+  });
+  const websiteIds = websites.map(a => {
+    return { key: `website:${a.website_uuid}`, value: Number(a.website_id) };
+  });
+
+  await addRedis(sessionUuids);
+  await addRedis(websiteIds);
+
+  await redis.set('initialized', 'initialized');
+}
+
+async function addRedis(ids) {
+  for (let i = 0; i < ids.length; i++) {
+    const { key, value } = ids[i];
+    await redis.set(key, value);
+  }
+}

lib/relational.js

@@ -23,7 +23,7 @@ function logQuery(e) {
   console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
 }
 
-function getPrismaClient(options) {
+function getClient(options) {
   const prisma = new PrismaClient(options);
 
   if (process.env.LOG_QUERY) {
@@ -32,7 +32,7 @@ function getPrismaClient(options) {
 
   return prisma;
 }
-const prisma = global.prisma || getPrismaClient(options);
+const prisma = global.prisma || getClient(options);
 
 if (process.env.NODE_ENV !== 'production') {
   global.prisma = prisma;

package.json

@@ -97,6 +97,7 @@
     "react-tooltip": "^4.2.21",
     "react-use-measure": "^2.0.4",
     "react-window": "^1.8.6",
+    "redis": "^4.3.0",
     "request-ip": "^3.3.0",
     "semver": "^7.3.6",
     "thenby": "^1.3.4",

pages/api/collect.js

@@ -12,7 +12,7 @@ import { uuid } from 'lib/crypto';
 export default async (req, res) => {
   await useCors(req, res);
 
-  if (isbot(req.headers['user-agent'])) {
+  if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) {
     return unauthorized(res);
   }
 

pages/api/heartbeat.js

@@ -1,5 +1,16 @@
+import { redis } from 'lib/redis';
 import { ok } from 'lib/response';
 
 export default async (req, res) => {
-  return ok(res, 'nice');
+  console.log('------------------------------------------------------------------------------');
+  // const redis = new createClient({
+  //   url: process.env.REDIS_URL,
+  // });
+
+  //console.log(redis.get);
+
+  const value = await redis.get('session:77c0b9de-677a-5268-8543-6018f0776a81');
+  console.log('complete');
+
+  return ok(res, value);
 };

queries/admin/website/createWebsite.js

@@ -1,4 +1,5 @@
 import { prisma, runQuery } from 'lib/relational';
+import redis from 'lib/redis';
 
 export async function createWebsite(user_id, data) {
   return runQuery(
@@ -12,5 +13,11 @@ export async function createWebsite(user_id, data) {
         ...data,
       },
     }),
-  );
+  ).then(async res => {
+    if (process.env.REDIS_URL) {
+      await redis.set(`website:${res.website_uuid}`, Number(res.website_id));
+    }
+
+    return res;
+  });
 }

queries/admin/website/deleteWebsite.js

@@ -1,23 +1,30 @@
 import { prisma, runQuery } from 'lib/relational';
+import redis from 'lib/redis';
 
 export async function deleteWebsite(website_id) {
   return runQuery(
-    prisma.$transaction([
+    prisma
-      prisma.pageview.deleteMany({
+      .$transaction([
-        where: { session: { website: { website_id } } },
+        prisma.pageview.deleteMany({
+          where: { session: { website: { website_id } } },
+        }),
+        prisma.event_data.deleteMany({
+          where: { event: { session: { website: { website_id } } } },
+        }),
+        prisma.event.deleteMany({
+          where: { session: { website: { website_id } } },
+        }),
+        prisma.session.deleteMany({
+          where: { website: { website_id } },
+        }),
+        prisma.website.delete({
+          where: { website_id },
+        }),
+      ])
+      .then(async res => {
+        if (process.env.REDIS_URL) {
+          await redis.del(`website:${res.website_uuid}`);
+        }
       }),
-      prisma.event_data.deleteMany({
-        where: { event: { session: { website: { website_id } } } },
-      }),
-      prisma.event.deleteMany({
-        where: { session: { website: { website_id } } },
-      }),
-      prisma.session.deleteMany({
-        where: { website: { website_id } },
-      }),
-      prisma.website.delete({
-        where: { website_id },
-      }),
-    ]),
   );
 }

queries/analytics/session/createSession.js

@@ -1,8 +1,9 @@
-import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';
-import { prisma, runQuery } from 'lib/relational';
 import clickhouse from 'lib/clickhouse';
-import kafka from 'lib/kafka';
+import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';
 import { runAnalyticsQuery } from 'lib/db';
+import kafka from 'lib/kafka';
+import redis from 'lib/redis';
+import { prisma, runQuery } from 'lib/relational';
 
 export async function createSession(...args) {
   return runAnalyticsQuery({
@@ -23,7 +24,13 @@ async function relationalQuery(website_id, data) {
         session_id: true,
       },
     }),
-  );
+  ).then(async res => {
+    if (process.env.REDIS_URL) {
+      await redis.set(`session:${res.session_uuid}`, '');
+    }
+
+    return res;
+  });
 }
 
 async function clickhouseQuery(
@@ -67,4 +74,6 @@ async function kafkaQuery(
   };
 
   await kafka.sendKafkaMessage(params, 'session');
+
+  await redis.set(`session:${session_uuid}`, '');
 }

queries/analytics/session/getSessions.js

@@ -14,11 +14,15 @@ async function relationalQuery(websites, start_at) {
   return runQuery(
     prisma.session.findMany({
       where: {
-        website: {
+        ...(websites && websites.length > 0
-          website_id: {
+          ? {
-            in: websites,
+              website: {
-          },
+                website_id: {
-        },
+                  in: websites,
+                },
+              },
+            }
+          : {}),
         created_at: {
           gte: start_at,
         },
@@ -31,7 +35,6 @@ async function clickhouseQuery(websites, start_at) {
   return clickhouse.rawQuery(
     `
     select
-      session_id,
       session_uuid,
       website_id,
       created_at,
@@ -43,8 +46,8 @@ async function clickhouseQuery(websites, start_at) {
       "language",
       country
     from session
-    where website_id in (${websites.join[',']}
+    where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'}
-      and created_at >= ${clickhouse.getDateFormat(start_at)})
+      and created_at >= ${clickhouse.getDateFormat(start_at)}
     `,
   );
 }

yarn.lock

@@ -1512,6 +1512,40 @@
     "@react-spring/shared" "~9.5.0"
     "@react-spring/types" "~9.5.0"
 
+"@redis/bloom@1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf"
+  integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==
+
+"@redis/client@1.3.0":
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77"
+  integrity sha512-XCFV60nloXAefDsPnYMjHGtvbtHR8fV5Om8cQ0JYqTNbWcQo/4AryzJ2luRj4blveWazRK/j40gES8M7Cp6cfQ==
+  dependencies:
+    cluster-key-slot "1.1.0"
+    generic-pool "3.8.2"
+    yallist "4.0.0"
+
+"@redis/graph@1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99"
+  integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==
+
+"@redis/json@1.0.3":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.3.tgz#a13fde1d22ebff0ae2805cd8e1e70522b08ea866"
+  integrity sha512-4X0Qv0BzD9Zlb0edkUoau5c1bInWSICqXAGrpwEltkncUwcxJIGEcVryZhLgb0p/3PkKaLIWkjhHRtLe9yiA7Q==
+
+"@redis/search@1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df"
+  integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==
+
+"@redis/time-series@1.0.3":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4"
+  integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==
+
 "@rollup/plugin-buble@^0.21.3":
   version "0.21.3"
   resolved "https://registry.npmjs.org/@rollup/plugin-buble/-/plugin-buble-0.21.3.tgz"
@@ -2458,6 +2492,11 @@ clone-regexp@^2.1.0:
   dependencies:
     is-regexp "^2.0.0"
 
+cluster-key-slot@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
+  integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
+
 color-convert@^1.9.0, color-convert@^1.9.3:
   version "1.9.3"
   resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz"
@@ -3573,6 +3612,11 @@ functions-have-names@^1.2.2:
   resolved "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz"
   integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==
 
+generic-pool@3.8.2:
+  version "3.8.2"
+  resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
+  integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
+
 gensync@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz"
@@ -5564,6 +5608,18 @@ redent@^3.0.0:
     indent-string "^4.0.0"
     strip-indent "^3.0.0"
 
+redis@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/redis/-/redis-4.3.0.tgz#c01de4cf13821406addb7fcb70afe203266c4c39"
+  integrity sha512-RXRUor0iU1vizu4viHoUyLpe1ZO/RngZp0V9DyXBHTI+7tC7rEz6Wzn4Sv9v0tTJeqGAzdJ+q5YVbNKKQ5hX9A==
+  dependencies:
+    "@redis/bloom" "1.0.2"
+    "@redis/client" "1.3.0"
+    "@redis/graph" "1.0.1"
+    "@redis/json" "1.0.3"
+    "@redis/search" "1.1.0"
+    "@redis/time-series" "1.0.3"
+
 redux@^4.0.0, redux@^4.0.4:
   version "4.2.0"
   resolved "https://registry.npmjs.org/redux/-/redux-4.2.0.tgz"
@@ -6711,7 +6767,7 @@ write-json-file@^4.3.0:
     sort-keys "^4.0.0"
     write-file-atomic "^3.0.0"
 
-yallist@^4.0.0:
+yallist@4.0.0, yallist@^4.0.0:
   version "4.0.0"
   resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz"
   integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==