diff --git a/lib/redis.js b/lib/redis.js new file mode 100644 index 00000000..97011bed --- /dev/null +++ b/lib/redis.js @@ -0,0 +1,59 @@ +import { createClient } from 'redis'; +import { startOfMonth } from 'date-fns'; +import { getSessions, getAllWebsites } from '/queries'; + +async function getClient() { + const redis = new createClient({ + url: process.env.REDIS_URL, + }); + + await redis.connect(); + + if (process.env.LOG_QUERY) { + redis.on('error', err => console.log('Redis Client Error', err)); + } + + return redis; +} + +let redis = null; + +(async () => { + redis = global.redis || (await getClient()); + + if (process.env.NODE_ENV !== 'production') { + global.redis = redis; + } + + const value = await redis.get('initialized'); + + if (!value) { + await stageData(); + } +})(); + +export default redis; + +async function stageData() { + const sessions = await getSessions([], startOfMonth(new Date()).toUTCString()); + const websites = await getAllWebsites(); + + const sessionUuids = sessions.map(a => { + return { key: `session:${a.session_uuid}`, value: '' }; + }); + const websiteIds = websites.map(a => { + return { key: `website:${a.website_uuid}`, value: Number(a.website_id) }; + }); + + await addRedis(sessionUuids); + await addRedis(websiteIds); + + await redis.set('initialized', 'initialized'); +} + +async function addRedis(ids) { + for (let i = 0; i < ids.length; i++) { + const { key, value } = ids[i]; + await redis.set(key, value); + } +} diff --git a/lib/relational.js b/lib/relational.js index 7d1ed8c2..28de7531 100644 --- a/lib/relational.js +++ b/lib/relational.js @@ -23,7 +23,7 @@ function logQuery(e) { console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`)); } -function getPrismaClient(options) { +function getClient(options) { const prisma = new PrismaClient(options); if (process.env.LOG_QUERY) { @@ -32,7 +32,7 @@ function getPrismaClient(options) { return prisma; } -const prisma = global.prisma || getPrismaClient(options); +const prisma = global.prisma || getClient(options); if (process.env.NODE_ENV !== 'production') { global.prisma = prisma; diff --git a/package.json b/package.json index 6142816a..dfba6613 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "react-tooltip": "^4.2.21", "react-use-measure": "^2.0.4", "react-window": "^1.8.6", + "redis": "^4.3.0", "request-ip": "^3.3.0", "semver": "^7.3.6", "thenby": "^1.3.4", diff --git a/pages/api/collect.js b/pages/api/collect.js index 472af421..c17dd761 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -12,7 +12,7 @@ import { uuid } from 'lib/crypto'; export default async (req, res) => { await useCors(req, res); - if (isbot(req.headers['user-agent'])) { + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { return unauthorized(res); } diff --git a/pages/api/heartbeat.js b/pages/api/heartbeat.js index a53213ad..6218d179 100644 --- a/pages/api/heartbeat.js +++ b/pages/api/heartbeat.js @@ -1,5 +1,16 @@ +import { redis } from 'lib/redis'; import { ok } from 'lib/response'; export default async (req, res) => { - return ok(res, 'nice'); + console.log('------------------------------------------------------------------------------'); + // const redis = new createClient({ + // url: process.env.REDIS_URL, + // }); + + //console.log(redis.get); + + const value = await redis.get('session:77c0b9de-677a-5268-8543-6018f0776a81'); + console.log('complete'); + + return ok(res, value); }; diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.js index 032a986a..7a589db6 100644 --- a/queries/admin/website/createWebsite.js +++ b/queries/admin/website/createWebsite.js @@ -1,4 +1,5 @@ import { prisma, runQuery } from 'lib/relational'; +import redis from 'lib/redis'; export async function createWebsite(user_id, data) { return runQuery( @@ -12,5 +13,11 @@ export async function createWebsite(user_id, data) { ...data, }, }), - ); + ).then(async res => { + if (process.env.REDIS_URL) { + await redis.set(`website:${res.website_uuid}`, Number(res.website_id)); + } + + return res; + }); } diff --git a/queries/admin/website/deleteWebsite.js b/queries/admin/website/deleteWebsite.js index 58dc453b..13c3160d 100644 --- a/queries/admin/website/deleteWebsite.js +++ b/queries/admin/website/deleteWebsite.js @@ -1,23 +1,30 @@ import { prisma, runQuery } from 'lib/relational'; +import redis from 'lib/redis'; export async function deleteWebsite(website_id) { return runQuery( - prisma.$transaction([ - prisma.pageview.deleteMany({ - where: { session: { website: { website_id } } }, + prisma + .$transaction([ + prisma.pageview.deleteMany({ + where: { session: { website: { website_id } } }, + }), + prisma.event_data.deleteMany({ + where: { event: { session: { website: { website_id } } } }, + }), + prisma.event.deleteMany({ + where: { session: { website: { website_id } } }, + }), + prisma.session.deleteMany({ + where: { website: { website_id } }, + }), + prisma.website.delete({ + where: { website_id }, + }), + ]) + .then(async res => { + if (process.env.REDIS_URL) { + await redis.del(`website:${res.website_uuid}`); + } }), - prisma.event_data.deleteMany({ - where: { event: { session: { website: { website_id } } } }, - }), - prisma.event.deleteMany({ - where: { session: { website: { website_id } } }, - }), - prisma.session.deleteMany({ - where: { website: { website_id } }, - }), - prisma.website.delete({ - where: { website_id }, - }), - ]), ); } diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index 676aeb48..4cf48566 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -1,8 +1,9 @@ -import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants'; -import { prisma, runQuery } from 'lib/relational'; import clickhouse from 'lib/clickhouse'; -import kafka from 'lib/kafka'; +import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants'; import { runAnalyticsQuery } from 'lib/db'; +import kafka from 'lib/kafka'; +import redis from 'lib/redis'; +import { prisma, runQuery } from 'lib/relational'; export async function createSession(...args) { return runAnalyticsQuery({ @@ -23,7 +24,13 @@ async function relationalQuery(website_id, data) { session_id: true, }, }), - ); + ).then(async res => { + if (process.env.REDIS_URL) { + await redis.set(`session:${res.session_uuid}`, ''); + } + + return res; + }); } async function clickhouseQuery( @@ -67,4 +74,6 @@ async function kafkaQuery( }; await kafka.sendKafkaMessage(params, 'session'); + + await redis.set(`session:${session_uuid}`, ''); } diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js index 3fcd1552..bcb0f19e 100644 --- a/queries/analytics/session/getSessions.js +++ b/queries/analytics/session/getSessions.js @@ -14,11 +14,15 @@ async function relationalQuery(websites, start_at) { return runQuery( prisma.session.findMany({ where: { - website: { - website_id: { - in: websites, - }, - }, + ...(websites && websites.length > 0 + ? { + website: { + website_id: { + in: websites, + }, + }, + } + : {}), created_at: { gte: start_at, }, @@ -31,7 +35,6 @@ async function clickhouseQuery(websites, start_at) { return clickhouse.rawQuery( ` select - session_id, session_uuid, website_id, created_at, @@ -43,8 +46,8 @@ async function clickhouseQuery(websites, start_at) { "language", country from session - where website_id in (${websites.join[',']} - and created_at >= ${clickhouse.getDateFormat(start_at)}) + where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'} + and created_at >= ${clickhouse.getDateFormat(start_at)} `, ); } diff --git a/yarn.lock b/yarn.lock index 98144f5e..4b96ba8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1512,6 +1512,40 @@ "@react-spring/shared" "~9.5.0" "@react-spring/types" "~9.5.0" +"@redis/bloom@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf" + integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw== + +"@redis/client@1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77" + integrity sha512-XCFV60nloXAefDsPnYMjHGtvbtHR8fV5Om8cQ0JYqTNbWcQo/4AryzJ2luRj4blveWazRK/j40gES8M7Cp6cfQ== + dependencies: + cluster-key-slot "1.1.0" + generic-pool "3.8.2" + yallist "4.0.0" + +"@redis/graph@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99" + integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ== + +"@redis/json@1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.3.tgz#a13fde1d22ebff0ae2805cd8e1e70522b08ea866" + integrity sha512-4X0Qv0BzD9Zlb0edkUoau5c1bInWSICqXAGrpwEltkncUwcxJIGEcVryZhLgb0p/3PkKaLIWkjhHRtLe9yiA7Q== + +"@redis/search@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df" + integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ== + +"@redis/time-series@1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" + integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== + "@rollup/plugin-buble@^0.21.3": version "0.21.3" resolved "https://registry.npmjs.org/@rollup/plugin-buble/-/plugin-buble-0.21.3.tgz" @@ -2458,6 +2492,11 @@ clone-regexp@^2.1.0: dependencies: is-regexp "^2.0.0" +cluster-key-slot@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" + integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== + color-convert@^1.9.0, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" @@ -3573,6 +3612,11 @@ functions-have-names@^1.2.2: resolved "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== +generic-pool@3.8.2: + version "3.8.2" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" + integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz" @@ -5564,6 +5608,18 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" +redis@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.3.0.tgz#c01de4cf13821406addb7fcb70afe203266c4c39" + integrity sha512-RXRUor0iU1vizu4viHoUyLpe1ZO/RngZp0V9DyXBHTI+7tC7rEz6Wzn4Sv9v0tTJeqGAzdJ+q5YVbNKKQ5hX9A== + dependencies: + "@redis/bloom" "1.0.2" + "@redis/client" "1.3.0" + "@redis/graph" "1.0.1" + "@redis/json" "1.0.3" + "@redis/search" "1.1.0" + "@redis/time-series" "1.0.3" + redux@^4.0.0, redux@^4.0.4: version "4.2.0" resolved "https://registry.npmjs.org/redux/-/redux-4.2.0.tgz" @@ -6711,7 +6767,7 @@ write-json-file@^4.3.0: sort-keys "^4.0.0" write-file-atomic "^3.0.0" -yallist@^4.0.0: +yallist@4.0.0, yallist@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==