From 467c7f289f290fd8699d0fdd8147a09a96212a4c Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sat, 27 Aug 2022 21:38:35 -0700 Subject: [PATCH] Database refactoring. --- lib/clickhouse.js | 60 +++++++++----- lib/constants.js | 31 +------ lib/db.js | 14 +++- lib/format.js | 2 +- lib/kafka.js | 82 +++++++++--------- lib/{relational.js => prisma.js} | 71 +++++++++------- lib/redis.js | 43 +++++----- queries/admin/account/createAccount.js | 10 +-- queries/admin/account/deleteAccount.js | 50 +++++------ queries/admin/account/getAccountById.js | 14 ++-- queries/admin/account/getAccountByUsername.js | 14 ++-- queries/admin/account/getAccounts.js | 32 ++++--- queries/admin/account/updateAccount.js | 16 ++-- queries/admin/website/createWebsite.js | 20 ++--- queries/admin/website/deleteWebsite.js | 50 ++++++----- queries/admin/website/getAllWebsites.js | 35 ++++---- queries/admin/website/getUserWebsites.js | 20 ++--- queries/admin/website/getWebsiteById.js | 14 ++-- queries/admin/website/getWebsiteByShareId.js | 14 ++-- queries/admin/website/getWebsiteByUuid.js | 14 ++-- queries/admin/website/resetWebsite.js | 24 +++++- queries/admin/website/updateWebsite.js | 16 ++-- queries/analytics/event/getEventMetrics.js | 29 +++---- queries/analytics/event/getEvents.js | 41 +++++---- queries/analytics/event/saveEvent.js | 43 +++++----- .../analytics/pageview/getPageviewMetrics.js | 29 +++---- .../analytics/pageview/getPageviewParams.js | 15 ++-- .../analytics/pageview/getPageviewStats.js | 31 ++++--- queries/analytics/pageview/getPageviews.js | 37 ++++----- queries/analytics/pageview/savePageView.js | 44 +++++----- queries/analytics/session/createSession.js | 37 +++++---- queries/analytics/session/getSessionByUuid.js | 38 ++++----- .../analytics/session/getSessionMetrics.js | 29 +++---- queries/analytics/session/getSessions.js | 23 +++-- queries/analytics/stats/getActiveVisitors.js | 26 +++--- queries/analytics/stats/getWebsiteStats.js | 83 +++++++++---------- scripts/copy-db-files.js | 6 +- 37 files changed, 566 insertions(+), 591 deletions(-) rename lib/{relational.js => prisma.js} (78%) diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 7ab06b9e..1548b77b 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -1,40 +1,51 @@ import { ClickHouse } from 'clickhouse'; import dateFormat from 'dateformat'; +import debug from 'debug'; import { FILTER_IGNORED } from 'lib/constants'; -import { CLICKHOUSE_DATE_FORMATS } from './constants'; +import { CLICKHOUSE } from 'lib/db'; + +export const CLICKHOUSE_DATE_FORMATS = { + minute: '%Y-%m-%d %H:%M:00', + hour: '%Y-%m-%d %H:00:00', + day: '%Y-%m-%d', + month: '%Y-%m-01', + year: '%Y-01-01', +}; + +const log = debug('clickhouse'); function getClient() { - if (!process.env.ANALYTICS_URL) { + if (!process.env.CLICKHOUSE_URL) { return null; } - const url = new URL(process.env.ANALYTICS_URL); - const database = url.pathname.replace('/', ''); + const { + hostname, + port, + pathname, + username = 'default', + password, + } = new URL(process.env.CLICKHOUSE_URL); - return new ClickHouse({ - url: url.hostname, - port: Number(url.port), - basicAuth: url.password - ? { - username: url.username || 'default', - password: url.password, - } - : null, + const client = new ClickHouse({ + url: hostname, + port: Number(port), format: 'json', config: { - database, + database: pathname.replace('/', ''), }, + basicAuth: password ? { username, password } : null, }); + + if (process.env.NODE_ENV !== 'production') { + global[CLICKHOUSE] = clickhouse; + } + + log('Clickhouse initialized'); + + return client; } -const clickhouse = global.clickhouse || getClient(); - -if (process.env.NODE_ENV !== 'production') { - global.clickhouse = clickhouse; -} - -export { clickhouse }; - function getDateStringQuery(data, unit) { return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`; } @@ -176,7 +187,12 @@ async function findFirst(data) { return data[0] ?? null; } +// Initialization +const clickhouse = global[CLICKHOUSE] || getClient(); + export default { + client: clickhouse, + log, getDateStringQuery, getDateQuery, getDateFormat, diff --git a/lib/constants.js b/lib/constants.js index e20a74e6..feef540d 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -67,36 +67,6 @@ export const EVENT_COLORS = [ '#ffec16', ]; -export const RELATIONAL = 'relational'; -export const POSTGRESQL = 'postgresql'; -export const MYSQL = 'mysql'; -export const CLICKHOUSE = 'clickhouse'; -export const KAFKA = 'kafka'; - -export const MYSQL_DATE_FORMATS = { - minute: '%Y-%m-%d %H:%i:00', - hour: '%Y-%m-%d %H:00:00', - day: '%Y-%m-%d', - month: '%Y-%m-01', - year: '%Y-01-01', -}; - -export const POSTGRESQL_DATE_FORMATS = { - minute: 'YYYY-MM-DD HH24:MI:00', - hour: 'YYYY-MM-DD HH24:00:00', - day: 'YYYY-MM-DD', - month: 'YYYY-MM-01', - year: 'YYYY-01-01', -}; - -export const CLICKHOUSE_DATE_FORMATS = { - minute: '%Y-%m-%d %H:%M:00', - hour: '%Y-%m-%d %H:00:00', - day: '%Y-%m-%d', - month: '%Y-%m-01', - year: '%Y-01-01', -}; - export const FILTER_IGNORED = Symbol.for('filter-ignored'); export const DOMAIN_REGEX = @@ -107,6 +77,7 @@ export const LAPTOP_SCREEN_WIDTH = 1024; export const MOBILE_SCREEN_WIDTH = 479; export const URL_LENGTH = 500; +export const EVENT_NAME_LENGTH = 50; export const DESKTOP_OS = [ 'Windows 3.11', diff --git a/lib/db.js b/lib/db.js index 13139cd5..9db00c92 100644 --- a/lib/db.js +++ b/lib/db.js @@ -1,5 +1,11 @@ -import { POSTGRESQL, RELATIONAL, MYSQL, KAFKA, CLICKHOUSE } from 'lib/constants'; +export const PRISMA = 'prisma'; +export const POSTGRESQL = 'postgresql'; +export const MYSQL = 'mysql'; +export const CLICKHOUSE = 'clickhouse'; +export const KAFKA = 'kafka'; +export const KAFKA_PRODUCER = 'kafka-producer'; +// Fixes issue with converting bigint values BigInt.prototype.toJSON = function () { return Number(this); }; @@ -14,11 +20,11 @@ export function getDatabaseType(url = process.env.DATABASE_URL) { return type; } -export async function runAnalyticsQuery(queries) { - const db = getDatabaseType(process.env.ANALYTICS_URL || process.env.DATABASE_URL); +export async function runQuery(queries) { + const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL); if (db === POSTGRESQL || db === MYSQL) { - return queries[RELATIONAL](); + return queries[PRISMA](); } if (db === CLICKHOUSE) { diff --git a/lib/format.js b/lib/format.js index a336c1c4..a85c3ef7 100644 --- a/lib/format.js +++ b/lib/format.js @@ -74,7 +74,7 @@ export function stringToColor(str) { let color = '#'; for (let i = 0; i < 3; i++) { let value = (hash >> (i * 8)) & 0xff; - color += ('00' + value.toString(16)).substr(-2); + color += ('00' + value.toString(16)).substring(-2); } return color; } diff --git a/lib/kafka.js b/lib/kafka.js index 39320728..01b69603 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -1,57 +1,53 @@ import { Kafka, logLevel } from 'kafkajs'; import dateFormat from 'dateformat'; +import debug from 'debug'; +import { KAFKA, KAFKA_PRODUCER } from 'lib/db'; -export function getClient() { - if (!process.env.KAFKA_URL) { +const log = debug('kafka'); + +function getClient() { + if (!process.env.KAFKA_URL || !process.env.KAFKA_BROKER) { return null; } - const url = new URL(process.env.KAFKA_URL); + const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); - if (url.username.length === 0 && url.password.length === 0) { - return new Kafka({ - clientId: 'umami', - brokers: brokers, - connectionTimeout: 3000, - logLevel: logLevel.ERROR, - }); - } else { - return new Kafka({ - clientId: 'umami', - brokers: brokers, - connectionTimeout: 3000, - ssl: true, - sasl: { - mechanism: 'plain', - username: url.username, - password: url.password, - }, - }); - } -} -const kafka = global.kafka || getClient(); -let kafkaProducer = null; + const ssl = + username && password + ? { + ssl: true, + sasl: { + mechanism: 'plain', + username, + password, + }, + } + : {}; -(async () => { - if (kafka) { - kafkaProducer = global.kakfaProducer || (await getProducer()); - } + const client = new Kafka({ + clientId: 'umami', + brokers: brokers, + connectionTimeout: 3000, + logLevel: logLevel.ERROR, + ...ssl, + }); if (process.env.NODE_ENV !== 'production') { - global.kafka = kafka; - if (kafka) { - global.kakfaProducer = kafkaProducer; - } + global[KAFKA] = client; } -})(); -export { kafka, kafkaProducer }; + return client; +} async function getProducer() { const producer = kafka.producer(); await producer.connect(); + if (process.env.NODE_ENV !== 'production') { + global[KAFKA_PRODUCER] = producer; + } + return producer; } @@ -60,7 +56,7 @@ function getDateFormat(date) { } async function sendMessage(params, topic) { - await kafkaProducer.send({ + await producer.send({ topic, messages: [ { @@ -72,7 +68,19 @@ async function sendMessage(params, topic) { }); } +// Initialization +let kafka; +let producer; + +(async () => { + kafka = global[KAFKA] || getClient(); + producer = global[KAFKA_PRODUCER] || (await getProducer()); +})(); + export default { + client: kafka, + producer: producer, + log, getDateFormat, sendMessage, }; diff --git a/lib/relational.js b/lib/prisma.js similarity index 78% rename from lib/relational.js rename to lib/prisma.js index d86f0607..af8759ba 100644 --- a/lib/relational.js +++ b/lib/prisma.js @@ -1,19 +1,30 @@ import { PrismaClient } from '@prisma/client'; import chalk from 'chalk'; -import { - FILTER_IGNORED, - MYSQL, - MYSQL_DATE_FORMATS, - POSTGRESQL, - POSTGRESQL_DATE_FORMATS, -} from 'lib/constants'; -import { getDatabaseType } from 'lib/db'; import moment from 'moment-timezone'; import debug from 'debug'; +import { PRISMA, MYSQL, POSTGRESQL } from 'lib/db'; +import { FILTER_IGNORED } from 'lib/constants'; +import { getDatabaseType } from 'lib/db'; + +const MYSQL_DATE_FORMATS = { + minute: '%Y-%m-%d %H:%i:00', + hour: '%Y-%m-%d %H:00:00', + day: '%Y-%m-%d', + month: '%Y-%m-01', + year: '%Y-01-01', +}; + +const POSTGRESQL_DATE_FORMATS = { + minute: 'YYYY-MM-DD HH24:MI:00', + hour: 'YYYY-MM-DD HH24:00:00', + day: 'YYYY-MM-DD', + month: 'YYYY-MM-01', + year: 'YYYY-01-01', +}; const log = debug('prisma'); -const options = { +const PRISMA_OPTIONS = { log: [ { emit: 'event', @@ -33,16 +44,16 @@ function getClient(options) { prisma.$on('query', logQuery); } + if (process.env.NODE_ENV !== 'production') { + global[PRISMA] = prisma; + } + + log('Prisma initialized'); + return prisma; } -let prisma = global.prisma || getClient(options); - -if (process.env.NODE_ENV !== 'production') { - global.prisma = prisma; -} - -export function getDateQuery(field, unit, timezone) { +function getDateQuery(field, unit, timezone) { const db = getDatabaseType(process.env.DATABASE_URL); if (db === POSTGRESQL) { @@ -63,7 +74,7 @@ export function getDateQuery(field, unit, timezone) { } } -export function getTimestampInterval(field) { +function getTimestampInterval(field) { const db = getDatabaseType(process.env.DATABASE_URL); if (db === POSTGRESQL) { @@ -75,7 +86,7 @@ export function getTimestampInterval(field) { } } -export function getFilterQuery(table, column, filters = {}, params = []) { +function getFilterQuery(table, column, filters = {}, params = []) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -135,7 +146,7 @@ export function getFilterQuery(table, column, filters = {}, params = []) { return query.join('\n'); } -export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') { +function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') { const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } = filters; @@ -158,13 +169,7 @@ export function parseFilters(table, column, filters = {}, params = [], sessionKe }; } -export async function runQuery(query) { - return query.catch(e => { - throw e; - }); -} - -export async function rawQuery(query, params = []) { +async function rawQuery(query, params = []) { const db = getDatabaseType(process.env.DATABASE_URL); if (db !== POSTGRESQL && db !== MYSQL) { @@ -173,17 +178,23 @@ export async function rawQuery(query, params = []) { const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query; - return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params])); + return prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]); } -export { prisma }; +async function multiQuery(queries) { + return prisma.$transaction(queries); +} + +// Initialization +const prisma = global[PRISMA] || getClient(PRISMA_OPTIONS); export default { - prisma, + client: prisma, + log, getDateQuery, getTimestampInterval, getFilterQuery, parseFilters, - runQuery, rawQuery, + multiQuery, }; diff --git a/lib/redis.js b/lib/redis.js index 97011bed..e307729b 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -1,6 +1,10 @@ import { createClient } from 'redis'; import { startOfMonth } from 'date-fns'; import { getSessions, getAllWebsites } from '/queries'; +import debug from 'debug'; + +const log = debug('db:redis'); +const REDIS = Symbol.for('redis'); async function getClient() { const redis = new createClient({ @@ -10,30 +14,16 @@ async function getClient() { await redis.connect(); if (process.env.LOG_QUERY) { - redis.on('error', err => console.log('Redis Client Error', err)); + redis.on('error', err => log(err)); + } + + if (process.env.NODE_ENV !== 'production') { + global[REDIS] = redis; } return redis; } -let redis = null; - -(async () => { - redis = global.redis || (await getClient()); - - if (process.env.NODE_ENV !== 'production') { - global.redis = redis; - } - - const value = await redis.get('initialized'); - - if (!value) { - await stageData(); - } -})(); - -export default redis; - async function stageData() { const sessions = await getSessions([], startOfMonth(new Date()).toUTCString()); const websites = await getAllWebsites(); @@ -57,3 +47,18 @@ async function addRedis(ids) { await redis.set(key, value); } } + +// Initialization +let redis = null; + +(async () => { + redis = global[REDIS] || (await getClient()); + + const value = await redis.get('initialized'); + + if (!value) { + await stageData(); + } +})(); + +export default redis; diff --git a/queries/admin/account/createAccount.js b/queries/admin/account/createAccount.js index fb3997e8..2c4e1a40 100644 --- a/queries/admin/account/createAccount.js +++ b/queries/admin/account/createAccount.js @@ -1,9 +1,7 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function createAccount(data) { - return runQuery( - prisma.account.create({ - data, - }), - ); + return prisma.client.account.create({ + data, + }); } diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.js index f03b370e..209d73a9 100644 --- a/queries/admin/account/deleteAccount.js +++ b/queries/admin/account/deleteAccount.js @@ -1,28 +1,28 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function deleteAccount(user_id) { - return runQuery( - prisma.$transaction([ - prisma.pageview.deleteMany({ - where: { session: { website: { user_id } } }, - }), - prisma.event_data.deleteMany({ - where: { event: { session: { website: { user_id } } } }, - }), - prisma.event.deleteMany({ - where: { session: { website: { user_id } } }, - }), - prisma.session.deleteMany({ - where: { website: { user_id } }, - }), - prisma.website.deleteMany({ - where: { user_id }, - }), - prisma.account.delete({ - where: { - user_id, - }, - }), - ]), - ); + const { client } = prisma; + + return client.$transaction([ + client.pageview.deleteMany({ + where: { session: { website: { user_id } } }, + }), + client.event_data.deleteMany({ + where: { event: { session: { website: { user_id } } } }, + }), + client.event.deleteMany({ + where: { session: { website: { user_id } } }, + }), + client.session.deleteMany({ + where: { website: { user_id } }, + }), + client.website.deleteMany({ + where: { user_id }, + }), + client.account.delete({ + where: { + user_id, + }, + }), + ]); } diff --git a/queries/admin/account/getAccountById.js b/queries/admin/account/getAccountById.js index 4c3d647c..74c4a4a9 100644 --- a/queries/admin/account/getAccountById.js +++ b/queries/admin/account/getAccountById.js @@ -1,11 +1,9 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getAccountById(user_id) { - return runQuery( - prisma.account.findUnique({ - where: { - user_id, - }, - }), - ); + return prisma.client.account.findUnique({ + where: { + user_id, + }, + }); } diff --git a/queries/admin/account/getAccountByUsername.js b/queries/admin/account/getAccountByUsername.js index 30480a71..ff64c8ce 100644 --- a/queries/admin/account/getAccountByUsername.js +++ b/queries/admin/account/getAccountByUsername.js @@ -1,11 +1,9 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getAccountByUsername(username) { - return runQuery( - prisma.account.findUnique({ - where: { - username, - }, - }), - ); + return prisma.client.account.findUnique({ + where: { + username, + }, + }); } diff --git a/queries/admin/account/getAccounts.js b/queries/admin/account/getAccounts.js index 94f70c35..154626d1 100644 --- a/queries/admin/account/getAccounts.js +++ b/queries/admin/account/getAccounts.js @@ -1,21 +1,19 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getAccounts() { - return runQuery( - prisma.account.findMany({ - orderBy: [ - { is_admin: 'desc' }, - { - username: 'asc', - }, - ], - select: { - user_id: true, - username: true, - is_admin: true, - created_at: true, - updated_at: true, + return prisma.client.account.findMany({ + orderBy: [ + { is_admin: 'desc' }, + { + username: 'asc', }, - }), - ); + ], + select: { + user_id: true, + username: true, + is_admin: true, + created_at: true, + updated_at: true, + }, + }); } diff --git a/queries/admin/account/updateAccount.js b/queries/admin/account/updateAccount.js index 7a337f59..80509dec 100644 --- a/queries/admin/account/updateAccount.js +++ b/queries/admin/account/updateAccount.js @@ -1,12 +1,10 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function updateAccount(user_id, data) { - return runQuery( - prisma.account.update({ - where: { - user_id, - }, - data, - }), - ); + return prisma.client.account.update({ + where: { + user_id, + }, + data, + }); } diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.js index 7a589db6..d5eaaf31 100644 --- a/queries/admin/website/createWebsite.js +++ b/queries/admin/website/createWebsite.js @@ -1,9 +1,9 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; import redis from 'lib/redis'; export async function createWebsite(user_id, data) { - return runQuery( - prisma.website.create({ + return prisma.client.website + .create({ data: { account: { connect: { @@ -12,12 +12,12 @@ export async function createWebsite(user_id, data) { }, ...data, }, - }), - ).then(async res => { - if (process.env.REDIS_URL) { - await redis.set(`website:${res.website_uuid}`, Number(res.website_id)); - } + }) + .then(async res => { + if (process.env.REDIS_URL) { + await redis.set(`website:${res.website_uuid}`, Number(res.website_id)); + } - return res; - }); + return res; + }); } diff --git a/queries/admin/website/deleteWebsite.js b/queries/admin/website/deleteWebsite.js index 13c3160d..15030444 100644 --- a/queries/admin/website/deleteWebsite.js +++ b/queries/admin/website/deleteWebsite.js @@ -1,30 +1,28 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; import redis from 'lib/redis'; export async function deleteWebsite(website_id) { - return runQuery( - prisma - .$transaction([ - prisma.pageview.deleteMany({ - where: { session: { website: { website_id } } }, - }), - prisma.event_data.deleteMany({ - where: { event: { session: { website: { website_id } } } }, - }), - prisma.event.deleteMany({ - where: { session: { website: { website_id } } }, - }), - prisma.session.deleteMany({ - where: { website: { website_id } }, - }), - prisma.website.delete({ - where: { website_id }, - }), - ]) - .then(async res => { - if (process.env.REDIS_URL) { - await redis.del(`website:${res.website_uuid}`); - } - }), - ); + const { client, multiQuery } = prisma; + + return multiQuery([ + client.pageview.deleteMany({ + where: { session: { website: { website_id } } }, + }), + client.event_data.deleteMany({ + where: { event: { session: { website: { website_id } } } }, + }), + client.event.deleteMany({ + where: { session: { website: { website_id } } }, + }), + client.session.deleteMany({ + where: { website: { website_id } }, + }), + client.website.delete({ + where: { website_id }, + }), + ]).then(async res => { + if (process.env.REDIS_URL) { + await redis.del(`website:${res.website_uuid}`); + } + }); } diff --git a/queries/admin/website/getAllWebsites.js b/queries/admin/website/getAllWebsites.js index f8d2a3e9..23afd813 100644 --- a/queries/admin/website/getAllWebsites.js +++ b/queries/admin/website/getAllWebsites.js @@ -1,24 +1,23 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getAllWebsites() { - let data = await runQuery( - prisma.website.findMany({ - orderBy: [ - { - user_id: 'asc', - }, - { - name: 'asc', - }, - ], - include: { - account: { - select: { - username: true, - }, + let data = await prisma.client.website.findMany({ + orderBy: [ + { + user_id: 'asc', + }, + { + name: 'asc', + }, + ], + include: { + account: { + select: { + username: true, }, }, - }), - ); + }, + }); + return data.map(i => ({ ...i, account: i.account.username })); } diff --git a/queries/admin/website/getUserWebsites.js b/queries/admin/website/getUserWebsites.js index 729ce893..5a9662c0 100644 --- a/queries/admin/website/getUserWebsites.js +++ b/queries/admin/website/getUserWebsites.js @@ -1,14 +1,12 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getUserWebsites(user_id) { - return runQuery( - prisma.website.findMany({ - where: { - user_id, - }, - orderBy: { - name: 'asc', - }, - }), - ); + return prisma.client.website.findMany({ + where: { + user_id, + }, + orderBy: { + name: 'asc', + }, + }); } diff --git a/queries/admin/website/getWebsiteById.js b/queries/admin/website/getWebsiteById.js index f4495b93..9b8cfe93 100644 --- a/queries/admin/website/getWebsiteById.js +++ b/queries/admin/website/getWebsiteById.js @@ -1,11 +1,9 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getWebsiteById(website_id) { - return runQuery( - prisma.website.findUnique({ - where: { - website_id, - }, - }), - ); + return prisma.client.website.findUnique({ + where: { + website_id, + }, + }); } diff --git a/queries/admin/website/getWebsiteByShareId.js b/queries/admin/website/getWebsiteByShareId.js index 5630f89e..fbdeebc9 100644 --- a/queries/admin/website/getWebsiteByShareId.js +++ b/queries/admin/website/getWebsiteByShareId.js @@ -1,11 +1,9 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getWebsiteByShareId(share_id) { - return runQuery( - prisma.website.findUnique({ - where: { - share_id, - }, - }), - ); + return prisma.client.website.findUnique({ + where: { + share_id, + }, + }); } diff --git a/queries/admin/website/getWebsiteByUuid.js b/queries/admin/website/getWebsiteByUuid.js index dd0f24ee..1b9aeb4c 100644 --- a/queries/admin/website/getWebsiteByUuid.js +++ b/queries/admin/website/getWebsiteByUuid.js @@ -1,11 +1,9 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function getWebsiteByUuid(website_uuid) { - return runQuery( - prisma.website.findUnique({ - where: { - website_uuid, - }, - }), - ); + return prisma.client.website.findUnique({ + where: { + website_uuid, + }, + }); } diff --git a/queries/admin/website/resetWebsite.js b/queries/admin/website/resetWebsite.js index 92bb959c..661cd65c 100644 --- a/queries/admin/website/resetWebsite.js +++ b/queries/admin/website/resetWebsite.js @@ -1,5 +1,25 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; +import redis from 'lib/redis'; export async function resetWebsite(website_id) { - return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`); + const { client, multiQuery } = prisma; + + return multiQuery([ + client.pageview.deleteMany({ + where: { session: { website: { website_id } } }, + }), + client.event_data.deleteMany({ + where: { event: { session: { website: { website_id } } } }, + }), + client.event.deleteMany({ + where: { session: { website: { website_id } } }, + }), + client.session.deleteMany({ + where: { website: { website_id } }, + }), + ]).then(async res => { + if (process.env.REDIS_URL) { + await redis.del(`website:${res.website_uuid}`); + } + }); } diff --git a/queries/admin/website/updateWebsite.js b/queries/admin/website/updateWebsite.js index 9de1a803..54f01f66 100644 --- a/queries/admin/website/updateWebsite.js +++ b/queries/admin/website/updateWebsite.js @@ -1,12 +1,10 @@ -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; export async function updateWebsite(website_id, data) { - return runQuery( - prisma.website.update({ - where: { - website_id, - }, - data, - }), - ); + return prisma.client.website.update({ + where: { + website_id, + }, + data, + }); } diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 531701e8..84c00777 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -1,11 +1,10 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { getDateQuery, getFilterQuery, rawQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getEventMetrics(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -18,11 +17,11 @@ async function relationalQuery( unit = 'day', filters = {}, ) { + const { rawQuery, getDateQuery, getFilterQuery } = prisma; const params = [website_id, start_at, end_at]; return rawQuery( - ` - select + `select event_name x, ${getDateQuery('created_at', unit, timezone)} t, count(*) y @@ -31,8 +30,7 @@ async function relationalQuery( and created_at between $2 and $3 ${getFilterQuery('event', filters, params)} group by 1, 2 - order by 2 - `, + order by 2`, params, ); } @@ -45,21 +43,20 @@ async function clickhouseQuery( unit = 'day', filters = {}, ) { + const { rawQuery, getDateQuery, getBetweenDates } = prisma; const params = [website_id]; - return clickhouse.rawQuery( - ` - select + return rawQuery( + `select event_name x, - ${clickhouse.getDateQuery('created_at', unit, timezone)} t, + ${getDateQuery('created_at', unit, timezone)} t, count(*) y from event where website_id= $1 - and ${clickhouse.getBetweenDates('created_at', start_at, end_at)} + and ${getBetweenDates('created_at', start_at, end_at)} ${clickhouse.getFilterQuery('event', filters, params)} group by x, t - order by t - `, + order by t`, params, ); } diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 6bfb6fb8..317a8f2e 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -1,36 +1,34 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runAnalyticsQuery } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export function getEvents(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } function relationalQuery(websites, start_at) { - return runQuery( - prisma.event.findMany({ - where: { - website: { - website_id: { - in: websites, - }, - }, - created_at: { - gte: start_at, + return prisma.client.event.findMany({ + where: { + website: { + website_id: { + in: websites, }, }, - }), - ); + created_at: { + gte: start_at, + }, + }, + }); } function clickhouseQuery(websites, start_at) { - return clickhouse.rawQuery( - ` - select + const { rawQuery, getDateFormat } = clickhouse; + + return rawQuery( + `select event_id, website_id, session_id, @@ -39,7 +37,6 @@ function clickhouseQuery(websites, start_at) { event_name from event where website_id in (${websites.join[',']} - and created_at >= ${clickhouse.getDateFormat(start_at)}) - `, + and created_at >= ${getDateFormat(start_at)})`, ); } diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index b54144ed..10e894ae 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -1,12 +1,12 @@ -import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import kafka from 'lib/kafka'; -import { prisma, runQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db'; +import { URL_LENGTH, EVENT_NAME_LENGTH } from 'lib/constants'; export async function saveEvent(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), [KAFKA]: () => kafkaQuery(...args), }); @@ -16,8 +16,8 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_ const data = { website_id, session_id, - url: url?.substr(0, URL_LENGTH), - event_name: event_name?.substr(0, 50), + url: url?.substring(0, URL_LENGTH), + event_name: event_name?.substring(0, EVENT_NAME_LENGTH), }; if (event_data) { @@ -28,39 +28,38 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_ }; } - return runQuery( - prisma.event.create({ - data, - }), - ); + return prisma.client.event.create({ + data, + }); } async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, event_name }) { + const { rawQuery, getDateFormat } = clickhouse; const params = [ website_id, event_uuid, session_uuid, - url?.substr(0, URL_LENGTH), - event_name?.substr(0, 50), + url?.substring(0, URL_LENGTH), + event_name?.substring(0, EVENT_NAME_LENGTH), ]; - return clickhouse.rawQuery( - ` - insert into umami.event (created_at, website_id, session_uuid, url, event_name) - values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`, + return rawQuery( + `insert into umami.event (created_at, website_id, session_uuid, url, event_name) + values (${getDateFormat(new Date())}, $1, $2, $3, $4);`, params, ); } async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_name }) { + const { getDateFormat, sendMessage } = kafka; const params = { event_uuid: event_uuid, website_id: website_id, session_uuid: session_uuid, - created_at: kafka.getDateFormat(new Date()), - url: url?.substr(0, URL_LENGTH), - event_name: event_name?.substr(0, 50), + created_at: getDateFormat(new Date()), + url: url?.substring(0, URL_LENGTH), + event_name: event_name?.substring(0, EVENT_NAME_LENGTH), }; - await kafka.sendMessage(params, 'event'); + await sendMessage(params, 'event'); } diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index 40af97d0..d34f4c17 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -1,16 +1,16 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { parseFilters, rawQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getPageviewMetrics(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) { + const { rawQuery, parseFilters } = prisma; const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, @@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt ); return rawQuery( - ` - select ${column} x, count(*) y + `select ${column} x, count(*) y from ${table} ${joinSession} where ${table}.website_id=$1 @@ -30,15 +29,15 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt ${joinSession && sessionQuery} ${eventQuery} group by 1 - order by 2 desc - `, + order by 2 desc`, params, ); } async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) { + const { rawQuery, parseFilters, getBetweenDates } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters( + const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, column, filters, @@ -46,19 +45,17 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt 'session_uuid', ); - return clickhouse.rawQuery( - ` - select ${column} x, count(*) y + return rawQuery( + `select ${column} x, count(*) y from ${table} ${joinSession} where ${table}.website_id= $1 - and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)} + and ${getBetweenDates(table + '.created_at', start_at, end_at)} ${pageviewQuery} ${joinSession && sessionQuery} ${eventQuery} group by x - order by y desc - `, + order by y desc`, params, ); } diff --git a/queries/analytics/pageview/getPageviewParams.js b/queries/analytics/pageview/getPageviewParams.js index be6e5148..fa445da5 100644 --- a/queries/analytics/pageview/getPageviewParams.js +++ b/queries/analytics/pageview/getPageviewParams.js @@ -1,14 +1,15 @@ -import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/relational'; -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import prisma from 'lib/prisma'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getPageviewParams(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) { + const { parseFilters, rawQuery } = prisma; const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, @@ -18,8 +19,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt ); return rawQuery( - ` - select url x, + `select url x, count(*) y from ${table} ${joinSession} @@ -30,8 +30,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt ${joinSession && sessionQuery} ${eventQuery} group by 1 - order by 2 desc - `, + order by 2 desc`, params, ); } diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 207f109b..da25ab0c 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -1,11 +1,10 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; -import { getDateQuery, parseFilters, rawQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getPageviewStats(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -20,6 +19,7 @@ async function relationalQuery( filters = {}, sessionKey = 'session_id', ) { + const { getDateQuery, parseFilters, rawQuery } = prisma; const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', @@ -29,8 +29,7 @@ async function relationalQuery( ); return rawQuery( - ` - select ${getDateQuery('pageview.created_at', unit, timezone)} t, + `select ${getDateQuery('pageview.created_at', unit, timezone)} t, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from pageview ${joinSession} @@ -38,8 +37,7 @@ async function relationalQuery( and pageview.created_at between $2 and $3 ${pageviewQuery} ${sessionQuery} - group by 1 - `, + group by 1`, params, ); } @@ -54,6 +52,7 @@ async function clickhouseQuery( filters = {}, sessionKey = 'session_uuid', ) { + const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse; const params = [website_id]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', @@ -63,24 +62,22 @@ async function clickhouseQuery( sessionKey, ); - return clickhouse.rawQuery( - ` - select - ${clickhouse.getDateStringQuery('g.t', unit)} as t, + return rawQuery( + `select + ${getDateStringQuery('g.t', unit)} as t, g.y as y from (select - ${clickhouse.getDateQuery('created_at', unit, timezone)} t, + ${getDateQuery('created_at', unit, timezone)} t, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from pageview ${joinSession} where pageview.website_id= $1 - and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)} + and ${getBetweenDates('pageview.created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} group by t) g - order by t - `, + order by t`, params, ); } diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index fbbd96f8..8ce704d8 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -1,36 +1,32 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; -import { prisma, runQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getPageviews(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(websites, start_at) { - return runQuery( - prisma.pageview.findMany({ - where: { - website: { - website_id: { - in: websites, - }, - }, - created_at: { - gte: start_at, + return prisma.client.pageview.findMany({ + where: { + website: { + website_id: { + in: websites, }, }, - }), - ); + created_at: { + gte: start_at, + }, + }, + }); } async function clickhouseQuery(websites, start_at) { return clickhouse.rawQuery( - ` - select + `select view_id, website_id, session_id, @@ -38,7 +34,6 @@ async function clickhouseQuery(websites, start_at) { url from pageview where website_id in (${websites.join[',']} - and created_at >= ${clickhouse.getDateFormat(start_at)}) - `, + and created_at >= ${clickhouse.getDateFormat(start_at)})`, ); } diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index c2b9527e..36698127 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -1,54 +1,52 @@ -import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runAnalyticsQuery } from 'lib/db'; import kafka from 'lib/kafka'; -import { prisma, runQuery } from 'lib/relational'; +import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db'; +import { URL_LENGTH } from 'lib/constants'; export async function savePageView(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), [KAFKA]: () => kafkaQuery(...args), }); } async function relationalQuery(website_id, { session_id, url, referrer }) { - return runQuery( - prisma.pageview.create({ - data: { - website_id, - session_id, - url: url?.substr(0, URL_LENGTH), - referrer: referrer?.substr(0, URL_LENGTH), - }, - }), - ); + return prisma.client.pageview.create({ + data: { + website_id, + session_id, + url: url?.substring(0, URL_LENGTH), + referrer: referrer?.substring(0, URL_LENGTH), + }, + }); } async function clickhouseQuery(website_id, { session_uuid, url, referrer }) { const params = [ website_id, session_uuid, - url?.substr(0, URL_LENGTH), - referrer?.substr(0, URL_LENGTH), + url?.substring(0, URL_LENGTH), + referrer?.substring(0, URL_LENGTH), ]; return clickhouse.rawQuery( - ` - insert into umami.pageview (created_at, website_id, session_uuid, url, referrer) + `insert into umami.pageview (created_at, website_id, session_uuid, url, referrer) values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`, params, ); } async function kafkaQuery(website_id, { session_uuid, url, referrer }) { + const { getDateFormat, sendMessage } = kafka; const params = { website_id: website_id, session_uuid: session_uuid, - created_at: kafka.getDateFormat(new Date()), - url: url?.substr(0, URL_LENGTH), - referrer: referrer?.substr(0, URL_LENGTH), + created_at: getDateFormat(new Date()), + url: url?.substring(0, URL_LENGTH), + referrer: referrer?.substring(0, URL_LENGTH), }; - await kafka.sendMessage(params, 'pageview'); + await sendMessage(params, 'pageview'); } diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index a4419b7e..994f9ef5 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -1,21 +1,20 @@ +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants'; -import { runAnalyticsQuery } from 'lib/db'; import kafka from 'lib/kafka'; import redis from 'lib/redis'; -import { prisma, runQuery } from 'lib/relational'; +import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db'; export async function createSession(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), [KAFKA]: () => kafkaQuery(...args), }); } async function relationalQuery(website_id, data) { - return runQuery( - prisma.session.create({ + return prisma.client.session + .create({ data: { website_id, ...data, @@ -23,14 +22,14 @@ async function relationalQuery(website_id, data) { select: { session_id: true, }, - }), - ).then(async res => { - if (process.env.REDIS_URL) { - await redis.set(`session:${res.session_uuid}`, ''); - } + }) + .then(async res => { + if (process.env.REDIS_URL) { + await redis.set(`session:${res.session_uuid}`, ''); + } - return res; - }); + return res; + }); } async function clickhouseQuery( @@ -48,10 +47,11 @@ async function clickhouseQuery( language, country ? country : null, ]; + const { rawQuery, getDateFormat } = clickhouse; - await clickhouse.rawQuery( + await rawQuery( `insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country) - values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`, + values (${getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`, params, ); } @@ -60,10 +60,11 @@ async function kafkaQuery( website_id, { session_uuid, hostname, browser, os, screen, language, country, device }, ) { + const { getDateFormat, sendMessage } = kafka; const params = { session_uuid: session_uuid, website_id: website_id, - created_at: kafka.getDateFormat(new Date()), + created_at: getDateFormat(new Date()), hostname: hostname, browser: browser, os: os, @@ -73,7 +74,7 @@ async function kafkaQuery( country: country ? country : null, }; - await kafka.sendMessage(params, 'session'); + await sendMessage(params, 'session'); await redis.set(`session:${session_uuid}`, ''); } diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js index 95302992..6e92a2ca 100644 --- a/queries/analytics/session/getSessionByUuid.js +++ b/queries/analytics/session/getSessionByUuid.js @@ -1,32 +1,28 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; -import { prisma, runQuery } from 'lib/relational'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runAnalyticsQuery } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getSessionByUuid(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(session_uuid) { - return runQuery( - prisma.session.findUnique({ - where: { - session_uuid, - }, - }), - ); + return prisma.client.session.findUnique({ + where: { + session_uuid, + }, + }); } async function clickhouseQuery(session_uuid) { + const { rawQuery, findFirst } = clickhouse; const params = [session_uuid]; - return clickhouse - .rawQuery( - ` - select + return rawQuery( + `select session_uuid, website_id, created_at, @@ -35,12 +31,10 @@ async function clickhouseQuery(session_uuid) { os, device, screen, - "language", + language, country from session - where session_uuid = $1 - `, - params, - ) - .then(result => clickhouse.findFirst(result)); + where session_uuid = $1`, + params, + ).then(result => findFirst(result)); } diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index e0d5a24c..36eb7568 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -1,16 +1,16 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runAnalyticsQuery } from 'lib/db'; -import { parseFilters, rawQuery } from 'lib/relational'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getSessionMetrics(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(website_id, start_at, end_at, field, filters = {}) { + const { parseFilters, rawQuery } = prisma; const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', @@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {} ); return rawQuery( - ` - select ${field} x, count(*) y + `select ${field} x, count(*) y from session as x where x.session_id in ( select pageview.session_id @@ -33,15 +32,15 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {} ${sessionQuery} ) group by 1 - order by 2 desc - `, + order by 2 desc`, params, ); } async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) { + const { parseFilters, getBetweenDates, rawQuery } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters( + const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', null, filters, @@ -49,22 +48,20 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {} 'session_uuid', ); - return clickhouse.rawQuery( - ` - select ${field} x, count(*) y + return rawQuery( + `select ${field} x, count(*) y from session as x where x.session_uuid in ( select pageview.session_uuid from pageview ${joinSession} where pageview.website_id=$1 - and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)} + and ${getBetweenDates('pageview.created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} ) group by x - order by y desc - `, + order by y desc`, params, ); } diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js index bcb0f19e..bf070651 100644 --- a/queries/analytics/session/getSessions.js +++ b/queries/analytics/session/getSessions.js @@ -1,18 +1,17 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runAnalyticsQuery } from 'lib/db'; -import { prisma, runQuery } from 'lib/relational'; +import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db'; export async function getSessions(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(websites, start_at) { return runQuery( - prisma.session.findMany({ + prisma.client.session.findMany({ where: { ...(websites && websites.length > 0 ? { @@ -32,9 +31,10 @@ async function relationalQuery(websites, start_at) { } async function clickhouseQuery(websites, start_at) { - return clickhouse.rawQuery( - ` - select + const { rawQuery, getDateFormat } = clickhouse; + + return rawQuery( + `select session_uuid, website_id, created_at, @@ -43,11 +43,10 @@ async function clickhouseQuery(websites, start_at) { os, device, screen, - "language", + language, country from session where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'} - and created_at >= ${clickhouse.getDateFormat(start_at)} - `, + and created_at >= ${getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index 4e61d3eb..023c00a4 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -1,12 +1,11 @@ import { subMinutes } from 'date-fns'; -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; -import { rawQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getActiveVisitors(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -15,27 +14,24 @@ async function relationalQuery(website_id) { const date = subMinutes(new Date(), 5); const params = [website_id, date]; - return rawQuery( - ` - select count(distinct session_id) x + return prisma.rawQuery( + `select count(distinct session_id) x from pageview where website_id = $1 - and created_at >= $2 - `, + and created_at >= $2`, params, ); } async function clickhouseQuery(website_id) { + const { rawQuery, getDateFormat } = clickhouse; const params = [website_id]; - return clickhouse.rawQuery( - ` - select count(distinct session_uuid) x + return rawQuery( + `select count(distinct session_uuid) x from pageview where website_id = $1 - and created_at >= ${clickhouse.getDateFormat(subMinutes(new Date(), 5))} - `, + and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`, params, ); } diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 081a16ac..5d280563 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -1,16 +1,16 @@ -import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; -import { getDateQuery, getTimestampInterval, parseFilters, rawQuery } from 'lib/relational'; -import { runAnalyticsQuery } from 'lib/db'; +import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; +import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getWebsiteStats(...args) { - return runAnalyticsQuery({ - [RELATIONAL]: () => relationalQuery(...args), + return runQuery({ + [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(website_id, start_at, end_at, filters = {}) { + const { getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', @@ -20,32 +20,31 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) { ); return rawQuery( - ` - select sum(t.c) as "pageviews", + `select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(t.time) as "totaltime" from ( - select pageview.session_id, - ${getDateQuery('pageview.created_at', 'hour')}, - count(*) c, - ${getTimestampInterval('pageview.created_at')} as "time" - from pageview - ${joinSession} - where pageview.website_id=$1 - and pageview.created_at between $2 and $3 - ${pageviewQuery} - ${sessionQuery} - group by 1, 2 - ) t - `, + select pageview.session_id, + ${getDateQuery('pageview.created_at', 'hour')}, + count(*) c, + ${getTimestampInterval('pageview.created_at')} as "time" + from pageview + ${joinSession} + where pageview.website_id=$1 + and pageview.created_at between $2 and $3 + ${pageviewQuery} + ${sessionQuery} + group by 1, 2 + ) t`, params, ); } async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { + const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters( + const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', null, filters, @@ -53,28 +52,26 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { 'session_uuid', ); - return clickhouse.rawQuery( - ` - select - sum(t.c) as "pageviews", - count(distinct t.session_uuid) as "uniques", - sum(if(t.c = 1, 1, 0)) as "bounces", - sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" - from ( - select pageview.session_uuid, - ${clickhouse.getDateQuery('pageview.created_at', 'day')} time_series, - count(*) c, - min(created_at) min_time, - max(created_at) max_time - from pageview - ${joinSession} - where pageview.website_id = $1 - and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)} - ${pageviewQuery} - ${sessionQuery} - group by pageview.session_uuid, time_series - ) t; - `, + return rawQuery( + `select + sum(t.c) as "pageviews", + count(distinct t.session_uuid) as "uniques", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" + from ( + select pageview.session_uuid, + ${getDateQuery('pageview.created_at', 'day')} time_series, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from pageview + ${joinSession} + where pageview.website_id = $1 + and ${getBetweenDates('pageview.created_at', start_at, end_at)} + ${pageviewQuery} + ${sessionQuery} + group by pageview.session_uuid, time_series + ) t;`, params, ); } diff --git a/scripts/copy-db-files.js b/scripts/copy-db-files.js index a7897fca..3e902d45 100644 --- a/scripts/copy-db-files.js +++ b/scripts/copy-db-files.js @@ -3,10 +3,8 @@ const fse = require('fs-extra'); const path = require('path'); const del = require('del'); -function getDatabaseType() { - const type = - process.env.DATABASE_TYPE || - (process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]); +function getDatabaseType(url = process.env.DATABASE_URL) { + const type = process.env.DATABASE_TYPE || (url && url.split(':')[0]); if (type === 'postgres') { return 'postgresql';