From 20b8337e3554f5145c29f1800d5d8d10c556bf64 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 15 Jul 2022 16:47:38 -0700 Subject: [PATCH 1/8] checkpoint --- components/metrics/ActiveUsers.js | 2 +- lib/constants.js | 9 + lib/db.js | 227 +++++++++++- lib/queries.js | 155 --------- package.json | 1 + queries/admin/account/createAccount.js | 3 +- queries/admin/account/deleteAccount.js | 3 +- queries/admin/account/getAccountById.js | 3 +- queries/admin/account/getAccountByUsername.js | 3 +- queries/admin/account/getAccounts.js | 3 +- queries/admin/account/updateAccount.js | 3 +- queries/admin/website/createWebsite.js | 3 +- queries/admin/website/deleteWebsite.js | 3 +- queries/admin/website/getAllWebsites.js | 3 +- queries/admin/website/getUserWebsites.js | 3 +- queries/admin/website/getWebsiteById.js | 3 +- queries/admin/website/getWebsiteByShareId.js | 3 +- queries/admin/website/getWebsiteByUuid.js | 3 +- queries/admin/website/resetWebsite.js | 3 +- queries/admin/website/updateWebsite.js | 3 +- queries/analytics/event/getEventMetrics.js | 42 ++- queries/analytics/event/getEvents.js | 3 +- queries/analytics/event/saveEvent.js | 4 +- .../analytics/pageview/getPageviewMetrics.js | 2 +- .../analytics/pageview/getPageviewStats.js | 2 +- queries/analytics/pageview/getPageviews.js | 3 +- queries/analytics/pageview/savePageView.js | 4 +- queries/analytics/session/createSession.js | 3 +- queries/analytics/session/getSessionByUuid.js | 3 +- .../analytics/session/getSessionMetrics.js | 2 +- queries/analytics/session/getSessions.js | 3 +- queries/analytics/stats/getActiveVisitors.js | 2 +- queries/analytics/stats/getWebsiteStats.js | 2 +- yarn.lock | 328 +++++++++++++++++- 34 files changed, 620 insertions(+), 222 deletions(-) delete mode 100644 lib/queries.js diff --git a/components/metrics/ActiveUsers.js b/components/metrics/ActiveUsers.js index 653fc783..becef9fe 100644 --- a/components/metrics/ActiveUsers.js +++ b/components/metrics/ActiveUsers.js @@ -5,7 +5,7 @@ import useFetch from 'hooks/useFetch'; import Dot from 'components/common/Dot'; import styles from './ActiveUsers.module.css'; -export default function ActiveUsers({ websiteId, className, value, interval = 60000 }) { +export default function ActiveUsers({ websiteId, className, value, interval = 60000000 }) { const url = websiteId ? `/website/${websiteId}/active` : null; const { data } = useFetch(url, { interval, diff --git a/lib/constants.js b/lib/constants.js index 13af1f32..a8c872e2 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -66,6 +66,7 @@ export const EVENT_COLORS = [ export const POSTGRESQL = 'postgresql'; export const MYSQL = 'mysql'; +export const CLICKHOUSE = 'clickhouse'; export const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -83,6 +84,14 @@ export const POSTGRESQL_DATE_FORMATS = { year: 'YYYY-01-01', }; +export const CLICKHOUSE_DATE_FORMATS = { + minute: '%Y-%m-%d %H:%M:00', + hour: '%Y-%m-%d %H:00:00', + day: '%Y-%m-%d', + month: '%Y-%m-01', + year: '%Y-01-01', +}; + export const DOMAIN_REGEX = /^(localhost(:[1-9]\d{0,4})?|((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63})$/; diff --git a/lib/db.js b/lib/db.js index 12bfd6d3..c5d144ef 100644 --- a/lib/db.js +++ b/lib/db.js @@ -1,5 +1,14 @@ import { PrismaClient } from '@prisma/client'; +import { ClickHouse } from 'clickhouse'; import chalk from 'chalk'; +import { + MYSQL, + MYSQL_DATE_FORMATS, + POSTGRESQL, + POSTGRESQL_DATE_FORMATS, + CLICKHOUSE, +} from 'lib/constants'; +import moment from 'moment-timezone'; BigInt.prototype.toJSON = function () { return this.toString(); @@ -18,20 +27,218 @@ function logQuery(e) { console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`)); } -let prisma; +function initializePrisma(options) { + let prisma; -if (process.env.NODE_ENV === 'production') { - prisma = new PrismaClient(options); -} else { - if (!global.prisma) { - global.prisma = new PrismaClient(options); + if (process.env.NODE_ENV === 'production') { + prisma = new PrismaClient(options); + } else { + if (!global.prisma) { + global.prisma = new PrismaClient(options); + } + + prisma = global.prisma; } - prisma = global.prisma; + if (process.env.LOG_QUERY) { + prisma.$on('query', logQuery); + } + + return prisma; } -if (process.env.LOG_QUERY) { - prisma.$on('query', logQuery); +function initializeClickhouse() { + if (process.env.ANALYTICS_URL) { + return null; + } + + return new ClickHouse({ + url: process.env.ANALYTICS_URL, + format: 'json', + }); } -export default prisma; +const prisma = initializePrisma(options); +const clickhouse = initializeClickhouse(); + +export { prisma, clickhouse }; + +export function getDatabase() { + const type = + process.env.DATABASE_TYPE || + (process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]); + + if (type === 'postgres') { + return POSTGRESQL; + } + + return type; +} + +export function getAnalyticsDatabase() { + const type = + process.env.ANALYTICS_TYPE || + (process.env.ANALYTICS_URL && process.env.ANALYTICS_URL.split(':')[0]); + + if (type === 'postgres') { + return POSTGRESQL; + } + + if (!type) { + return getDatabase(); + } + + return type; +} + +export function getDateStringQuery(data, unit) { + const db = getDatabase(); + + if (db === POSTGRESQL) { + return `to_char(${data}, '${POSTGRESQL_DATE_FORMATS[unit]}')`; + } + + if (db === MYSQL) { + return `DATE_FORMAT(${data}, '${MYSQL_DATE_FORMATS[unit]}')`; + } +} + +export function getDateQuery(field, unit, timezone) { + const db = getDatabase(); + + if (db === POSTGRESQL) { + if (timezone) { + return `date_trunc('${unit}', ${field} at time zone '${timezone}')`; + } + return `date_trunc('${unit}', ${field})`; + } + + if (db === MYSQL) { + if (timezone) { + const tz = moment.tz(timezone).format('Z'); + + return `convert_tz(${field},'+00:00','${tz}')`; + } + + return `${field}`; + } +} + +export function getTimestampInterval(field) { + const db = getDatabase(); + + if (db === POSTGRESQL) { + return `floor(extract(epoch from max(${field}) - min(${field})))`; + } + + if (db === MYSQL) { + return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`; + } +} + +export function getFilterQuery(table, filters = {}, params = []) { + const query = Object.keys(filters).reduce((arr, key) => { + const value = filters[key]; + + if (value === undefined) { + return arr; + } + + switch (key) { + case 'url': + if (table === 'pageview' || table === 'event') { + arr.push(`and ${table}.${key}=$${params.length + 1}`); + params.push(decodeURIComponent(value)); + } + break; + + case 'os': + case 'browser': + case 'device': + case 'country': + if (table === 'session') { + arr.push(`and ${table}.${key}=$${params.length + 1}`); + params.push(decodeURIComponent(value)); + } + break; + + case 'event_type': + if (table === 'event') { + arr.push(`and ${table}.${key}=$${params.length + 1}`); + params.push(decodeURIComponent(value)); + } + break; + + case 'referrer': + if (table === 'pageview') { + arr.push(`and ${table}.referrer like $${params.length + 1}`); + params.push(`%${decodeURIComponent(value)}%`); + } + break; + + case 'domain': + if (table === 'pageview') { + arr.push(`and ${table}.referrer not like $${params.length + 1}`); + arr.push(`and ${table}.referrer not like '/%'`); + params.push(`%://${value}/%`); + } + break; + } + + return arr; + }, []); + + return query.join('\n'); +} + +export function parseFilters(table, filters = {}, params = []) { + const { domain, url, event_url, referrer, os, browser, device, country, event_type } = filters; + + const pageviewFilters = { domain, url, referrer }; + const sessionFilters = { os, browser, device, country }; + const eventFilters = { url: event_url, event_type }; + + return { + pageviewFilters, + sessionFilters, + eventFilters, + event: { event_type }, + joinSession: + os || browser || device || country + ? `inner join session on ${table}.session_id = session.session_id` + : '', + pageviewQuery: getFilterQuery('pageview', pageviewFilters, params), + sessionQuery: getFilterQuery('session', sessionFilters, params), + eventQuery: getFilterQuery('event', eventFilters, params), + }; +} + +export async function runQuery(query) { + return query.catch(e => { + throw e; + }); +} + +export async function rawQuery(query, params = []) { + const db = getDatabase(); + + if (db !== POSTGRESQL && db !== MYSQL) { + return Promise.reject(new Error('Unknown database.')); + } + + const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query; + + return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params])); +} + +export function runAnalyticsQuery(relational, clickhouse) { + const db = getAnalyticsDatabase(); + + if (db === POSTGRESQL || db === MYSQL) { + return runQuery(relational); + } + + if (db === CLICKHOUSE) { + return runQuery(clickhouse); + } +} diff --git a/lib/queries.js b/lib/queries.js deleted file mode 100644 index 98fc6aa3..00000000 --- a/lib/queries.js +++ /dev/null @@ -1,155 +0,0 @@ -import { MYSQL, MYSQL_DATE_FORMATS, POSTGRESQL, POSTGRESQL_DATE_FORMATS } from 'lib/constants'; -import prisma from 'lib/db'; -import moment from 'moment-timezone'; - -export function getDatabase() { - const type = - process.env.DATABASE_TYPE || - (process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]); - - if (type === 'postgres') { - return 'postgresql'; - } - - return type; -} - -export function getDateStringQuery(data, unit) { - const db = getDatabase(); - - if (db === POSTGRESQL) { - return `to_char(${data}, '${POSTGRESQL_DATE_FORMATS[unit]}')`; - } - - if (db === MYSQL) { - return `DATE_FORMAT(${data}, '${MYSQL_DATE_FORMATS[unit]}')`; - } -} - -export function getDateQuery(field, unit, timezone) { - const db = getDatabase(); - - if (db === POSTGRESQL) { - if (timezone) { - return `date_trunc('${unit}', ${field} at time zone '${timezone}')`; - } - return `date_trunc('${unit}', ${field})`; - } - - if (db === MYSQL) { - if (timezone) { - const tz = moment.tz(timezone).format('Z'); - - return `convert_tz(${field},'+00:00','${tz}')`; - } - - return `${field}`; - } -} - -export function getTimestampInterval(field) { - const db = getDatabase(); - - if (db === POSTGRESQL) { - return `floor(extract(epoch from max(${field}) - min(${field})))`; - } - - if (db === MYSQL) { - return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`; - } -} - -export function getFilterQuery(table, filters = {}, params = []) { - const query = Object.keys(filters).reduce((arr, key) => { - const value = filters[key]; - - if (value === undefined) { - return arr; - } - - switch (key) { - case 'url': - if (table === 'pageview' || table === 'event') { - arr.push(`and ${table}.${key}=$${params.length + 1}`); - params.push(decodeURIComponent(value)); - } - break; - - case 'os': - case 'browser': - case 'device': - case 'country': - if (table === 'session') { - arr.push(`and ${table}.${key}=$${params.length + 1}`); - params.push(decodeURIComponent(value)); - } - break; - - case 'event_type': - if (table === 'event') { - arr.push(`and ${table}.${key}=$${params.length + 1}`); - params.push(decodeURIComponent(value)); - } - break; - - case 'referrer': - if (table === 'pageview') { - arr.push(`and ${table}.referrer like $${params.length + 1}`); - params.push(`%${decodeURIComponent(value)}%`); - } - break; - - case 'domain': - if (table === 'pageview') { - arr.push(`and ${table}.referrer not like $${params.length + 1}`); - arr.push(`and ${table}.referrer not like '/%'`); - params.push(`%://${value}/%`); - } - break; - } - - return arr; - }, []); - - return query.join('\n'); -} - -export function parseFilters(table, filters = {}, params = []) { - const { domain, url, event_url, referrer, os, browser, device, country, event_type } = filters; - - const pageviewFilters = { domain, url, referrer }; - const sessionFilters = { os, browser, device, country }; - const eventFilters = { url: event_url, event_type }; - - return { - pageviewFilters, - sessionFilters, - eventFilters, - event: { event_type }, - joinSession: - os || browser || device || country - ? `inner join session on ${table}.session_id = session.session_id` - : '', - pageviewQuery: getFilterQuery('pageview', pageviewFilters, params), - sessionQuery: getFilterQuery('session', sessionFilters, params), - eventQuery: getFilterQuery('event', eventFilters, params), - }; -} - -export async function runQuery(query) { - return query.catch(e => { - throw e; - }); -} - -export async function rawQuery(query, params = []) { - const db = getDatabase(); - - if (db !== POSTGRESQL && db !== MYSQL) { - return Promise.reject(new Error('Unknown database.')); - } - - const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query; - - return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params])); -} diff --git a/package.json b/package.json index 556f5e8f..d234624d 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "chalk": "^4.1.1", "chart.js": "^2.9.4", "classnames": "^2.3.1", + "clickhouse": "^2.5.0", "colord": "^2.9.2", "cors": "^2.8.5", "cross-spawn": "^7.0.3", diff --git a/queries/admin/account/createAccount.js b/queries/admin/account/createAccount.js index 2902e526..193ed719 100644 --- a/queries/admin/account/createAccount.js +++ b/queries/admin/account/createAccount.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function createAccount(data) { return runQuery( diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.js index 242b851c..402b4111 100644 --- a/queries/admin/account/deleteAccount.js +++ b/queries/admin/account/deleteAccount.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function deleteAccount(user_id) { return runQuery( diff --git a/queries/admin/account/getAccountById.js b/queries/admin/account/getAccountById.js index bb3ae85f..de2605e1 100644 --- a/queries/admin/account/getAccountById.js +++ b/queries/admin/account/getAccountById.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getAccountById(user_id) { return runQuery( diff --git a/queries/admin/account/getAccountByUsername.js b/queries/admin/account/getAccountByUsername.js index 5e786a2a..d476c38e 100644 --- a/queries/admin/account/getAccountByUsername.js +++ b/queries/admin/account/getAccountByUsername.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getAccountByUsername(username) { return runQuery( diff --git a/queries/admin/account/getAccounts.js b/queries/admin/account/getAccounts.js index 647e3626..e6ebedf2 100644 --- a/queries/admin/account/getAccounts.js +++ b/queries/admin/account/getAccounts.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getAccounts() { return runQuery( diff --git a/queries/admin/account/updateAccount.js b/queries/admin/account/updateAccount.js index 3fd252dd..b8d718f4 100644 --- a/queries/admin/account/updateAccount.js +++ b/queries/admin/account/updateAccount.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function updateAccount(user_id, data) { return runQuery( diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.js index a0b33a98..3cb6887f 100644 --- a/queries/admin/website/createWebsite.js +++ b/queries/admin/website/createWebsite.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function createWebsite(user_id, data) { return runQuery( diff --git a/queries/admin/website/deleteWebsite.js b/queries/admin/website/deleteWebsite.js index 0bdd3491..81cef845 100644 --- a/queries/admin/website/deleteWebsite.js +++ b/queries/admin/website/deleteWebsite.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function deleteWebsite(website_id) { return runQuery( diff --git a/queries/admin/website/getAllWebsites.js b/queries/admin/website/getAllWebsites.js index 5cc12cba..7e413fe8 100644 --- a/queries/admin/website/getAllWebsites.js +++ b/queries/admin/website/getAllWebsites.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getAllWebsites() { let data = await runQuery( diff --git a/queries/admin/website/getUserWebsites.js b/queries/admin/website/getUserWebsites.js index 28619417..4cd75997 100644 --- a/queries/admin/website/getUserWebsites.js +++ b/queries/admin/website/getUserWebsites.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getUserWebsites(user_id) { return runQuery( diff --git a/queries/admin/website/getWebsiteById.js b/queries/admin/website/getWebsiteById.js index ad2d9b11..8cf0f50f 100644 --- a/queries/admin/website/getWebsiteById.js +++ b/queries/admin/website/getWebsiteById.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getWebsiteById(website_id) { return runQuery( diff --git a/queries/admin/website/getWebsiteByShareId.js b/queries/admin/website/getWebsiteByShareId.js index d6074e09..9fcc1ab3 100644 --- a/queries/admin/website/getWebsiteByShareId.js +++ b/queries/admin/website/getWebsiteByShareId.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getWebsiteByShareId(share_id) { return runQuery( diff --git a/queries/admin/website/getWebsiteByUuid.js b/queries/admin/website/getWebsiteByUuid.js index 58ada3a3..36c41e67 100644 --- a/queries/admin/website/getWebsiteByUuid.js +++ b/queries/admin/website/getWebsiteByUuid.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getWebsiteByUuid(website_uuid) { return runQuery( diff --git a/queries/admin/website/resetWebsite.js b/queries/admin/website/resetWebsite.js index 45ce2315..18f249b8 100644 --- a/queries/admin/website/resetWebsite.js +++ b/queries/admin/website/resetWebsite.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function resetWebsite(website_id) { return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`); diff --git a/queries/admin/website/updateWebsite.js b/queries/admin/website/updateWebsite.js index 29561bb2..e2a16925 100644 --- a/queries/admin/website/updateWebsite.js +++ b/queries/admin/website/updateWebsite.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function updateWebsite(website_id, data) { return runQuery( diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 1bbc3cac..52be39c0 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -1,6 +1,17 @@ -import { getDateQuery, getDateStringQuery, getFilterQuery, rawQuery } from 'lib/queries'; +import { + getDateQuery, + getDateStringQuery, + getFilterQuery, + rawQuery, + runAnalyticsQuery, + clickhouse, +} from 'lib/db'; -export function getEventMetrics( +export function getEventMetrics(...args) { + return runAnalyticsQuery(relationalQuery(...args), clickhouseQuery(...args)); +} + +function relationalQuery( website_id, start_at, end_at, @@ -26,3 +37,30 @@ export function getEventMetrics( params, ); } + +function clickhouseQuery( + website_id, + start_at, + end_at, + timezone = 'utc', + unit = 'day', + filters = {}, +) { + const params = [website_id, start_at, end_at]; + + return clickhouse.query( + ` + select + event_value x, + ${getDateStringQuery(getDateQuery('created_at', unit, timezone), unit)} t, + count(*) y + from event + where website_id=$1 + and created_at between $2 and $3 + ${getFilterQuery('event', filters, params)} + group by 1, 2 + order by 2 + `, + params, + ); +} diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 9d8671db..f1a318bc 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getEvents(websites, start_at) { return runQuery( diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 8f5bf8d2..17379135 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -1,5 +1,5 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; + import { URL_LENGTH } from 'lib/constants'; export async function saveEvent(website_id, session_id, url, event_type, event_value) { diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index 12667706..1772bce2 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -1,4 +1,4 @@ -import { parseFilters, rawQuery } from 'lib/queries'; +import { parseFilters, rawQuery } from 'lib/db'; export function getPageviewMetrics(website_id, start_at, end_at, field, table, filters = {}) { const params = [website_id, start_at, end_at]; diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index e96b1824..eccf5dc7 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -1,4 +1,4 @@ -import { parseFilters, rawQuery, getDateQuery, getDateStringQuery } from 'lib/queries'; +import { parseFilters, rawQuery, getDateQuery, getDateStringQuery } from 'lib/db'; export function getPageviewStats( website_id, diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index 8fffd539..09efbea1 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getPageviews(websites, start_at) { return runQuery( diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index 427c6143..2061e375 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -1,5 +1,5 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; + import { URL_LENGTH } from 'lib/constants'; export async function savePageView(website_id, session_id, url, referrer) { diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index b6e8e950..f22aa2b0 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function createSession(website_id, data) { return runQuery( diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js index 1d643efa..96d27a85 100644 --- a/queries/analytics/session/getSessionByUuid.js +++ b/queries/analytics/session/getSessionByUuid.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getSessionByUuid(session_uuid) { return runQuery( diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index 356163e9..53294757 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -1,4 +1,4 @@ -import { parseFilters, rawQuery } from 'lib/queries'; +import { parseFilters, rawQuery } from 'lib/db'; export function getSessionMetrics(website_id, start_at, end_at, field, filters = {}) { const params = [website_id, start_at, end_at]; diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js index 450d1ad2..4679670c 100644 --- a/queries/analytics/session/getSessions.js +++ b/queries/analytics/session/getSessions.js @@ -1,5 +1,4 @@ -import { runQuery } from 'lib/queries'; -import prisma from 'lib/db'; +import { prisma, runQuery } from 'lib/db'; export async function getSessions(websites, start_at) { return runQuery( diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index 3f923311..042e364a 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -1,4 +1,4 @@ -import { rawQuery } from 'lib/queries'; +import { rawQuery } from 'lib/db'; import { subMinutes } from 'date-fns'; export function getActiveVisitors(website_id) { diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 19eb3340..de43aa16 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -1,4 +1,4 @@ -import { parseFilters, rawQuery, getDateQuery, getTimestampInterval } from 'lib/queries'; +import { parseFilters, rawQuery, getDateQuery, getTimestampInterval } from 'lib/db'; export function getWebsiteStats(website_id, start_at, end_at, filters = {}) { const params = [website_id, start_at, end_at]; diff --git a/yarn.lock b/yarn.lock index 717e1227..bfcfb6c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1884,6 +1884,14 @@ resolved "https://registry.npmjs.org/@vue/shared/-/shared-3.2.36.tgz" integrity sha512-JtB41wXl7Au3+Nl3gD16Cfpj7k/6aCroZ6BbOiCMFCMvrOpkg/qQUXTso2XowaNqBbnkuGHurLAqkLBxNGc1hQ== +JSONStream@1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.4.tgz#615bb2adb0cd34c8f4c447b5f6512fa1d8f16a2e" + integrity sha512-Y7vfi3I5oMOYIr+WxV8NZxDSwcbNgzdKYsTNInmycOq9bUYwGg9ryu57Wg5NLmCjqdFPNUmpMBo3kSJN9tCbXg== + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + acorn-dynamic-import@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz" @@ -1936,7 +1944,7 @@ ajv-keywords@^5.0.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^6.10.0, ajv@^6.12.4: +ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -2058,6 +2066,18 @@ arrify@^1.0.1: resolved "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" integrity sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA== +asn1@~0.2.3: + version "0.2.6" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" + integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== + ast-types-flow@^0.0.7: version "0.0.7" resolved "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz" @@ -2068,6 +2088,11 @@ astral-regex@^2.0.0: resolved "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + at-least-node@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz" @@ -2085,6 +2110,16 @@ autoprefixer@^10.4.4: picocolors "^1.0.0" postcss-value-parser "^4.2.0" +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== + +aws4@^1.8.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + axe-core@^4.3.5: version "4.4.2" resolved "https://registry.npmjs.org/axe-core/-/axe-core-4.4.2.tgz" @@ -2152,6 +2187,13 @@ balanced-match@^2.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz" integrity sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA== +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== + dependencies: + tweetnacl "^0.14.3" + bcryptjs@^2.4.3: version "2.4.3" resolved "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz" @@ -2276,6 +2318,11 @@ caniuse-lite@^1.0.30001313, caniuse-lite@^1.0.30001317, caniuse-lite@^1.0.300013 resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001356.tgz" integrity sha512-/30854bktMLhxtjieIxsrJBfs2gTM1pel6MXKF3K+RdIVJZcsn2A2QdhsuR4/p9+R204fZw0zCBBhktX8xWuyQ== +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== + chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" @@ -2351,6 +2398,20 @@ cli-truncate@2.1.0, cli-truncate@^2.1.0: slice-ansi "^3.0.0" string-width "^4.2.0" +clickhouse@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/clickhouse/-/clickhouse-2.5.0.tgz#29cece9aeb2e4f449ffb75933f32725a8f7798b3" + integrity sha512-3eszr5FUDnApZDCQOnpHVmNk6opTnzDNdZyG8n3MF1kf/dts47e7o2DbKC2xEG22orGMski1Flvf1J6nkUhvZA== + dependencies: + JSONStream "1.3.4" + lodash "4.17.21" + querystring "0.2.0" + request "2.88.0" + stream2asynciter "1.0.3" + through "2.3.8" + tsv "0.2.0" + uuid "3.4.0" + clone-regexp@^2.1.0: version "2.2.0" resolved "https://registry.npmjs.org/clone-regexp/-/clone-regexp-2.2.0.tgz" @@ -2397,6 +2458,13 @@ colorette@^2.0.16: resolved "https://registry.npmjs.org/colorette/-/colorette-2.0.16.tgz" integrity sha512-hUewv7oMjCp+wkBv5Rm0v87eJhq4woh5rSR+42YSQJKecCqgIqNkZ6lAlQms/BwHPJA5NKMRlpxPRv0n8HQW6g== +combined-stream@^1.0.6, combined-stream@~1.0.6: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@2, commander@^2.20.0: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" @@ -2442,6 +2510,11 @@ core-js-pure@^3.20.2: resolved "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.22.8.tgz" integrity sha512-bOxbZIy9S5n4OVH63XaLVXZ49QKicjowDx/UELyJ68vxfCRpYsbyh/WNZNfEfAk+ekA8vSjt+gCDpvh672bc3w== +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== + cors@^2.8.5: version "2.8.5" resolved "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz" @@ -2651,6 +2724,13 @@ damerau-levenshtein@^1.0.7: resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz" integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA== +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== + dependencies: + assert-plus "^1.0.0" + data-uri-to-buffer@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz" @@ -2749,6 +2829,11 @@ del@^6.0.0: rimraf "^3.0.2" slash "^3.0.0" +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + detect-browser@^5.2.0: version "5.3.0" resolved "https://registry.npmjs.org/detect-browser/-/detect-browser-5.3.0.tgz" @@ -2815,6 +2900,14 @@ dotenv@^10.0.0: resolved "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz" integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + electron-to-chromium@^1.4.118, electron-to-chromium@^1.4.84: version "1.4.143" resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.143.tgz" @@ -3191,6 +3284,11 @@ execall@^2.0.0: dependencies: clone-regexp "^2.1.0" +extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + extract-react-intl-messages@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/extract-react-intl-messages/-/extract-react-intl-messages-4.1.1.tgz" @@ -3212,6 +3310,16 @@ extract-react-intl-messages@^4.1.1: sort-keys "^4.0.0" write-json-file "^4.3.0" +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== + +extsprintf@^1.2.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" + integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== + fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" @@ -3318,6 +3426,20 @@ flatted@^3.1.0: resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz" integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== + +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + formdata-polyfill@^4.0.10: version "4.0.10" resolved "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz" @@ -3441,6 +3563,13 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== + dependencies: + assert-plus "^1.0.0" + glob-parent@^5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz" @@ -3527,6 +3656,19 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4: resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q== + +har-validator@~5.1.0: + version "5.1.5" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" + integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== + dependencies: + ajv "^6.12.3" + har-schema "^2.0.0" + hard-rejection@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz" @@ -3597,6 +3739,15 @@ html-tags@^3.2.0: resolved "https://registry.npmjs.org/html-tags/-/html-tags-3.2.0.tgz" integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg== +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ== + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" @@ -3871,7 +4022,7 @@ is-symbol@^1.0.2, is-symbol@^1.0.3: dependencies: has-symbols "^1.0.2" -is-typedarray@^1.0.0: +is-typedarray@^1.0.0, is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= @@ -3898,6 +4049,11 @@ isexe@^2.0.0: resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== + jest-worker@^26.2.1: version "26.6.2" resolved "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz" @@ -3927,6 +4083,11 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz" @@ -3957,6 +4118,11 @@ json-schema-traverse@^1.0.0: resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz" integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== +json-schema@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" + integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== + json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz" @@ -3969,6 +4135,11 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== + json5@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz" @@ -3997,6 +4168,21 @@ jsonify@~0.0.0: resolved "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz" integrity sha512-trvBk1ki43VZptdBI5rIlG4YOzyeH/WefQt5rj1grasPn4iiZWKet8nkgc4GlsAylaztn0qZfUYOiTsASJFdNA== +jsonparse@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg== + +jsprim@^1.2.2: + version "1.4.2" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb" + integrity sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw== + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.4.0" + verror "1.10.0" + "jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.2.1: version "3.3.0" resolved "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.0.tgz" @@ -4151,7 +4337,7 @@ lodash.truncate@^4.4.2: resolved "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz" integrity sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw== -lodash@^4.17.21: +lodash@4.17.21, lodash@^4.17.21: version "4.17.21" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -4293,6 +4479,18 @@ micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.2" picomatch "^2.3.1" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" @@ -4500,6 +4698,11 @@ nth-check@^2.0.1: dependencies: boolbase "^1.0.0" +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + object-assign@^4, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz" @@ -4712,6 +4915,11 @@ path-type@^4.0.0: resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz" @@ -5099,11 +5307,31 @@ prop-types@^15.7.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" +psl@^1.1.24: + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== + +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + integrity sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ== + punycode@^2.1.0: version "2.1.1" resolved "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +qs@~6.5.2: + version "6.5.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" + integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g== + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" @@ -5360,6 +5588,32 @@ request-ip@^2.1.3: dependencies: is_js "^0.9.0" +request@2.88.0: + version "2.88.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" + integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.0" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.4.3" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + require-from-string@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz" @@ -5458,7 +5712,7 @@ rxjs@^7.5.1: dependencies: tslib "^2.1.0" -safe-buffer@^5.1.0: +safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -5468,6 +5722,11 @@ safe-buffer@~5.1.1: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== +safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + scheduler@^0.20.2: version "0.20.2" resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz" @@ -5683,11 +5942,31 @@ sprintf-js@~1.0.2: resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +sshpk@^1.7.0: + version "1.17.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" + integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + stable@^0.1.8: version "0.1.8" resolved "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz" integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== +stream2asynciter@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/stream2asynciter/-/stream2asynciter-1.0.3.tgz#7ba9046846c8b1caf36ec30d64a73514f7f44c5a" + integrity sha512-9/dEZW+LQjuW6ub5hmWi4n9Pn8W8qA8k7NAE1isecesA164e73xTdy1CJ3S9o9YS+O21HuiK7T+4uS7FgKDy4w== + string-argv@0.3.1: version "0.3.1" resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz" @@ -5970,7 +6249,7 @@ thenby@^1.3.4: resolved "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz" integrity sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ== -through@^2.3.8: +through@2.3.8, "through@>=2.2.7 <3", through@^2.3.8: version "2.3.8" resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -6011,6 +6290,14 @@ topojson-client@^3.1.0: dependencies: commander "2" +tough-cookie@~2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" + integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ== + dependencies: + psl "^1.1.24" + punycode "^1.4.1" + trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz" @@ -6043,6 +6330,23 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tsv@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/tsv/-/tsv-0.2.0.tgz#92869a3cb5f50332f3dc90fca82be667db6f72d6" + integrity sha512-GG6xbOP85giXXom0dS6z9uyDsxktznjpa1AuDlPrIXDqDnbhjr9Vk6Us8iz6U1nENL4CPS2jZDvIjEdaZsmc4Q== + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz" @@ -6175,6 +6479,11 @@ util-deprecate@^1.0.2: resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= +uuid@3.4.0, uuid@^3.3.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + uuid@^7.0.3: version "7.0.3" resolved "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz" @@ -6203,6 +6512,15 @@ vary@^1: resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + vue@^3.2.23: version "3.2.36" resolved "https://registry.npmjs.org/vue/-/vue-3.2.36.tgz" From 4f12933f42b7b44c772fe80cb84f21d4c6294e13 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 15 Jul 2022 22:21:37 -0700 Subject: [PATCH 2/8] checkpoint --- lib/db.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/db.js b/lib/db.js index c5d144ef..3d89210c 100644 --- a/lib/db.js +++ b/lib/db.js @@ -11,7 +11,7 @@ import { import moment from 'moment-timezone'; BigInt.prototype.toJSON = function () { - return this.toString(); + return Number(this); }; const options = { @@ -235,10 +235,10 @@ export function runAnalyticsQuery(relational, clickhouse) { const db = getAnalyticsDatabase(); if (db === POSTGRESQL || db === MYSQL) { - return runQuery(relational); + return relational(); } if (db === CLICKHOUSE) { - return runQuery(clickhouse); + return runQuery(clickhouse()); } } From 6ea2282f827e6273c3af282ee0f3576830583c10 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Wed, 20 Jul 2022 21:31:26 -0700 Subject: [PATCH 3/8] convert analytics queries --- lib/constants.js | 1 + lib/db.js | 94 +++++++++++++++++-- queries/analytics/event/getEventMetrics.js | 34 ++++--- queries/analytics/event/getEvents.js | 35 ++++++- .../analytics/pageview/getPageviewMetrics.js | 43 ++++++++- .../analytics/pageview/getPageviewStats.js | 56 ++++++++++- queries/analytics/pageview/getPageviews.js | 34 ++++++- queries/analytics/session/getSessionByUuid.js | 36 ++++++- .../analytics/session/getSessionMetrics.js | 42 ++++++++- queries/analytics/session/getSessions.js | 40 +++++++- queries/analytics/stats/getActiveVisitors.js | 28 +++++- queries/analytics/stats/getWebsiteStats.js | 79 +++++++++++++--- 12 files changed, 470 insertions(+), 52 deletions(-) diff --git a/lib/constants.js b/lib/constants.js index a8c872e2..53c8bd25 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -64,6 +64,7 @@ export const EVENT_COLORS = [ '#ffec16', ]; +export const RELATIONAL = 'relational'; export const POSTGRESQL = 'postgresql'; export const MYSQL = 'mysql'; export const CLICKHOUSE = 'clickhouse'; diff --git a/lib/db.js b/lib/db.js index 3d89210c..9409178a 100644 --- a/lib/db.js +++ b/lib/db.js @@ -7,8 +7,10 @@ import { POSTGRESQL, POSTGRESQL_DATE_FORMATS, CLICKHOUSE, + RELATIONAL, } from 'lib/constants'; import moment from 'moment-timezone'; +import { CLICKHOUSE_DATE_FORMATS } from './constants'; BigInt.prototype.toJSON = function () { return Number(this); @@ -48,19 +50,48 @@ function initializePrisma(options) { } function initializeClickhouse() { - if (process.env.ANALYTICS_URL) { + if (!process.env.ANALYTICS_URL) { return null; } + const url = new URL(process.env.ANALYTICS_URL); + + const database = url.pathname.replace('/', ''); + return new ClickHouse({ - url: process.env.ANALYTICS_URL, + url: url.hostname, + port: Number(url.port), + basicAuth: url.password + ? { + username: url.username || 'default', + password: url.password, + } + : null, format: 'json', + config: { + database, + }, }); + + // return new ClickHouse({ + // url: 'http://164.92.95.2', + // port: 8123, + // basicAuth: { + // username: 'default', + // password: 'shhhthisissupersecret!', + // }, + // format: 'json', + // config: { + // database: 'umami_dev', + // }, + // }); } const prisma = initializePrisma(options); const clickhouse = initializeClickhouse(); +console.log('clickhouse1: ', clickhouse); + export { prisma, clickhouse }; export function getDatabase() { @@ -103,6 +134,10 @@ export function getDateStringQuery(data, unit) { } } +export function getDateStringQueryClickhouse(data, unit) { + return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`; +} + export function getDateQuery(field, unit, timezone) { const db = getDatabase(); @@ -124,15 +159,31 @@ export function getDateQuery(field, unit, timezone) { } } -export function getTimestampInterval(field) { +export function getDateQueryClickhouse(field, unit, timezone) { + if (timezone) { + return `date_trunc('${unit}', ${field},'${timezone}')`; + } + return `date_trunc('${unit}', ${field})`; +} + +export function getDateFormatClickhouse(date) { + return `parseDateTimeBestEffort('${date.toUTCString()}')`; +} + +export function getBetweenDatesClickhouse(field, start_at, end_at) { + return `${field} between ${getDateFormatClickhouse(start_at)} + and ${getDateFormatClickhouse(end_at)}`; +} + +export function getTimestampInterval(maxColumn, minColumn) { const db = getDatabase(); if (db === POSTGRESQL) { - return `floor(extract(epoch from max(${field}) - min(${field})))`; + return `floor(extract(epoch from max(${maxColumn}) - min(${minColumn})))`; } if (db === MYSQL) { - return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`; + return `floor(unix_timestamp(max(${maxColumn})) - unix_timestamp(min(${minColumn})))`; } } @@ -149,6 +200,7 @@ export function getFilterQuery(table, filters = {}, params = []) { if (table === 'pageview' || table === 'event') { arr.push(`and ${table}.${key}=$${params.length + 1}`); params.push(decodeURIComponent(value)); + console.log(params); } break; @@ -213,6 +265,22 @@ export function parseFilters(table, filters = {}, params = []) { }; } +export function replaceQueryClickhouse(string, params = []) { + let formattedString = string; + + params.forEach((a, i) => { + let replace = a; + + if (typeof a === 'string' || a instanceof String) { + replace = `'${replace}'`; + } + + formattedString = formattedString.replace(`$${i + 1}`, replace); + }); + + return formattedString; +} + export async function runQuery(query) { return query.catch(e => { throw e; @@ -231,14 +299,24 @@ export async function rawQuery(query, params = []) { return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params])); } -export function runAnalyticsQuery(relational, clickhouse) { +export async function rawQueryClickhouse(query, params = [], debug = false) { + let formattedQuery = replaceQueryClickhouse(query, params); + + if (debug || process.env.LOG_QUERY) { + console.log(formattedQuery); + } + + return clickhouse.query(formattedQuery).toPromise(); +} + +export async function runAnalyticsQuery(queries) { const db = getAnalyticsDatabase(); if (db === POSTGRESQL || db === MYSQL) { - return relational(); + return queries[`${RELATIONAL}`](); } if (db === CLICKHOUSE) { - return runQuery(clickhouse()); + return queries[`${CLICKHOUSE}`](); } } diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 52be39c0..59e55dcf 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -1,17 +1,23 @@ +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; import { + rawQueryClickhouse, + getBetweenDatesClickhouse, getDateQuery, + getDateQueryClickhouse, getDateStringQuery, getFilterQuery, rawQuery, runAnalyticsQuery, - clickhouse, } from 'lib/db'; -export function getEventMetrics(...args) { - return runAnalyticsQuery(relationalQuery(...args), clickhouseQuery(...args)); +export async function getEventMetrics(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); } -function relationalQuery( +async function relationalQuery( website_id, start_at, end_at, @@ -38,28 +44,28 @@ function relationalQuery( ); } -function clickhouseQuery( +async function clickhouseQuery( website_id, start_at, end_at, - timezone = 'utc', + timezone = 'UTC', unit = 'day', filters = {}, ) { - const params = [website_id, start_at, end_at]; + const params = [website_id]; - return clickhouse.query( + return rawQueryClickhouse( ` select event_value x, - ${getDateStringQuery(getDateQuery('created_at', unit, timezone), unit)} t, + ${getDateQueryClickhouse('created_at', unit, timezone)} t, count(*) y from event - where website_id=$1 - and created_at between $2 and $3 - ${getFilterQuery('event', filters, params)} - group by 1, 2 - order by 2 + where website_id= $1 + and ${getBetweenDatesClickhouse('created_at', start_at, end_at)} + ${getFilterQuery('event', filters, params)} + group by x, t + order by t `, params, ); diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index f1a318bc..059c8014 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -1,6 +1,20 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + rawQueryClickhouse, + getDateFormatClickhouse, + prisma, + runAnalyticsQuery, + runQuery, +} from 'lib/db'; -export async function getEvents(websites, start_at) { +export function getEvents(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +function relationalQuery(websites, start_at) { return runQuery( prisma.event.findMany({ where: { @@ -16,3 +30,20 @@ export async function getEvents(websites, start_at) { }), ); } + +function clickhouseQuery(websites, start_at) { + return rawQueryClickhouse( + ` + select + event_id, + website_id, + session_id, + created_at, + url, + event_type + from event + where website_id in (${websites.join[',']} + and created_at >= ${getDateFormatClickhouse(start_at)}) + `, + ); +} diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index 1772bce2..fb2cf8a5 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -1,6 +1,20 @@ -import { parseFilters, rawQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + rawQueryClickhouse, + runAnalyticsQuery, + parseFilters, + rawQuery, + getBetweenDatesClickhouse, +} from 'lib/db'; -export function getPageviewMetrics(website_id, start_at, end_at, field, table, filters = {}) { +export async function getPageviewMetrics(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(website_id, start_at, end_at, field, table, filters = {}) { const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, @@ -24,3 +38,28 @@ export function getPageviewMetrics(website_id, start_at, end_at, field, table, f params, ); } + +async function clickhouseQuery(website_id, start_at, end_at, field, table, filters = {}) { + const params = [website_id]; + const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( + table, + filters, + params, + ); + + return rawQueryClickhouse( + ` + select ${field} x, count(*) y + from ${table} + ${joinSession} + where ${table}.website_id= $1 + and ${getBetweenDatesClickhouse(table + '.created_at', start_at, end_at)} + ${pageviewQuery} + ${joinSession && sessionQuery} + ${eventQuery} + group by x + order by y desc + `, + params, + ); +} diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index eccf5dc7..8a26280c 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -1,6 +1,24 @@ -import { parseFilters, rawQuery, getDateQuery, getDateStringQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + rawQueryClickhouse, + getBetweenDatesClickhouse, + getDateQuery, + getDateQueryClickhouse, + getDateStringQuery, + getDateStringQueryClickhouse, + parseFilters, + rawQuery, + runAnalyticsQuery, +} from 'lib/db'; -export function getPageviewStats( +export async function getPageviewStats(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( website_id, start_at, end_at, @@ -32,3 +50,37 @@ export function getPageviewStats( params, ); } + +async function clickhouseQuery( + website_id, + start_at, + end_at, + timezone = 'UTC', + unit = 'day', + count = '*', + filters = {}, +) { + const params = [website_id]; + const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); + + return rawQueryClickhouse( + ` + select + ${getDateStringQueryClickhouse('g.t', unit)} as t, + g.y as y + from + (select + ${getDateQueryClickhouse('created_at', unit, timezone)} t, + count(${count}) y + from pageview + ${joinSession} + where pageview.website_id= $1 + and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)} + ${pageviewQuery} + ${sessionQuery} + group by t) g + order by t + `, + params, + ); +} diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index 09efbea1..d7b28611 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -1,6 +1,20 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + rawQueryClickhouse, + getDateFormatClickhouse, + prisma, + runAnalyticsQuery, + runQuery, +} from 'lib/db'; -export async function getPageviews(websites, start_at) { +export async function getPageviews(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websites, start_at) { return runQuery( prisma.pageview.findMany({ where: { @@ -16,3 +30,19 @@ export async function getPageviews(websites, start_at) { }), ); } + +async function clickhouseQuery(websites, start_at) { + return rawQueryClickhouse( + ` + select + view_id, + website_id, + session_id, + created_at, + url + from pageview + where website_id in (${websites.join[',']} + and created_at >= ${getDateFormatClickhouse(start_at)}) + `, + ); +} diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js index 96d27a85..042b4914 100644 --- a/queries/analytics/session/getSessionByUuid.js +++ b/queries/analytics/session/getSessionByUuid.js @@ -1,6 +1,14 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { rawQueryClickhouse, prisma, runAnalyticsQuery, runQuery } from 'lib/db'; -export async function getSessionByUuid(session_uuid) { +export async function getSessionByUuid(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(session_uuid) { return runQuery( prisma.session.findUnique({ where: { @@ -9,3 +17,27 @@ export async function getSessionByUuid(session_uuid) { }), ); } + +async function clickhouseQuery(session_uuid) { + const params = [session_uuid]; + + return rawQueryClickhouse( + ` + select + session_id, + session_uuid, + website_id, + created_at, + hostname, + browser, + os, + device, + screen, + "language", + country + from session + where session_id = $1 + `, + params, + ); +} diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index 53294757..432b696c 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -1,6 +1,20 @@ -import { parseFilters, rawQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + getBetweenDatesClickhouse, + parseFilters, + rawQuery, + rawQueryClickhouse, + runAnalyticsQuery, +} from 'lib/db'; -export function getSessionMetrics(website_id, start_at, end_at, field, filters = {}) { +export async function getSessionMetrics(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(website_id, start_at, end_at, field, filters = {}) { const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); @@ -23,3 +37,27 @@ export function getSessionMetrics(website_id, start_at, end_at, field, filters = params, ); } + +async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) { + const params = [website_id]; + const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); + + return rawQueryClickhouse( + ` + select ${field} x, count(*) y + from session as x + where x.session_id in ( + select pageview.session_id + from pageview + ${joinSession} + where pageview.website_id=$1 + and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)} + ${pageviewQuery} + ${sessionQuery} + ) + group by x + order by y desc + `, + params, + ); +} diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js index 4679670c..09b55b8d 100644 --- a/queries/analytics/session/getSessions.js +++ b/queries/analytics/session/getSessions.js @@ -1,6 +1,20 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + getDateFormatClickhouse, + prisma, + rawQueryClickhouse, + runAnalyticsQuery, + runQuery, +} from 'lib/db'; -export async function getSessions(websites, start_at) { +export async function getSessions(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websites, start_at) { return runQuery( prisma.session.findMany({ where: { @@ -16,3 +30,25 @@ export async function getSessions(websites, start_at) { }), ); } + +async function clickhouseQuery(websites, start_at) { + return rawQueryClickhouse( + ` + select + session_id, + session_uuid, + website_id, + created_at, + hostname, + browser, + os, + device, + screen, + "language", + country + from session + where website_id in (${websites.join[',']} + and created_at >= ${getDateFormatClickhouse(start_at)}) + `, + ); +} diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index 042e364a..2789f769 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -1,7 +1,15 @@ -import { rawQuery } from 'lib/db'; import { subMinutes } from 'date-fns'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { getDateFormatClickhouse, rawQuery, rawQueryClickhouse, runAnalyticsQuery } from 'lib/db'; -export function getActiveVisitors(website_id) { +export async function getActiveVisitors(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(website_id) { const date = subMinutes(new Date(), 5); const params = [website_id, date]; @@ -9,9 +17,23 @@ export function getActiveVisitors(website_id) { ` select count(distinct session_id) x from pageview - where website_id=$1 + where website_id = $1 and created_at >= $2 `, params, ); } + +async function clickhouseQuery(website_id) { + const params = [website_id]; + + return rawQueryClickhouse( + ` + select count(distinct session_id) x + from pageview + where website_id = $1 + and created_at >= ${getDateFormatClickhouse(subMinutes(new Date(), 5))} + `, + params, + ); +} diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index de43aa16..f1ee921a 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -1,6 +1,23 @@ -import { parseFilters, rawQuery, getDateQuery, getTimestampInterval } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + getDateQuery, + getBetweenDatesClickhouse, + getDateQueryClickhouse, + getTimestampInterval, + parseFilters, + rawQuery, + rawQueryClickhouse, + runAnalyticsQuery, +} from 'lib/db'; -export function getWebsiteStats(website_id, start_at, end_at, filters = {}) { +export async function getWebsiteStats(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(website_id, start_at, end_at, filters = {}) { const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); @@ -9,21 +26,57 @@ export function getWebsiteStats(website_id, start_at, end_at, filters = {}) { select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", sum(case when t.c = 1 then 1 else 0 end) as "bounces", + sum(case when m2 < m1 + interval '1 hour' then ${getTimestampInterval( + 'm2', + 'm1', + )} else 0 end) as "totaltime" sum(t.time) as "totaltime" from ( - select pageview.session_id, - ${getDateQuery('pageview.created_at', 'hour')}, - count(*) c, - ${getTimestampInterval('pageview.created_at')} as "time" - from pageview - ${joinSession} - where pageview.website_id=$1 - and pageview.created_at between $2 and $3 - ${pageviewQuery} - ${sessionQuery} - group by 1, 2 + select + pageview.session_id, + ${getDateQuery('pageview.created_at', 'hour')}, + count(*) c, + min(created_at) m1, + max(created_at) m2 + from pageview + ${joinSession} + where pageview.website_id=$1 + and pageview.created_at between $2 and $3 + ${pageviewQuery} + ${sessionQuery} + group by 1, 2 ) t `, params, ); } + +async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { + const params = [website_id]; + const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); + + return rawQueryClickhouse( + ` + select + sum(t.c) as "pageviews", + count(distinct t.session_id) as "uniques", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" + from ( + select pageview.session_id, + ${getDateQueryClickhouse('pageview.created_at', 'day')} time_series, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from pageview + ${joinSession} + where pageview.website_id = $1 + and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)} + ${pageviewQuery} + ${sessionQuery} + group by pageview.session_id, time_series + ) t; + `, + params, + ); +} From 65910c7348ff2b61673c915844642e4a983034ce Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 22 Jul 2022 14:43:19 -0700 Subject: [PATCH 4/8] clickhouse inserts --- lib/db.js | 17 +------ lib/session.js | 5 +++ pages/api/collect.js | 6 +-- pages/api/website/[id]/pageviews.js | 2 +- queries/analytics/event/saveEvent.js | 35 +++++++++++++-- .../analytics/pageview/getPageviewMetrics.js | 1 + .../analytics/pageview/getPageviewStats.js | 13 ++++-- queries/analytics/pageview/savePageView.js | 34 ++++++++++++-- queries/analytics/session/createSession.js | 44 ++++++++++++++++++- queries/analytics/session/getSessionByUuid.js | 3 +- .../analytics/session/getSessionMetrics.js | 11 +++-- queries/analytics/stats/getActiveVisitors.js | 2 +- queries/analytics/stats/getWebsiteStats.js | 13 ++++-- 13 files changed, 146 insertions(+), 40 deletions(-) diff --git a/lib/db.js b/lib/db.js index 9409178a..93a54bad 100644 --- a/lib/db.js +++ b/lib/db.js @@ -72,19 +72,6 @@ function initializeClickhouse() { database, }, }); - - // return new ClickHouse({ - // url: 'http://164.92.95.2', - // port: 8123, - // basicAuth: { - // username: 'default', - // password: 'shhhthisissupersecret!', - // }, - // format: 'json', - // config: { - // database: 'umami_dev', - // }, - // }); } const prisma = initializePrisma(options); @@ -243,7 +230,7 @@ export function getFilterQuery(table, filters = {}, params = []) { return query.join('\n'); } -export function parseFilters(table, filters = {}, params = []) { +export function parseFilters(table, filters = {}, params = [], sessionKey = 'session_id') { const { domain, url, event_url, referrer, os, browser, device, country, event_type } = filters; const pageviewFilters = { domain, url, referrer }; @@ -257,7 +244,7 @@ export function parseFilters(table, filters = {}, params = []) { event: { event_type }, joinSession: os || browser || device || country - ? `inner join session on ${table}.session_id = session.session_id` + ? `inner join session on ${table}.${sessionKey} = session.${sessionKey}` : '', pageviewQuery: getFilterQuery('pageview', pageviewFilters, params), sessionQuery: getFilterQuery('session', sessionFilters, params), diff --git a/lib/session.js b/lib/session.js index 15a349fd..edb8aeeb 100644 --- a/lib/session.js +++ b/lib/session.js @@ -37,6 +37,8 @@ export async function getSession(req) { let session = await getSessionByUuid(session_uuid); + session = Array.isArray(session) && session[0] ? session[0] : null; + if (!session) { try { session = await createSession(website_id, { @@ -50,6 +52,8 @@ export async function getSession(req) { device, }); + console.log(session); + if (!session) { return null; } @@ -65,5 +69,6 @@ export async function getSession(req) { return { website_id, session_id, + session_uuid, }; } diff --git a/pages/api/collect.js b/pages/api/collect.js index acd031a9..4ff533b9 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -60,7 +60,7 @@ export default async (req, res) => { await useSession(req, res); const { - session: { website_id, session_id }, + session: { website_id, session_id, session_uuid }, } = req; const { type, payload } = getJsonBody(req); @@ -72,9 +72,9 @@ export default async (req, res) => { } if (type === 'pageview') { - await savePageView(website_id, session_id, url, referrer); + await savePageView(website_id, { session_id, session_uuid, url, referrer }); } else if (type === 'event') { - await saveEvent(website_id, session_id, url, event_type, event_value); + await saveEvent(website_id, { session_id, session_uuid, url, event_type, event_value }); } else { return badRequest(res); } diff --git a/pages/api/website/[id]/pageviews.js b/pages/api/website/[id]/pageviews.js index 78145e8b..3bd572d9 100644 --- a/pages/api/website/[id]/pageviews.js +++ b/pages/api/website/[id]/pageviews.js @@ -34,7 +34,7 @@ export default async (req, res) => { device, country, }), - getPageviewStats(websiteId, startDate, endDate, tz, unit, 'distinct pageview.session_id', { + getPageviewStats(websiteId, startDate, endDate, tz, unit, 'distinct pageview.', { url, os, browser, diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 17379135..abb32cbb 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -1,8 +1,20 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL, URL_LENGTH } from 'lib/constants'; +import { + getDateFormatClickhouse, + prisma, + rawQueryClickhouse, + runAnalyticsQuery, + runQuery, +} from 'lib/db'; -import { URL_LENGTH } from 'lib/constants'; +export async function saveEvent(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} -export async function saveEvent(website_id, session_id, url, event_type, event_value) { +async function relationalQuery(website_id, { session_id, url, event_type, event_value }) { return runQuery( prisma.event.create({ data: { @@ -15,3 +27,20 @@ export async function saveEvent(website_id, session_id, url, event_type, event_v }), ); } + +async function clickhouseQuery(website_id, { session_uuid, url, event_type, event_value }) { + const params = [ + website_id, + session_uuid, + url?.substr(0, URL_LENGTH), + event_type?.substr(0, 50), + event_value?.substr(0, 50), + ]; + + return rawQueryClickhouse( + ` + insert into umami_dev.event (created_at, website_id, session_uuid, url, event_type, event_value) + values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5);`, + params, + ); +} diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index fb2cf8a5..3b16d703 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -45,6 +45,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, table, filte table, filters, params, + 'session_uuid', ); return rawQueryClickhouse( diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 8a26280c..980c6a86 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -26,6 +26,7 @@ async function relationalQuery( unit = 'day', count = '*', filters = {}, + sessionKey = 'session_id', ) { const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); @@ -37,7 +38,7 @@ async function relationalQuery( g.y as y from (select ${getDateQuery('pageview.created_at', unit, timezone)} t, - count(${count}) y + count(${count != '*' ? `${count}${sessionKey}` : count}) y from pageview ${joinSession} where pageview.website_id=$1 @@ -59,9 +60,15 @@ async function clickhouseQuery( unit = 'day', count = '*', filters = {}, + sessionKey = 'session_uuid', ) { const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); + const { pageviewQuery, sessionQuery, joinSession } = parseFilters( + 'pageview', + filters, + params, + sessionKey, + ); return rawQueryClickhouse( ` @@ -71,7 +78,7 @@ async function clickhouseQuery( from (select ${getDateQueryClickhouse('created_at', unit, timezone)} t, - count(${count}) y + count(${count != '*' ? `${count}${sessionKey}` : count}) y from pageview ${joinSession} where pageview.website_id= $1 diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index 2061e375..72ab35fb 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -1,8 +1,20 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL, URL_LENGTH } from 'lib/constants'; +import { + getDateFormatClickhouse, + prisma, + rawQueryClickhouse, + runAnalyticsQuery, + runQuery, +} from 'lib/db'; -import { URL_LENGTH } from 'lib/constants'; +export async function savePageView(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} -export async function savePageView(website_id, session_id, url, referrer) { +async function relationalQuery(website_id, { session_id, url, referrer }) { return runQuery( prisma.pageview.create({ data: { @@ -14,3 +26,19 @@ export async function savePageView(website_id, session_id, url, referrer) { }), ); } + +async function clickhouseQuery(website_id, { session_uuid, url, referrer }) { + const params = [ + website_id, + session_uuid, + url?.substr(0, URL_LENGTH), + referrer?.substr(0, URL_LENGTH), + ]; + + return rawQueryClickhouse( + ` + insert into umami_dev.pageview (created_at, website_id, session_uuid, url, referrer) + values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`, + params, + ); +} diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index f22aa2b0..22b41ebd 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -1,6 +1,21 @@ -import { prisma, runQuery } from 'lib/db'; +import { CLICKHOUSE, RELATIONAL } from 'lib/constants'; +import { + getDateFormatClickhouse, + prisma, + rawQueryClickhouse, + runAnalyticsQuery, + runQuery, +} from 'lib/db'; +import { getSessionByUuid } from 'queries'; -export async function createSession(website_id, data) { +export async function createSession(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(website_id, data) { return runQuery( prisma.session.create({ data: { @@ -13,3 +28,28 @@ export async function createSession(website_id, data) { }), ); } + +async function clickhouseQuery( + website_id, + { session_uuid, hostname, browser, os, screen, language, country, device }, +) { + const params = [ + session_uuid, + website_id, + hostname, + browser, + os, + device, + screen, + language, + country ? country : null, + ]; + + await rawQueryClickhouse( + `insert into umami_dev.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country) + values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`, + params, + ); + + return getSessionByUuid(session_uuid); +} diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js index 042b4914..fde70e7f 100644 --- a/queries/analytics/session/getSessionByUuid.js +++ b/queries/analytics/session/getSessionByUuid.js @@ -24,7 +24,6 @@ async function clickhouseQuery(session_uuid) { return rawQueryClickhouse( ` select - session_id, session_uuid, website_id, created_at, @@ -36,7 +35,7 @@ async function clickhouseQuery(session_uuid) { "language", country from session - where session_id = $1 + where session_uuid = $1 `, params, ); diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index 432b696c..ee6d1dc0 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -40,14 +40,19 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {} async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) { const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); + const { pageviewQuery, sessionQuery, joinSession } = parseFilters( + 'pageview', + filters, + params, + 'session_uuid', + ); return rawQueryClickhouse( ` select ${field} x, count(*) y from session as x - where x.session_id in ( - select pageview.session_id + where x.session_uuid in ( + select pageview.session_uuid from pageview ${joinSession} where pageview.website_id=$1 diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index 2789f769..04884521 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -29,7 +29,7 @@ async function clickhouseQuery(website_id) { return rawQueryClickhouse( ` - select count(distinct session_id) x + select count(distinct session_uuid) x from pageview where website_id = $1 and created_at >= ${getDateFormatClickhouse(subMinutes(new Date(), 5))} diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index f1ee921a..574a877d 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -53,17 +53,22 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) { async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params); + const { pageviewQuery, sessionQuery, joinSession } = parseFilters( + 'pageview', + filters, + params, + 'session_uuid', + ); return rawQueryClickhouse( ` select sum(t.c) as "pageviews", - count(distinct t.session_id) as "uniques", + count(distinct t.session_uuid) as "uniques", sum(if(t.c = 1, 1, 0)) as "bounces", sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" from ( - select pageview.session_id, + select pageview.session_uuid, ${getDateQueryClickhouse('pageview.created_at', 'day')} time_series, count(*) c, min(created_at) min_time, @@ -74,7 +79,7 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} - group by pageview.session_id, time_series + group by pageview.session_uuid, time_series ) t; `, params, From 810ae101ed1fa911e222bb6bdb4843c21f60cf28 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 22 Jul 2022 17:24:45 -0700 Subject: [PATCH 5/8] fix timer --- components/metrics/ActiveUsers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/metrics/ActiveUsers.js b/components/metrics/ActiveUsers.js index becef9fe..653fc783 100644 --- a/components/metrics/ActiveUsers.js +++ b/components/metrics/ActiveUsers.js @@ -5,7 +5,7 @@ import useFetch from 'hooks/useFetch'; import Dot from 'components/common/Dot'; import styles from './ActiveUsers.module.css'; -export default function ActiveUsers({ websiteId, className, value, interval = 60000000 }) { +export default function ActiveUsers({ websiteId, className, value, interval = 60000 }) { const url = websiteId ? `/website/${websiteId}/active` : null; const { data } = useFetch(url, { interval, From aa8057f9fed72143bee3859e853b553e05a06837 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 22 Jul 2022 17:47:26 -0700 Subject: [PATCH 6/8] fix stats query --- lib/db.js | 4 ++-- lib/session.js | 2 -- queries/analytics/stats/getWebsiteStats.js | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/db.js b/lib/db.js index 894c60a2..08e76261 100644 --- a/lib/db.js +++ b/lib/db.js @@ -146,11 +146,11 @@ export function getTimestampInterval(maxColumn, minColumn) { const db = getDatabase(); if (db === POSTGRESQL) { - return `floor(extract(epoch from max(${maxColumn}) - min(${minColumn})))`; + return `floor(extract(epoch from (${maxColumn}) - (${minColumn})))`; } if (db === MYSQL) { - return `floor(unix_timestamp(max(${maxColumn})) - unix_timestamp(min(${minColumn})))`; + return `floor(unix_timestamp(${maxColumn}) - unix_timestamp(${minColumn}))`; } } diff --git a/lib/session.js b/lib/session.js index edb8aeeb..4e0513b1 100644 --- a/lib/session.js +++ b/lib/session.js @@ -52,8 +52,6 @@ export async function getSession(req) { device, }); - console.log(session); - if (!session) { return null; } diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 818be97c..815bacdd 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -35,7 +35,6 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) { 'm2', 'm1', )} else 0 end) as "totaltime" - sum(t.time) as "totaltime" from ( select pageview.session_id, From 1c26581cd1172874aa07655e517d322e9ae06016 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 22 Jul 2022 19:50:43 -0700 Subject: [PATCH 7/8] fix parseFilters --- .../analytics/pageview/getPageviewParams.js | 25 +++++++++++-------- .../analytics/pageview/getPageviewStats.js | 1 + .../analytics/session/getSessionMetrics.js | 1 + queries/analytics/stats/getWebsiteStats.js | 1 + 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/queries/analytics/pageview/getPageviewParams.js b/queries/analytics/pageview/getPageviewParams.js index 827141b9..fd25c77e 100644 --- a/queries/analytics/pageview/getPageviewParams.js +++ b/queries/analytics/pageview/getPageviewParams.js @@ -1,15 +1,14 @@ -import { getDatabase, parseFilters, rawQuery } from 'lib/db'; -import { MYSQL, POSTGRESQL } from 'lib/constants'; +import { getDatabase, parseFilters, rawQuery, runAnalyticsQuery } from 'lib/db'; +import { MYSQL, POSTGRESQL, CLICKHOUSE, RELATIONAL } from 'lib/constants'; -export function getPageviewParams( - param, - website_id, - start_at, - end_at, - column, - table, - filters = {}, -) { +export async function getPageviewParams(...args) { + return runAnalyticsQuery({ + [`${RELATIONAL}`]: () => relationalQuery(...args), + [`${CLICKHOUSE}`]: () => clickhouseQuery(...args), + }); +} + +function relationalQuery(param, website_id, start_at, end_at, column, table, filters = {}) { const params = [param, website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, @@ -43,3 +42,7 @@ export function getPageviewParams( params, ); } + +function clickhouseQuery() { + return Promise.reject(new Error('Not implemented.')); +} diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 1591e5d2..1cca2fca 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -64,6 +64,7 @@ async function clickhouseQuery( const params = [website_id]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', + null, filters, params, sessionKey, diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index 5defb207..b9bbc708 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -47,6 +47,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {} const params = [website_id]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', + null, filters, params, 'session_uuid', diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 815bacdd..53e90b51 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -59,6 +59,7 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { const params = [website_id]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', + null, filters, params, 'session_uuid', From f569055771024d3eee4e9740fe2ecf1f164ecf02 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 22 Jul 2022 22:42:01 -0700 Subject: [PATCH 8/8] PR comments --- lib/db.js | 14 ++++----- .../analytics/pageview/getPageviewStats.js | 4 +-- queries/analytics/stats/getWebsiteStats.js | 29 ++++++++----------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/lib/db.js b/lib/db.js index 08e76261..10a08296 100644 --- a/lib/db.js +++ b/lib/db.js @@ -29,7 +29,7 @@ function logQuery(e) { console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`)); } -function initializePrisma(options) { +function getPrismaClient(options) { const prisma = new PrismaClient(options); if (process.env.LOG_QUERY) { @@ -39,7 +39,7 @@ function initializePrisma(options) { return prisma; } -function initializeClickhouse() { +function getClickhouseClient() { if (!process.env.ANALYTICS_URL) { return null; } @@ -63,8 +63,8 @@ function initializeClickhouse() { }); } -const prisma = global.prisma || initializePrisma(options); -const clickhouse = global.clickhouse || initializeClickhouse(); +const prisma = global.prisma || getPrismaClient(options); +const clickhouse = global.clickhouse || getClickhouseClient(); if (process.env.NODE_ENV !== 'production') { global.prisma = prisma; @@ -142,15 +142,15 @@ export function getBetweenDatesClickhouse(field, start_at, end_at) { and ${getDateFormatClickhouse(end_at)}`; } -export function getTimestampInterval(maxColumn, minColumn) { +export function getTimestampInterval(field) { const db = getDatabase(); if (db === POSTGRESQL) { - return `floor(extract(epoch from (${maxColumn}) - (${minColumn})))`; + return `floor(extract(epoch from max(${field}) - min(${field})))`; } if (db === MYSQL) { - return `floor(unix_timestamp(${maxColumn}) - unix_timestamp(${minColumn}))`; + return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`; } } diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 1cca2fca..0a320074 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -38,7 +38,7 @@ async function relationalQuery( return rawQuery( ` select ${getDateQuery('pageview.created_at', unit, timezone)} t, - count(${count != '*' ? `${count}${sessionKey}` : count}) y + count(${count !== '*' ? `${count}${sessionKey}` : count}) y from pageview ${joinSession} where pageview.website_id=$1 @@ -78,7 +78,7 @@ async function clickhouseQuery( from (select ${getDateQueryClickhouse('created_at', unit, timezone)} t, - count(${count != '*' ? `${count}${sessionKey}` : count}) y + count(${count !== '*' ? `${count}${sessionKey}` : count}) y from pageview ${joinSession} where pageview.website_id= $1 diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 53e90b51..9328b85b 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -31,24 +31,19 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) { select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", sum(case when t.c = 1 then 1 else 0 end) as "bounces", - sum(case when m2 < m1 + interval '1 hour' then ${getTimestampInterval( - 'm2', - 'm1', - )} else 0 end) as "totaltime" + sum(t.time) as "totaltime" from ( - select - pageview.session_id, - ${getDateQuery('pageview.created_at', 'hour')}, - count(*) c, - min(created_at) m1, - max(created_at) m2 - from pageview - ${joinSession} - where pageview.website_id=$1 - and pageview.created_at between $2 and $3 - ${pageviewQuery} - ${sessionQuery} - group by 1, 2 + select pageview.session_id, + ${getDateQuery('pageview.created_at', 'hour')}, + count(*) c, + ${getTimestampInterval('pageview.created_at')} as "time" + from pageview + ${joinSession} + where pageview.website_id=$1 + and pageview.created_at between $2 and $3 + ${pageviewQuery} + ${sessionQuery} + group by 1, 2 ) t `, params,