From 96add409b69f76a1a910199143b87957017107f1 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Sat, 8 Oct 2022 16:12:33 -0700 Subject: [PATCH 1/5] update api to new CH columns --- db/clickhouse/schema.sql | 18 +++++++++--------- lib/clickhouse.js | 5 +++++ lib/session.js | 3 ++- pages/api/collect.js | 13 +++++++++---- pages/api/websites/[id]/active.js | 3 ++- pages/api/websites/[id]/events.js | 3 ++- pages/api/websites/[id]/metrics.js | 3 ++- pages/api/websites/[id]/pageviews.js | 5 +++-- pages/api/websites/[id]/stats.js | 5 +++-- queries/analytics/event/getEventMetrics.js | 4 ++-- queries/analytics/event/getEvents.js | 12 ++++++++---- queries/analytics/event/saveEvent.js | 4 ++-- .../analytics/pageview/getPageviewMetrics.js | 4 ++-- queries/analytics/pageview/getPageviewStats.js | 6 +++--- queries/analytics/pageview/getPageviews.js | 10 ++++++++-- queries/analytics/pageview/savePageView.js | 4 ++-- queries/analytics/session/createSession.js | 4 ++-- queries/analytics/stats/getActiveVisitors.js | 6 +++--- queries/analytics/stats/getWebsiteStats.js | 10 +++++----- 19 files changed, 74 insertions(+), 48 deletions(-) diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 6bcf899c..a6e68c62 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -3,9 +3,9 @@ SET allow_experimental_object_type = 1; -- Create Event CREATE TABLE event ( - website_id UInt32, - session_uuid UUID, - event_uuid Nullable(UUID), + website_id UUID, + session_id UUID, + event_id Nullable(UUID), --session hostname LowCardinality(String), browser LowCardinality(String), @@ -27,9 +27,9 @@ CREATE TABLE event SETTINGS index_granularity = 8192; CREATE TABLE event_queue ( - website_id UInt32, - session_uuid UUID, - event_uuid Nullable(UUID), + website_id UUID, + session_id UUID, + event_id Nullable(UUID), url String, referrer String, hostname LowCardinality(String), @@ -52,9 +52,9 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro kafka_skip_broken_messages = 1; CREATE MATERIALIZED VIEW event_queue_mv TO event AS -SELECT website_id, - session_uuid, - event_uuid, +SELECT website_id UUID, + session_id UUID, + event_id, url, referrer, hostname, diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 9913c0be..d6b6260b 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -60,6 +60,10 @@ function getDateFormat(date) { return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; } +function getCommaSeparatedStringFormat(data, column) { + return data.map(obj => `'${obj[column]}'`).join(','); +} + function getBetweenDates(field, start_at, end_at) { return `${field} between ${getDateFormat(start_at)} and ${getDateFormat(end_at)}`; @@ -180,6 +184,7 @@ export default { getDateStringQuery, getDateQuery, getDateFormat, + getCommaSeparatedStringFormat, getBetweenDates, getFilterQuery, parseFilters, diff --git a/lib/session.js b/lib/session.js index 9e95cb11..61954168 100644 --- a/lib/session.js +++ b/lib/session.js @@ -66,7 +66,7 @@ export async function getSession(req) { if (!sessionId) { try { - session = await createSession(websiteId, { + session = await createSession(websiteId, website_uuid, { session_uuid, hostname, browser, @@ -98,6 +98,7 @@ export async function getSession(req) { return { website_id: websiteId, + website_uuid: website_uuid, session, }; } diff --git a/pages/api/collect.js b/pages/api/collect.js index f253d09e..5899035b 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -59,7 +59,7 @@ export default async (req, res) => { await useSession(req, res); const { - session: { website_id, session }, + session: { website_id, website_uuid, session }, } = req; const { type, payload } = getJsonBody(req); @@ -73,9 +73,9 @@ export default async (req, res) => { const event_uuid = uuid(); if (type === 'pageview') { - await savePageView(website_id, { session, url, referrer }); + await savePageView(website_id, website_uuid, { session, url, referrer }); } else if (type === 'event') { - await saveEvent(website_id, { + await saveEvent(website_id, website_uuid, { session, event_uuid, url, @@ -87,7 +87,12 @@ export default async (req, res) => { } const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + { + website_id, + website_uuid, + session_id: session.session_id, + session_uuid: session.session_uuid, + }, secret(), ); diff --git a/pages/api/websites/[id]/active.js b/pages/api/websites/[id]/active.js index 20550427..845f3b31 100644 --- a/pages/api/websites/[id]/active.js +++ b/pages/api/websites/[id]/active.js @@ -14,8 +14,9 @@ export default async (req, res) => { const { id } = req.query; const websiteId = +id; + const website_uuid = id; - const result = await getActiveVisitors(websiteId); + const result = await getActiveVisitors(websiteId, website_uuid); return ok(res, result); } diff --git a/pages/api/websites/[id]/events.js b/pages/api/websites/[id]/events.js index c633a585..93d01ff9 100644 --- a/pages/api/websites/[id]/events.js +++ b/pages/api/websites/[id]/events.js @@ -21,10 +21,11 @@ export default async (req, res) => { } const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); - const events = await getEventMetrics(websiteId, startDate, endDate, tz, unit, { + const events = await getEventMetrics(websiteId, website_uuid, startDate, endDate, tz, unit, { url, event_name, }); diff --git a/pages/api/websites/[id]/metrics.js b/pages/api/websites/[id]/metrics.js index 206209c6..7b596f8e 100644 --- a/pages/api/websites/[id]/metrics.js +++ b/pages/api/websites/[id]/metrics.js @@ -44,6 +44,7 @@ export default async (req, res) => { const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -106,7 +107,7 @@ export default async (req, res) => { query: type === 'query' && table !== 'event' ? true : undefined, }; - const data = await getPageviewMetrics(websiteId, { + const data = await getPageviewMetrics(websiteId, website_uuid, { startDate, endDate, column, diff --git a/pages/api/websites/[id]/pageviews.js b/pages/api/websites/[id]/pageviews.js index f00fffa1..2b492233 100644 --- a/pages/api/websites/[id]/pageviews.js +++ b/pages/api/websites/[id]/pageviews.js @@ -18,6 +18,7 @@ export default async (req, res) => { req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -26,7 +27,7 @@ export default async (req, res) => { } const [pageviews, sessions] = await Promise.all([ - getPageviewStats(websiteId, { + getPageviewStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, timezone: tz, @@ -41,7 +42,7 @@ export default async (req, res) => { country, }, }), - getPageviewStats(websiteId, { + getPageviewStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, timezone: tz, diff --git a/pages/api/websites/[id]/stats.js b/pages/api/websites/[id]/stats.js index a7741af5..54b4ae75 100644 --- a/pages/api/websites/[id]/stats.js +++ b/pages/api/websites/[id]/stats.js @@ -14,6 +14,7 @@ export default async (req, res) => { const { id, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -21,7 +22,7 @@ export default async (req, res) => { const prevStartDate = new Date(+start_at - distance); const prevEndDate = new Date(+end_at - distance); - const metrics = await getWebsiteStats(websiteId, { + const metrics = await getWebsiteStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, filters: { @@ -33,7 +34,7 @@ export default async (req, res) => { country, }, }); - const prevPeriod = await getWebsiteStats(websiteId, { + const prevPeriod = await getWebsiteStats(websiteId, website_uuid, { start_at: prevStartDate, end_at: prevEndDate, filters: { diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index c37f1b39..014b645c 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -36,7 +36,7 @@ async function relationalQuery( } async function clickhouseQuery( - website_id, + website_uuid, start_at, end_at, timezone = 'UTC', @@ -44,7 +44,7 @@ async function clickhouseQuery( filters = {}, ) { const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; return rawQuery( `select diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 441eed64..7238ecea 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -25,19 +25,23 @@ function relationalQuery(websites, start_at) { } function clickhouseQuery(websites, start_at) { - const { rawQuery, getDateFormat } = clickhouse; + const { rawQuery, getDateFormat, getCommaSeparatedStringFormat } = clickhouse; return rawQuery( `select - event_uuid, + event_id, website_id, - session_uuid, + session_id, created_at, url, event_name from event where event_name != '' - and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and ${ + websites && websites.length > 0 + ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + : '0 = 0' + } and created_at >= ${getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 3b10f4cb..a319e646 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -32,14 +32,14 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_ } async function clickhouseQuery( - website_id, + website_uuid, { session: { country, ...sessionArgs }, event_uuid, url, event_name, event_data }, ) { const { getDateFormat, sendMessage } = kafka; const params = { event_uuid, - website_id, + website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: event_name?.substring(0, EVENT_NAME_LENGTH), diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index bea3502c..0d7b4024 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -34,9 +34,9 @@ async function relationalQuery(website_id, { startDate, endDate, column, table, ); } -async function clickhouseQuery(website_id, { startDate, endDate, column, filters = {} }) { +async function clickhouseQuery(website_uuid, { startDate, endDate, column, filters = {} }) { const { rawQuery, parseFilters, getBetweenDates } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params); return rawQuery( diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 4f74cf98..c6cc0163 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -45,11 +45,11 @@ async function relationalQuery( } async function clickhouseQuery( - website_id, + website_uuid, { start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} }, ) { const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( @@ -59,7 +59,7 @@ async function clickhouseQuery( from (select ${getDateQuery('created_at', unit, timezone)} t, - count(${count !== '*' ? 'distinct session_uuid' : count}) y + count(${count !== '*' ? 'distinct session_id' : count}) y from event where event_name = '' and website_id= $1 diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index bc909b72..45a5865f 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -25,15 +25,21 @@ async function relationalQuery(websites, start_at) { } async function clickhouseQuery(websites, start_at) { + const { getCommaSeparatedStringFormat } = clickhouse; + return clickhouse.rawQuery( `select website_id, - session_uuid, + session_id, created_at, url from event where event_name = '' - and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and ${ + websites && websites.length > 0 + ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + : '0 = 0' + } and created_at >= ${clickhouse.getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index 826d9275..5f861a15 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -22,12 +22,12 @@ async function relationalQuery(website_id, { session: { session_id }, url, refer } async function clickhouseQuery( - website_id, + website_uuid, { session: { country, ...sessionArgs }, url, referrer }, ) { const { getDateFormat, sendMessage } = kafka; const params = { - website_id: website_id, + website_id: website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), referrer: referrer?.substring(0, URL_LENGTH), diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index 5130b960..0f7d32c6 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -39,14 +39,14 @@ async function relationalQuery(website_id, data) { } async function clickhouseQuery( - website_id, + website_uuid, { session_uuid, hostname, browser, os, screen, language, country, device }, ) { const { getDateFormat, sendMessage } = kafka; const params = { session_uuid, - website_id, + website_uuid, created_at: getDateFormat(new Date()), hostname, browser, diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index efc6a8c2..09cfcdfc 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -23,12 +23,12 @@ async function relationalQuery(website_id) { ); } -async function clickhouseQuery(website_id) { +async function clickhouseQuery(website_uuid) { const { rawQuery, getDateFormat } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; return rawQuery( - `select count(distinct session_uuid) x + `select count(distinct session_id) x from event where website_id = $1 and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`, diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 0c3d432a..17ec8d69 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -41,19 +41,19 @@ async function relationalQuery(website_id, { start_at, end_at, filters = {} }) { ); } -async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { +async function clickhouseQuery(website_uuid, { start_at, end_at, filters = {} }) { const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select sum(t.c) as "pageviews", - count(distinct t.session_uuid) as "uniques", + count(distinct t.session_id) as "uniques", sum(if(t.c = 1, 1, 0)) as "bounces", sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" from ( - select session_uuid, + select session_id, ${getDateQuery('created_at', 'day')} time_series, count(*) c, min(created_at) min_time, @@ -64,7 +64,7 @@ async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { and ${getBetweenDates('created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} - group by session_uuid, time_series + group by session_id, time_series ) t;`, params, ); From 4073d57e64ebcb8765f28af6d053de52f8b3d30d Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 10 Oct 2022 17:01:48 -0700 Subject: [PATCH 2/5] roll back id adds --- lib/clickhouse.js | 4 ++-- pages/api/websites/[id]/active.js | 3 +-- pages/api/websites/[id]/events.js | 3 +-- pages/api/websites/[id]/metrics.js | 3 +-- pages/api/websites/[id]/pageviews.js | 5 ++--- pages/api/websites/[id]/stats.js | 7 +++---- queries/analytics/event/getEventMetrics.js | 1 - queries/analytics/event/getEvents.js | 2 +- queries/analytics/event/saveEvent.js | 6 ++---- queries/analytics/pageview/getPageviews.js | 2 +- 10 files changed, 14 insertions(+), 22 deletions(-) diff --git a/lib/clickhouse.js b/lib/clickhouse.js index d6b6260b..a5c977fd 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -60,8 +60,8 @@ function getDateFormat(date) { return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; } -function getCommaSeparatedStringFormat(data, column) { - return data.map(obj => `'${obj[column]}'`).join(','); +function getCommaSeparatedStringFormat(data) { + return data.map(a => `'${a}'`).join(','); } function getBetweenDates(field, start_at, end_at) { diff --git a/pages/api/websites/[id]/active.js b/pages/api/websites/[id]/active.js index 845f3b31..20550427 100644 --- a/pages/api/websites/[id]/active.js +++ b/pages/api/websites/[id]/active.js @@ -14,9 +14,8 @@ export default async (req, res) => { const { id } = req.query; const websiteId = +id; - const website_uuid = id; - const result = await getActiveVisitors(websiteId, website_uuid); + const result = await getActiveVisitors(websiteId); return ok(res, result); } diff --git a/pages/api/websites/[id]/events.js b/pages/api/websites/[id]/events.js index d2d5bbe6..308e1efc 100644 --- a/pages/api/websites/[id]/events.js +++ b/pages/api/websites/[id]/events.js @@ -21,11 +21,10 @@ export default async (req, res) => { } const websiteId = +id; - const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); - const events = await getEventMetrics(websiteId, website_uuid, startDate, endDate, tz, unit, { + const events = await getEventMetrics(websiteId, startDate, endDate, tz, unit, { url, eventName: event_name, }); diff --git a/pages/api/websites/[id]/metrics.js b/pages/api/websites/[id]/metrics.js index 7b596f8e..206209c6 100644 --- a/pages/api/websites/[id]/metrics.js +++ b/pages/api/websites/[id]/metrics.js @@ -44,7 +44,6 @@ export default async (req, res) => { const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; - const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -107,7 +106,7 @@ export default async (req, res) => { query: type === 'query' && table !== 'event' ? true : undefined, }; - const data = await getPageviewMetrics(websiteId, website_uuid, { + const data = await getPageviewMetrics(websiteId, { startDate, endDate, column, diff --git a/pages/api/websites/[id]/pageviews.js b/pages/api/websites/[id]/pageviews.js index 2b492233..f00fffa1 100644 --- a/pages/api/websites/[id]/pageviews.js +++ b/pages/api/websites/[id]/pageviews.js @@ -18,7 +18,6 @@ export default async (req, res) => { req.query; const websiteId = +id; - const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -27,7 +26,7 @@ export default async (req, res) => { } const [pageviews, sessions] = await Promise.all([ - getPageviewStats(websiteId, website_uuid, { + getPageviewStats(websiteId, { start_at: startDate, end_at: endDate, timezone: tz, @@ -42,7 +41,7 @@ export default async (req, res) => { country, }, }), - getPageviewStats(websiteId, website_uuid, { + getPageviewStats(websiteId, { start_at: startDate, end_at: endDate, timezone: tz, diff --git a/pages/api/websites/[id]/stats.js b/pages/api/websites/[id]/stats.js index 5aeb3810..ee1403f8 100644 --- a/pages/api/websites/[id]/stats.js +++ b/pages/api/websites/[id]/stats.js @@ -1,4 +1,4 @@ -import { getWebsiteStats, getWebsiteById } from 'queries'; +import { getWebsiteStats } from 'queries'; import { methodNotAllowed, ok, unauthorized } from 'next-basics'; import { allowQuery } from 'lib/auth'; import { useCors } from 'lib/middleware'; @@ -14,7 +14,6 @@ export default async (req, res) => { const { website_id, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +website_id; - let websiteUuid = (await getWebsiteById(websiteId)).websiteUuid; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -22,7 +21,7 @@ export default async (req, res) => { const prevStartDate = new Date(+start_at - distance); const prevEndDate = new Date(+end_at - distance); - const metrics = await getWebsiteStats(websiteUuid, { + const metrics = await getWebsiteStats(websiteId, { start_at: startDate, end_at: endDate, filters: { @@ -34,7 +33,7 @@ export default async (req, res) => { country, }, }); - const prevPeriod = await getWebsiteStats(websiteUuid, { + const prevPeriod = await getWebsiteStats(websiteId, { start_at: prevStartDate, end_at: prevEndDate, filters: { diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 1a8f262f..447e1cc8 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -37,7 +37,6 @@ async function relationalQuery( async function clickhouseQuery( websiteId, - website_uuid, start_at, end_at, timezone = 'UTC', diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 6f15686f..dde19992 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -39,7 +39,7 @@ function clickhouseQuery(websites, start_at) { where event_name != '' and ${ websites && websites.length > 0 - ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + ? `website_id in (${getCommaSeparatedStringFormat(websites)})` : '0 = 0' } and created_at >= ${getDateFormat(start_at)}`, diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index e2948237..3c3725c3 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -34,15 +34,13 @@ async function relationalQuery(websiteId, { sessionId, url, eventName, eventData async function clickhouseQuery( websiteId, { session: { country, sessionUuid, ...sessionArgs }, eventUuid, url, eventName, eventData }, - website_uuid, ) { const { getDateFormat, sendMessage } = kafka; const params = { - session_uuid: sessionUuid, - event_uuid: eventUuid, + session_id: sessionUuid, + event_id: eventUuid, website_id: websiteId, - website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: eventName?.substring(0, EVENT_NAME_LENGTH), diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index 69176d81..d78c86ad 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -37,7 +37,7 @@ async function clickhouseQuery(websites, start_at) { where event_name = '' and ${ websites && websites.length > 0 - ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + ? `website_id in (${getCommaSeparatedStringFormat(websites)})` : '0 = 0' } and created_at >= ${clickhouse.getDateFormat(start_at)}`, From c147742d7adbc325604a5b1530697f1ee7e784b4 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 11 Oct 2022 19:37:38 -0700 Subject: [PATCH 3/5] use uuid --- components/metrics/ActiveUsers.js | 2 +- components/metrics/WebsiteHeader.js | 8 +++---- components/pages/WebsiteList.js | 8 +++---- components/settings/WebsiteSettings.js | 2 +- pages/api/websites/[id]/active.js | 4 +--- pages/api/websites/[id]/events.js | 4 +--- pages/api/websites/[id]/index.js | 16 +++++--------- pages/api/websites/[id]/metrics.js | 22 ++++++++++++++----- pages/api/websites/[id]/pageviews.js | 16 +++++++++++--- pages/api/websites/[id]/reset.js | 3 +-- pages/api/websites/[id]/stats.js | 13 +++++++++-- queries/analytics/event/getEventMetrics.js | 8 ++++--- .../analytics/pageview/getPageviewMetrics.js | 3 ++- .../analytics/pageview/getPageviewParams.js | 3 ++- .../analytics/pageview/getPageviewStats.js | 4 +++- .../analytics/session/getSessionMetrics.js | 4 +++- queries/analytics/stats/getActiveVisitors.js | 6 +++-- queries/analytics/stats/getWebsiteStats.js | 10 +++++---- 18 files changed, 84 insertions(+), 52 deletions(-) diff --git a/components/metrics/ActiveUsers.js b/components/metrics/ActiveUsers.js index 0ab266d3..6b7c4779 100644 --- a/components/metrics/ActiveUsers.js +++ b/components/metrics/ActiveUsers.js @@ -5,7 +5,7 @@ import useFetch from 'hooks/useFetch'; import Dot from 'components/common/Dot'; import styles from './ActiveUsers.module.css'; -export default function ActiveUsers({ websiteId, className, value, interval = 60000 }) { +export default function ActiveUsers({ websiteId, className, value, interval = 600000 }) { const url = websiteId ? `/websites/${websiteId}/active` : null; const { data } = useFetch(url, { interval, diff --git a/components/metrics/WebsiteHeader.js b/components/metrics/WebsiteHeader.js index 15f6d0ff..1a6bdf15 100644 --- a/components/metrics/WebsiteHeader.js +++ b/components/metrics/WebsiteHeader.js @@ -17,8 +17,8 @@ export default function WebsiteHeader({ websiteId, title, domain, showLink = fal {title} @@ -41,8 +41,8 @@ export default function WebsiteHeader({ websiteId, title, domain, showLink = fal {showLink && ( } size="small" diff --git a/components/pages/WebsiteList.js b/components/pages/WebsiteList.js index dfd041a1..24ac8bec 100644 --- a/components/pages/WebsiteList.js +++ b/components/pages/WebsiteList.js @@ -27,7 +27,7 @@ export default function WebsiteList({ websites, showCharts, limit }) { const ordered = useMemo( () => websites - .map(website => ({ ...website, order: websiteOrder.indexOf(website.id) || 0 })) + .map(website => ({ ...website, order: websiteOrder.indexOf(website.websiteUuid) || 0 })) .sort(firstBy('order')), [websites, websiteOrder], ); @@ -46,11 +46,11 @@ export default function WebsiteList({ websites, showCharts, limit }) { return (
- {ordered.map(({ id, name, domain }, index) => + {ordered.map(({ websiteUuid, name, domain }, index) => index < limit ? ( -
+
( - + {name} diff --git a/pages/api/websites/[id]/active.js b/pages/api/websites/[id]/active.js index 20550427..c29f9701 100644 --- a/pages/api/websites/[id]/active.js +++ b/pages/api/websites/[id]/active.js @@ -11,9 +11,7 @@ export default async (req, res) => { return unauthorized(res); } - const { id } = req.query; - - const websiteId = +id; + const { id: websiteId } = req.query; const result = await getActiveVisitors(websiteId); diff --git a/pages/api/websites/[id]/events.js b/pages/api/websites/[id]/events.js index 308e1efc..c1f96b2e 100644 --- a/pages/api/websites/[id]/events.js +++ b/pages/api/websites/[id]/events.js @@ -14,13 +14,11 @@ export default async (req, res) => { return unauthorized(res); } - const { id, start_at, end_at, unit, tz, url, event_name } = req.query; + const { id: websiteId, start_at, end_at, unit, tz, url, event_name } = req.query; if (!moment.tz.zone(tz) || !unitTypes.includes(unit)) { return badRequest(res); } - - const websiteId = +id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); diff --git a/pages/api/websites/[id]/index.js b/pages/api/websites/[id]/index.js index c6b8779f..ad55a296 100644 --- a/pages/api/websites/[id]/index.js +++ b/pages/api/websites/[id]/index.js @@ -1,14 +1,10 @@ -import { methodNotAllowed, ok, unauthorized, getRandomChars } from 'next-basics'; -import { deleteWebsite, getAccount, getWebsite, updateWebsite } from 'queries'; import { allowQuery } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; -import { validate } from 'uuid'; +import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { deleteWebsite, getAccount, getWebsite, getWebsiteByUuid, updateWebsite } from 'queries'; export default async (req, res) => { - const { id } = req.query; - - const websiteId = +id; - const where = validate(id) ? { websiteUuid: id } : { id: +id }; + const { id: websiteId } = req.query; if (req.method === 'GET') { await useCors(req, res); @@ -17,7 +13,7 @@ export default async (req, res) => { return unauthorized(res); } - const website = await getWebsite(where); + const website = await getWebsiteByUuid(websiteId); return ok(res, website); } @@ -33,7 +29,7 @@ export default async (req, res) => { account = await getAccount({ accountUuid }); } - const website = await getWebsite(where); + const website = await getWebsite(websiteId); const shareId = enable_share_url ? website.shareId || getRandomChars(8) : null; @@ -48,7 +44,7 @@ export default async (req, res) => { shareId: shareId, userId: account ? account.id : +owner, }, - where, + { websiteUuid: websiteId }, ); return ok(res); diff --git a/pages/api/websites/[id]/metrics.js b/pages/api/websites/[id]/metrics.js index 206209c6..29bfdc77 100644 --- a/pages/api/websites/[id]/metrics.js +++ b/pages/api/websites/[id]/metrics.js @@ -1,8 +1,8 @@ -import { getPageviewMetrics, getSessionMetrics, getWebsiteById } from 'queries'; -import { ok, methodNotAllowed, unauthorized, badRequest } from 'next-basics'; import { allowQuery } from 'lib/auth'; -import { useCors } from 'lib/middleware'; import { FILTER_IGNORED } from 'lib/constants'; +import { useCors } from 'lib/middleware'; +import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { getPageviewMetrics, getSessionMetrics, getWebsiteByUuid } from 'queries'; const sessionColumns = ['browser', 'os', 'device', 'screen', 'country', 'language']; const pageviewColumns = ['url', 'referrer', 'query']; @@ -41,9 +41,19 @@ export default async (req, res) => { return unauthorized(res); } - const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query; + const { + id: websiteId, + type, + start_at, + end_at, + url, + referrer, + os, + browser, + device, + country, + } = req.query; - const websiteId = +id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -83,7 +93,7 @@ export default async (req, res) => { let domain; if (type === 'referrer') { - const website = await getWebsiteById(websiteId); + const website = await getWebsiteByUuid(websiteId); if (!website) { return badRequest(res); diff --git a/pages/api/websites/[id]/pageviews.js b/pages/api/websites/[id]/pageviews.js index f00fffa1..acf7f11b 100644 --- a/pages/api/websites/[id]/pageviews.js +++ b/pages/api/websites/[id]/pageviews.js @@ -14,10 +14,20 @@ export default async (req, res) => { return unauthorized(res); } - const { id, start_at, end_at, unit, tz, url, referrer, os, browser, device, country } = - req.query; + const { + id: websiteId, + start_at, + end_at, + unit, + tz, + url, + referrer, + os, + browser, + device, + country, + } = req.query; - const websiteId = +id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); diff --git a/pages/api/websites/[id]/reset.js b/pages/api/websites/[id]/reset.js index 2f5d05b4..acfc9b0e 100644 --- a/pages/api/websites/[id]/reset.js +++ b/pages/api/websites/[id]/reset.js @@ -3,8 +3,7 @@ import { methodNotAllowed, ok, unauthorized } from 'next-basics'; import { allowQuery } from 'lib/auth'; export default async (req, res) => { - const { id } = req.query; - const websiteId = +id; + const { id: websiteId } = req.query; if (req.method === 'POST') { if (!(await allowQuery(req))) { diff --git a/pages/api/websites/[id]/stats.js b/pages/api/websites/[id]/stats.js index ee1403f8..c127eb0f 100644 --- a/pages/api/websites/[id]/stats.js +++ b/pages/api/websites/[id]/stats.js @@ -11,9 +11,18 @@ export default async (req, res) => { return unauthorized(res); } - const { website_id, start_at, end_at, url, referrer, os, browser, device, country } = req.query; + const { + id: websiteId, + start_at, + end_at, + url, + referrer, + os, + browser, + device, + country, + } = req.query; - const websiteId = +website_id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 447e1cc8..edf5de8c 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -23,11 +23,13 @@ async function relationalQuery( return rawQuery( `select event_name x, - ${getDateQuery('created_at', unit, timezone)} t, + ${getDateQuery('event.created_at', unit, timezone)} t, count(*) y from event - where website_id=$1 - and created_at between $2 and $3 + join website + on event.website_id = website.website_id + where website_uuid='${websiteId}' + and event.created_at between $2 and $3 ${getFilterQuery('event', filters, params)} group by 1, 2 order by 2`, diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index 80a4e957..e1c4d43f 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -22,8 +22,9 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, f return rawQuery( `select ${column} x, count(*) y from ${table} + ${` join website on ${table}.website_id = website.website_id`} ${joinSession} - where ${table}.website_id=$1 + where website.website_uuid='${websiteId}' and ${table}.created_at between $2 and $3 ${pageviewQuery} ${joinSession && sessionQuery} diff --git a/queries/analytics/pageview/getPageviewParams.js b/queries/analytics/pageview/getPageviewParams.js index bf0d6b5e..8ec26dec 100644 --- a/queries/analytics/pageview/getPageviewParams.js +++ b/queries/analytics/pageview/getPageviewParams.js @@ -22,8 +22,9 @@ async function relationalQuery(websiteId, start_at, end_at, column, table, filte `select url x, count(*) y from ${table} + ${` join website on ${table}.website_id = website.website_id`} ${joinSession} - where ${table}.website_id=$1 + where website.website_uuid='${websiteId}' and ${table}.created_at between $2 and $3 and ${table}.url like '%?%' ${pageviewQuery} diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 5a740dbd..ceed4daf 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -34,8 +34,10 @@ async function relationalQuery( `select ${getDateQuery('pageview.created_at', unit, timezone)} t, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from pageview + join website + on pageview.website_id = website.website_id ${joinSession} - where pageview.website_id=$1 + where website.website_uuid='${websiteId}' and pageview.created_at between $2 and $3 ${pageviewQuery} ${sessionQuery} diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index 7ad7ecfe..cbf3ed58 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -20,8 +20,10 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters = where x.session_id in ( select pageview.session_id from pageview + join website + on pageview.website_id = website.website_id ${joinSession} - where pageview.website_id=$1 + where website.website_uuid='${websiteId}' and pageview.created_at between $2 and $3 ${pageviewQuery} ${sessionQuery} diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index e919760a..9d6b1f09 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -17,8 +17,10 @@ async function relationalQuery(websiteId) { return prisma.rawQuery( `select count(distinct session_id) x from pageview - where website_id = $1 - and created_at >= $2`, + join website + on pageview.website_id = website.website_id + where website.website_uuid = '${websiteId}' + and pageview.created_at >= $2`, params, ); } diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 79156ef1..b6bf7b87 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -30,11 +30,13 @@ async function relationalQuery(websiteId, { start_at, end_at, filters = {} }) { count(*) c, ${getTimestampInterval('pageview.created_at')} as "time" from pageview + join website + on pageview.website_id = website.website_id ${joinSession} - where pageview.website_id=$1 - and pageview.created_at between $2 and $3 - ${pageviewQuery} - ${sessionQuery} + where website.website_uuid='${websiteId}' + and pageview.created_at between $2 and $3 + ${pageviewQuery} + ${sessionQuery} group by 1, 2 ) t`, params, From bce4476d12b5782c4320f2b58ab8b9c04b07efe4 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 11 Oct 2022 19:43:02 -0700 Subject: [PATCH 4/5] fix reset/delete --- components/metrics/ActiveUsers.js | 2 +- components/settings/WebsiteSettings.js | 28 +++++++++++++++----------- queries/admin/website/deleteWebsite.js | 14 ++++++------- queries/admin/website/resetWebsite.js | 8 ++++---- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/components/metrics/ActiveUsers.js b/components/metrics/ActiveUsers.js index 6b7c4779..0ab266d3 100644 --- a/components/metrics/ActiveUsers.js +++ b/components/metrics/ActiveUsers.js @@ -5,7 +5,7 @@ import useFetch from 'hooks/useFetch'; import Dot from 'components/common/Dot'; import styles from './ActiveUsers.module.css'; -export default function ActiveUsers({ websiteId, className, value, interval = 600000 }) { +export default function ActiveUsers({ websiteId, className, value, interval = 60000 }) { const url = websiteId ? `/websites/${websiteId}/active` : null; const { data } = useFetch(url, { interval, diff --git a/components/settings/WebsiteSettings.js b/components/settings/WebsiteSettings.js index f044bd68..80bd1f60 100644 --- a/components/settings/WebsiteSettings.js +++ b/components/settings/WebsiteSettings.js @@ -46,7 +46,7 @@ export default function WebsiteSettings() { icon={} size="small" tooltip={} - tooltipId={`button-share-${row.id}`} + tooltipId={`button-share-${row.websiteUuid}`} onClick={() => setShowUrl(row)} /> )} @@ -56,42 +56,46 @@ export default function WebsiteSettings() { tooltip={ } - tooltipId={`button-code-${row.id}`} + tooltipId={`button-code-${row.websiteUuid}`} onClick={() => setShowCode(row)} />