From ba31f48f1abc6c0c81fc42bbde504ff4b8f5d6f7 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 21 Oct 2022 21:33:23 -0700 Subject: [PATCH] Feat/um 49 query builder api (#1573) * add uuid to event. add indexes * eventdata api * add event data * remove test data * update list --- components/common/EventDataButton.js | 48 ++++ components/common/EventDataButton.module.css | 3 + components/forms/EventDataForm.js | 262 ++++++++++++++++++ components/forms/EventDataForm.module.css | 38 +++ components/metrics/DataTable.js | 32 ++- components/metrics/FilterTags.js | 4 +- components/metrics/FilterTags.module.css | 5 +- components/metrics/WebsiteHeader.js | 9 +- components/pages/TestConsole.js | 10 +- components/pages/WebsiteDetails.js | 2 + .../migrations/04_account_uuid/migration.sql | 11 - db/mysql/migrations/04_add_uuid/migration.sql | 35 +++ db/mysql/schema.prisma | 6 + .../migration.sql | 17 +- db/postgresql/schema.prisma | 6 + lang/en-US.json | 7 + lib/clickhouse.js | 43 ++- lib/prisma.js | 60 ++++ pages/api/collect.js | 11 +- pages/api/websites/[id]/eventdata.js | 40 +++ pages/console/[[...id]].js | 6 + pages/login.js | 2 +- queries/analytics/event/getEventData.js | 63 +++++ queries/analytics/event/getEventMetrics.js | 4 +- queries/analytics/event/saveEvent.js | 5 +- .../analytics/pageview/getPageviewMetrics.js | 4 +- .../analytics/pageview/getPageviewParams.js | 4 +- .../analytics/pageview/getPageviewStats.js | 4 +- .../analytics/session/getSessionMetrics.js | 4 +- queries/analytics/stats/getActiveVisitors.js | 4 +- queries/analytics/stats/getWebsiteStats.js | 4 +- queries/index.js | 1 + 32 files changed, 690 insertions(+), 64 deletions(-) create mode 100644 components/common/EventDataButton.js create mode 100644 components/common/EventDataButton.module.css create mode 100644 components/forms/EventDataForm.js create mode 100644 components/forms/EventDataForm.module.css delete mode 100644 db/mysql/migrations/04_account_uuid/migration.sql create mode 100644 db/mysql/migrations/04_add_uuid/migration.sql rename db/postgresql/migrations/{04_account_uuid => 04_add_uuid}/migration.sql (59%) create mode 100644 pages/api/websites/[id]/eventdata.js create mode 100644 queries/analytics/event/getEventData.js diff --git a/components/common/EventDataButton.js b/components/common/EventDataButton.js new file mode 100644 index 00000000..2b895840 --- /dev/null +++ b/components/common/EventDataButton.js @@ -0,0 +1,48 @@ +import List from 'assets/list-ul.svg'; +import Modal from 'components/common/Modal'; +import PropTypes from 'prop-types'; +import { useState } from 'react'; +import { FormattedMessage } from 'react-intl'; +import Button from './Button'; +import EventDataForm from 'components/forms/EventDataForm'; +import styles from './EventDataButton.module.css'; + +function EventDataButton({ websiteId }) { + const [showEventData, setShowEventData] = useState(false); + + function handleClick() { + if (!showEventData) { + setShowEventData(true); + } + } + + function handleClose() { + setShowEventData(false); + } + + return ( + <> + + {showEventData && ( + }> + + + )} + + ); +} + +EventDataButton.propTypes = { + websiteId: PropTypes.oneOfType([PropTypes.string, PropTypes.number]), +}; + +export default EventDataButton; diff --git a/components/common/EventDataButton.module.css b/components/common/EventDataButton.module.css new file mode 100644 index 00000000..cd2a2ed6 --- /dev/null +++ b/components/common/EventDataButton.module.css @@ -0,0 +1,3 @@ +.button { + width: fit-content; +} diff --git a/components/forms/EventDataForm.js b/components/forms/EventDataForm.js new file mode 100644 index 00000000..1bd27a12 --- /dev/null +++ b/components/forms/EventDataForm.js @@ -0,0 +1,262 @@ +import classNames from 'classnames'; +import Button from 'components/common/Button'; +import DateFilter from 'components/common/DateFilter'; +import DropDown from 'components/common/DropDown'; +import FormLayout, { + FormButtons, + FormError, + FormMessage, + FormRow, +} from 'components/layout/FormLayout'; +import DataTable from 'components/metrics/DataTable'; +import FilterTags from 'components/metrics/FilterTags'; +import { Field, Form, Formik } from 'formik'; +import useApi from 'hooks/useApi'; +import useDateRange from 'hooks/useDateRange'; +import { useState, useEffect } from 'react'; +import { FormattedMessage } from 'react-intl'; +import styles from './EventDataForm.module.css'; +import useTimezone from 'hooks/useTimezone'; + +export const filterOptions = [ + { label: 'Count', value: 'count' }, + { label: 'Average', value: 'avg' }, + { label: 'Minimum', value: 'min' }, + { label: 'Maxmimum', value: 'max' }, + { label: 'Sum', value: 'sum' }, +]; + +export const dateOptions = [ + { label: , value: '1day' }, + { + label: ( + + ), + value: '24hour', + }, + { + label: , + value: '-1day', + }, + { + label: , + value: '1week', + divider: true, + }, + { + label: ( + + ), + value: '7day', + }, + { + label: , + value: '1month', + divider: true, + }, + { + label: ( + + ), + value: '30day', + }, + { + label: ( + + ), + value: '90day', + }, + { label: , value: '1year' }, + { + label: , + value: 'custom', + divider: true, + }, +]; + +export default function EventDataForm({ websiteId, onClose, className }) { + const { post } = useApi(); + const [message, setMessage] = useState(); + const [columns, setColumns] = useState(); + const [filters, setFilters] = useState(); + const [data, setData] = useState([]); + const [dateRange, setDateRange] = useDateRange('report'); + const { startDate, endDate, value } = dateRange; + const [timezone] = useTimezone(); + const [isValid, setIsValid] = useState(false); + + useEffect(() => { + if (Object.keys(columns).length > 0) { + setIsValid(true); + } else { + setIsValid(false); + } + }, [columns]); + + const handleAddTag = (value, list, setState, resetForm) => { + setState({ ...list, [`${value.field}`]: value.value }); + resetForm(); + }; + + const handleRemoveTag = (value, list, setState) => { + const next = { ...list }; + + delete next[`${value}`]; + + setState(next); + }; + + const handleSubmit = async () => { + const params = { + website_id: websiteId, + start_at: +startDate, + end_at: +endDate, + timezone, + columns, + filters, + }; + + const { ok, data } = await post(`/websites/${websiteId}/eventdata`, params); + + if (!ok) { + setMessage(); + setData([]); + } else { + setData(data); + setMessage(null); + } + }; + + return ( + <> + {message} +
+
+ +
+ + + + +
+
+ + handleAddTag(value, columns, setColumns, resetForm) + } + > + {({ values, setFieldValue }) => ( +
+ + +
+ + +
+ + + +
+ setFieldValue('value', value)} + className={styles.dropdown} + name="value" + options={filterOptions} + /> + +
+ + + + +
+ )} + + handleRemoveTag(value, columns, setColumns)} + /> +
+
+ + handleAddTag(value, filters, setFilters, resetForm) + } + > + {({ values }) => ( +
+ + +
+ + +
+ + + +
+ + +
+ + + + +
+ )} + + handleRemoveTag(value, filters, setFilters)} + /> +
+ +
+
+ +
+
+ + + + + + ); +} diff --git a/components/forms/EventDataForm.module.css b/components/forms/EventDataForm.module.css new file mode 100644 index 00000000..19d76f77 --- /dev/null +++ b/components/forms/EventDataForm.module.css @@ -0,0 +1,38 @@ +.container { + display: flex; +} + +.form { + border-right: 1px solid var(--gray300); + width: 420px; +} + +.filters { + padding: 10px 5px; +} + +.filters + .filters { + border-top: 1px solid var(--gray300); + min-height: 250px; +} + +.table { + padding: 10px; + min-height: 430px; + min-width: 400px; +} + +.formButtons { + justify-content: flex-start; + margin-left: 20px; +} + +.dropdown { + min-height: 39px; + min-width: 240px; +} + +.filterTag { + flex-wrap: wrap; + margin: 10px 5px 5px 5px; +} diff --git a/components/metrics/DataTable.js b/components/metrics/DataTable.js index d7e58cf7..c9596106 100644 --- a/components/metrics/DataTable.js +++ b/components/metrics/DataTable.js @@ -16,6 +16,7 @@ export default function DataTable({ height, animate = true, virtualize = false, + showPercentage = true, }) { const [format, setFormat] = useState(true); const formatFunc = format ? formatLongNumber : formatNumber; @@ -38,6 +39,7 @@ export default function DataTable({ animate={animate && !virtualize} format={formatFunc} onClick={handleSetFormat} + showPercentage={showPercentage} /> ); }; @@ -68,7 +70,15 @@ export default function DataTable({ ); } -const AnimatedRow = ({ label, value = 0, percent, animate, format, onClick }) => { +const AnimatedRow = ({ + label, + value = 0, + percent, + animate, + format, + onClick, + showPercentage = true, +}) => { const props = useSpring({ width: percent, y: value, @@ -82,15 +92,17 @@ const AnimatedRow = ({ label, value = 0, percent, animate, format, onClick }) =>
{props.y?.interpolate(format)}
-
- `${n}%`) }} - /> - - {props.width.interpolate(n => `${n.toFixed(0)}%`)} - -
+ {showPercentage && ( +
+ `${n}%`) }} + /> + + {props.width.interpolate(n => `${n.toFixed(0)}%`)} + +
+ )} ); }; diff --git a/components/metrics/FilterTags.js b/components/metrics/FilterTags.js index be65540e..bb4174b8 100644 --- a/components/metrics/FilterTags.js +++ b/components/metrics/FilterTags.js @@ -5,12 +5,12 @@ import Button from 'components/common/Button'; import Times from 'assets/times.svg'; import styles from './FilterTags.module.css'; -export default function FilterTags({ params, onClick }) { +export default function FilterTags({ className, params, onClick }) { if (Object.keys(params).filter(key => params[key]).length === 0) { return null; } return ( -
+
{Object.keys(params).map(key => { if (!params[key]) { return null; diff --git a/components/metrics/FilterTags.module.css b/components/metrics/FilterTags.module.css index bb1536e5..50ae60a0 100644 --- a/components/metrics/FilterTags.module.css +++ b/components/metrics/FilterTags.module.css @@ -7,8 +7,5 @@ .tag { text-align: center; margin-bottom: 10px; -} - -.tag + .tag { - margin-left: 20px; + margin-right: 20px; } diff --git a/components/metrics/WebsiteHeader.js b/components/metrics/WebsiteHeader.js index 1a6bdf15..517ef140 100644 --- a/components/metrics/WebsiteHeader.js +++ b/components/metrics/WebsiteHeader.js @@ -1,14 +1,13 @@ -import React from 'react'; +import Arrow from 'assets/arrow-right.svg'; import classNames from 'classnames'; -import { FormattedMessage } from 'react-intl'; +import Favicon from 'components/common/Favicon'; import Link from 'components/common/Link'; import OverflowText from 'components/common/OverflowText'; -import PageHeader from 'components/layout/PageHeader'; import RefreshButton from 'components/common/RefreshButton'; import ButtonLayout from 'components/layout/ButtonLayout'; -import Favicon from 'components/common/Favicon'; +import PageHeader from 'components/layout/PageHeader'; +import { FormattedMessage } from 'react-intl'; import ActiveUsers from './ActiveUsers'; -import Arrow from 'assets/arrow-right.svg'; import styles from './WebsiteHeader.module.css'; export default function WebsiteHeader({ websiteId, title, domain, showLink = false }) { diff --git a/components/pages/TestConsole.js b/components/pages/TestConsole.js index 215c350c..6200f7e5 100644 --- a/components/pages/TestConsole.js +++ b/components/pages/TestConsole.js @@ -24,9 +24,9 @@ export default function TestConsole() { return null; } - const options = data.map(({ name, websiteId }) => ({ label: name, value: websiteId })); - const website = data.find(({ websiteId }) => websiteId === +websiteId); - const selectedValue = options.find(({ value }) => value === website?.websiteId)?.value; + const options = data.map(({ name, websiteUuid }) => ({ label: name, value: websiteUuid })); + const website = data.find(({ websiteUuid }) => websiteId === websiteUuid); + const selectedValue = options.find(({ value }) => value === website?.websiteUuid)?.value; function handleSelect(value) { router.push(`/console/${value}`); @@ -104,13 +104,13 @@ export default function TestConsole() {
Events - +
diff --git a/components/pages/WebsiteDetails.js b/components/pages/WebsiteDetails.js index 3fc234a4..b40bc9ee 100644 --- a/components/pages/WebsiteDetails.js +++ b/components/pages/WebsiteDetails.js @@ -24,6 +24,7 @@ import useFetch from 'hooks/useFetch'; import usePageQuery from 'hooks/usePageQuery'; import { DEFAULT_ANIMATION_DURATION } from 'lib/constants'; import styles from './WebsiteDetails.module.css'; +import EventDataButton from 'components/common/EventDataButton'; const messages = defineMessages({ pages: { id: 'metrics.pages', defaultMessage: 'Pages' }, @@ -183,6 +184,7 @@ export default function WebsiteDetails({ websiteId }) { + diff --git a/db/mysql/migrations/04_account_uuid/migration.sql b/db/mysql/migrations/04_account_uuid/migration.sql deleted file mode 100644 index 7b7b5dea..00000000 --- a/db/mysql/migrations/04_account_uuid/migration.sql +++ /dev/null @@ -1,11 +0,0 @@ --- AlterTable -ALTER TABLE `account` ADD COLUMN `account_uuid` VARCHAR(36); - --- Backfill UUID -UPDATE `account` SET account_uuid=(SELECT uuid()); - --- AlterTable -ALTER TABLE `account` MODIFY `account_uuid` VARCHAR(36) NOT NULL; - --- CreateIndex -CREATE UNIQUE INDEX `account_account_uuid_key` ON `account`(`account_uuid`); diff --git a/db/mysql/migrations/04_add_uuid/migration.sql b/db/mysql/migrations/04_add_uuid/migration.sql new file mode 100644 index 00000000..137ee2e3 --- /dev/null +++ b/db/mysql/migrations/04_add_uuid/migration.sql @@ -0,0 +1,35 @@ +-- AlterTable +ALTER TABLE `account` ADD COLUMN `account_uuid` VARCHAR(36); + +-- Backfill UUID +UPDATE `account` SET account_uuid=(SELECT uuid()); + +-- AlterTable +ALTER TABLE `account` MODIFY `account_uuid` VARCHAR(36) NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX `account_account_uuid_key` ON `account`(`account_uuid`); + +-- AlterTable +ALTER TABLE `event` ADD COLUMN `event_uuid` VARCHAR(36); + +-- Backfill UUID +UPDATE `event` SET event_uuid=(SELECT uuid()); + +-- AlterTable +ALTER TABLE `event` MODIFY `event_uuid` VARCHAR(36) NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX `event_event_uuid_key` ON `event`(`event_uuid`); + +-- CreateIndex +CREATE INDEX `account_account_uuid_idx` ON `account`(`account_uuid`); + +-- CreateIndex +CREATE INDEX `session_session_uuid_idx` ON `session`(`session_uuid`); + +-- CreateIndex +CREATE INDEX `website_website_uuid_idx` ON `website`(`website_uuid`); + +-- CreateIndex +CREATE INDEX `event_event_uuid_idx` ON `event`(`event_uuid`); \ No newline at end of file diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 6c740bfb..bdfafd43 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -16,6 +16,8 @@ model account { updatedAt DateTime? @default(now()) @map("updated_at") @db.Timestamp(0) accountUuid String @unique() @map("account_uuid") @db.VarChar(36) website website[] + + @@index([accountUuid]) } model event { @@ -25,6 +27,7 @@ model event { createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) url String @db.VarChar(500) eventName String @map("event_name") @db.VarChar(50) + eventUuid String @unique() @map("event_uuid") @db.VarChar(36) session session @relation(fields: [sessionId], references: [id]) website website @relation(fields: [websiteId], references: [id]) eventData eventData? @@ -32,6 +35,7 @@ model event { @@index([createdAt]) @@index([sessionId]) @@index([websiteId]) + @@index([eventUuid]) } model eventData { @@ -78,6 +82,7 @@ model session { @@index([createdAt]) @@index([websiteId]) + @@index([sessionUuid]) } model website { @@ -94,4 +99,5 @@ model website { session session[] @@index([userId]) + @@index([websiteUuid]) } diff --git a/db/postgresql/migrations/04_account_uuid/migration.sql b/db/postgresql/migrations/04_add_uuid/migration.sql similarity index 59% rename from db/postgresql/migrations/04_account_uuid/migration.sql rename to db/postgresql/migrations/04_add_uuid/migration.sql index b6929f64..21d4cf07 100644 --- a/db/postgresql/migrations/04_account_uuid/migration.sql +++ b/db/postgresql/migrations/04_add_uuid/migration.sql @@ -11,6 +11,18 @@ ALTER TABLE "account" ALTER COLUMN "account_uuid" SET NOT NULL; -- CreateIndex CREATE UNIQUE INDEX "account_account_uuid_key" ON "account"("account_uuid"); +-- AlterTable +ALTER TABLE "event" ADD COLUMN "event_uuid" UUID NULL; + +-- Backfill UUID +UPDATE "event" SET event_uuid = gen_random_uuid(); + +-- AlterTable +ALTER TABLE "event" ALTER COLUMN "event_uuid" SET NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX "event_event_uuid_key" ON "event"("event_uuid"); + -- CreateIndex CREATE INDEX "account_account_uuid_idx" ON "account"("account_uuid"); @@ -18,4 +30,7 @@ CREATE INDEX "account_account_uuid_idx" ON "account"("account_uuid"); CREATE INDEX "session_session_uuid_idx" ON "session"("session_uuid"); -- CreateIndex -CREATE INDEX "website_website_uuid_idx" ON "website"("website_uuid"); \ No newline at end of file +CREATE INDEX "website_website_uuid_idx" ON "website"("website_uuid"); + +-- CreateIndex +CREATE INDEX "event_event_uuid_idx" ON "event"("event_uuid"); \ No newline at end of file diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 920a6f78..ad1c7595 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -16,6 +16,8 @@ model account { updatedAt DateTime? @default(now()) @map("updated_at") @db.Timestamptz(6) accountUuid String @unique @map("account_uuid") @db.Uuid website website[] + + @@index([accountUuid]) } model event { @@ -25,6 +27,7 @@ model event { createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) url String @db.VarChar(500) eventName String @map("event_name") @db.VarChar(50) + eventUuid String @unique @map("event_uuid") @db.Uuid session session @relation(fields: [sessionId], references: [id]) website website @relation(fields: [websiteId], references: [id]) eventData eventData? @@ -32,6 +35,7 @@ model event { @@index([createdAt]) @@index([sessionId]) @@index([websiteId]) + @@index([eventUuid]) } model eventData { @@ -78,6 +82,7 @@ model session { @@index([createdAt]) @@index([websiteId]) + @@index([sessionUuid]) } model website { @@ -94,4 +99,5 @@ model website { session session[] @@index([userId]) + @@index([websiteUuid]) } diff --git a/lang/en-US.json b/lang/en-US.json index aa5e3ae3..f58a25cd 100644 --- a/lang/en-US.json +++ b/lang/en-US.json @@ -1,6 +1,8 @@ { "label.accounts": "Accounts", "label.add-account": "Add account", + "label.add-column": "Add column", + "label.add-filter": "Add filter", "label.add-website": "Add website", "label.administrator": "Administrator", "label.all": "All", @@ -25,6 +27,8 @@ "label.edit-account": "Edit account", "label.edit-website": "Edit website", "label.enable-share-url": "Enable share URL", + "label.event-data": "Event Data", + "label.field-name": "Field Name", "label.invalid": "Invalid", "label.invalid-domain": "Invalid domain", "label.language": "Language", @@ -48,6 +52,7 @@ "label.reset": "Reset", "label.reset-website": "Reset statistics", "label.save": "Save", + "label.search": "Search", "label.settings": "Settings", "label.share-url": "Share URL", "label.single-day": "Single day", @@ -58,8 +63,10 @@ "label.timezone": "Timezone", "label.today": "Today", "label.tracking-code": "Tracking code", + "label.type": "Type", "label.unknown": "Unknown", "label.username": "Username", + "label.value": "Value", "label.view-details": "View details", "label.websites": "Websites", "label.yesterday": "Yesterday", diff --git a/lib/clickhouse.js b/lib/clickhouse.js index a01a36cc..41b1df5a 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -65,8 +65,45 @@ function getCommaSeparatedStringFormat(data) { } function getBetweenDates(field, start_at, end_at) { - return `${field} between ${getDateFormat(start_at)} - and ${getDateFormat(end_at)}`; + return `${field} between ${getDateFormat(start_at)} and ${getDateFormat(end_at)}`; +} + +function getJsonField(column, property) { + return `${column}.${property}`; +} + +function getEventDataColumnsQuery(column, columns) { + const query = Object.keys(columns).reduce((arr, key) => { + const filter = columns[key]; + + if (filter === undefined) { + return arr; + } + + arr.push(`${filter}(${getJsonField(column, key)}) as ${key}_${filter}`); + + return arr; + }, []); + + return query.join(',\n'); +} + +function getEventDataFilterQuery(column, filters) { + const query = Object.keys(filters).reduce((arr, key) => { + const filter = filters[key]; + + if (filter === undefined) { + return arr; + } + + arr.push( + `${getJsonField(column, key)} = ${typeof filter === 'string' ? `'${filter}'` : filter}`, + ); + + return arr; + }, []); + + return query.join('\nand '); } function getFilterQuery(column, filters = {}, params = []) { @@ -186,6 +223,8 @@ export default { getDateFormat, getCommaSeparatedStringFormat, getBetweenDates, + getEventDataColumnsQuery, + getEventDataFilterQuery, getFilterQuery, parseFilters, findUnique, diff --git a/lib/prisma.js b/lib/prisma.js index b93afafa..ab1e6ebf 100644 --- a/lib/prisma.js +++ b/lib/prisma.js @@ -85,6 +85,64 @@ function getTimestampInterval(field) { } } +function getJsonField(column, property, isNumber) { + const db = getDatabaseType(process.env.DATABASE_URL); + + if (db === POSTGRESQL) { + let accessor = `${column} ->> '${property}'`; + + if (isNumber) { + accessor = `CAST(${accessor} AS DECIMAL)`; + } + + return accessor; + } + + if (db === MYSQL) { + return `${column} ->> "$.${property}"`; + } +} + +function getEventDataColumnsQuery(column, columns) { + const query = Object.keys(columns).reduce((arr, key) => { + const filter = columns[key]; + + if (filter === undefined) { + return arr; + } + + const isNumber = ['sum', 'avg', 'min', 'max'].some(a => a === filter); + + arr.push(`${filter}(${getJsonField(column, key, isNumber)}) as "${filter}(${key})"`); + + return arr; + }, []); + + return query.join(',\n'); +} + +function getEventDataFilterQuery(column, filters) { + const query = Object.keys(filters).reduce((arr, key) => { + const filter = filters[key]; + + if (filter === undefined) { + return arr; + } + + const isNumber = filter && typeof filter === 'number'; + + arr.push( + `${getJsonField(column, key, isNumber)} = ${ + typeof filter === 'string' ? `'${filter}'` : filter + }`, + ); + + return arr; + }, []); + + return query.join('\nand '); +} + function getFilterQuery(table, column, filters = {}, params = []) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -193,6 +251,8 @@ export default { getDateQuery, getTimestampInterval, getFilterQuery, + getEventDataColumnsQuery, + getEventDataFilterQuery, parseFilters, rawQuery, transaction, diff --git a/pages/api/collect.js b/pages/api/collect.js index 42fd309a..5a4411d5 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -58,13 +58,11 @@ export default async (req, res) => { await useSession(req, res); - const { - session: { website, session }, - } = req; + const { website, session } = req.session; const { type, payload } = getJsonBody(req); - let { url, referrer, eventName, eventData } = payload; + let { url, referrer, event_name: eventName, event_data: eventData } = payload; if (process.env.REMOVE_TRAILING_SLASH) { url = url.replace(/\/$/, ''); @@ -88,9 +86,8 @@ export default async (req, res) => { const token = createToken( { - websiteId: website.websiteUuid, - sessionId: session.sessionId, - sessionUuid: session.sessionUuid, + website, + session, }, secret(), ); diff --git a/pages/api/websites/[id]/eventdata.js b/pages/api/websites/[id]/eventdata.js new file mode 100644 index 00000000..86a17b77 --- /dev/null +++ b/pages/api/websites/[id]/eventdata.js @@ -0,0 +1,40 @@ +import moment from 'moment-timezone'; +import { getEventData } from 'queries'; +import { ok, badRequest, methodNotAllowed, unauthorized } from 'next-basics'; +import { allowQuery } from 'lib/auth'; +import { useAuth, useCors } from 'lib/middleware'; + +export default async (req, res) => { + await useCors(req, res); + await useAuth(req, res); + + if (req.method === 'POST') { + if (!(await allowQuery(req))) { + return unauthorized(res); + } + + const { id: websiteId } = req.query; + + const { start_at, end_at, timezone, event_name: eventName, columns, filters } = req.body; + + if (!moment.tz.zone(timezone)) { + return badRequest(res); + } + + const startDate = new Date(+start_at); + const endDate = new Date(+end_at); + + const events = await getEventData(websiteId, { + startDate, + endDate, + timezone, + eventName, + columns, + filters, + }); + + return ok(res, events); + } + + return methodNotAllowed(res); +}; diff --git a/pages/console/[[...id]].js b/pages/console/[[...id]].js index 8e37e77a..a13537f8 100644 --- a/pages/console/[[...id]].js +++ b/pages/console/[[...id]].js @@ -18,3 +18,9 @@ export default function ConsolePage({ enabled }) { ); } + +export async function getServerSideProps() { + return { + props: { enabled: !!process.env.ENABLE_TEST_CONSOLE }, + }; +} diff --git a/pages/login.js b/pages/login.js index d46110c8..12f70ac9 100644 --- a/pages/login.js +++ b/pages/login.js @@ -16,6 +16,6 @@ export default function LoginPage({ loginDisabled }) { export async function getServerSideProps() { return { - props: { loginDisabled: !!process.env.DISABLE_LOGIN || process.env.isCloudMode }, + props: { loginDisabled: !!process.env.DISABLE_LOGIN || !!process.env.isCloudMode }, }; } diff --git a/queries/analytics/event/getEventData.js b/queries/analytics/event/getEventData.js new file mode 100644 index 00000000..91302d30 --- /dev/null +++ b/queries/analytics/event/getEventData.js @@ -0,0 +1,63 @@ +import clickhouse from 'lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; + +export async function getEventData(...args) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId, { startDate, endDate, event_name, columns, filters }) { + const { rawQuery, getEventDataColumnsQuery, getEventDataFilterQuery } = prisma; + const params = [startDate, endDate]; + + return rawQuery( + `select + ${getEventDataColumnsQuery('event_data.event_data', columns)} + from event + join website + on event.website_id = website.website_id + join event_data + on event.event_id = event_data.event_id + where website_uuid='${websiteId}' + and event.created_at between $1 and $2 + ${event_name ? `and event_name = ${event_name}` : ''} + ${ + Object.keys(filters).length > 0 + ? `and ${getEventDataFilterQuery('event_data.event_data', filters)}` + : '' + }`, + params, + ).then(results => { + return Object.keys(results[0]).map(a => { + return { x: a, y: results[0][`${a}`] }; + }); + }); +} + +async function clickhouseQuery(websiteId, { startDate, endDate, event_name, columns, filters }) { + const { rawQuery, getBetweenDates, getEventDataColumnsQuery, getEventDataFilterQuery } = + clickhouse; + const params = [websiteId]; + + return rawQuery( + `select + ${getEventDataColumnsQuery('event_data', columns)} + from event + where website_id= $1 + ${event_name ? `and event_name = ${event_name}` : ''} + and ${getBetweenDates('created_at', startDate, endDate)} + ${ + Object.keys(filters).length > 0 + ? `and ${getEventDataFilterQuery('event_data', filters)}` + : '' + }`, + params, + ).then(results => { + return Object.keys(results[0]).map(a => { + return { x: a, y: results[0][`${a}`] }; + }); + }); +} diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index edf5de8c..605bb688 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -18,7 +18,7 @@ async function relationalQuery( filters = {}, ) { const { rawQuery, getDateQuery, getFilterQuery } = prisma; - const params = [websiteId, start_at, end_at]; + const params = [start_at, end_at]; return rawQuery( `select @@ -29,7 +29,7 @@ async function relationalQuery( join website on event.website_id = website.website_id where website_uuid='${websiteId}' - and event.created_at between $2 and $3 + and event.created_at between $1 and $2 ${getFilterQuery('event', filters, params)} group by 1, 2 order by 2`, diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 26cf18b7..de7a1cae 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -12,13 +12,14 @@ export async function saveEvent(...args) { async function relationalQuery( { websiteId }, - { session: { id: sessionId }, url, eventName, eventData }, + { session: { id: sessionId }, eventUuid, url, eventName, eventData }, ) { const data = { websiteId, sessionId, url: url?.substring(0, URL_LENGTH), eventName: eventName?.substring(0, EVENT_NAME_LENGTH), + eventUuid, }; if (eventData) { @@ -47,7 +48,7 @@ async function clickhouseQuery( created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: eventName?.substring(0, EVENT_NAME_LENGTH), - event_data: JSON.stringify(eventData), + event_data: eventData ? JSON.stringify(eventData) : null, ...sessionArgs, country: country ? country : null, }; diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index e1c4d43f..69607d00 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -11,7 +11,7 @@ export async function getPageviewMetrics(...args) { async function relationalQuery(websiteId, { startDate, endDate, column, table, filters = {} }) { const { rawQuery, parseFilters } = prisma; - const params = [websiteId, startDate, endDate]; + const params = [startDate, endDate]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, column, @@ -25,7 +25,7 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, f ${` join website on ${table}.website_id = website.website_id`} ${joinSession} where website.website_uuid='${websiteId}' - and ${table}.created_at between $2 and $3 + and ${table}.created_at between $1 and $2 ${pageviewQuery} ${joinSession && sessionQuery} ${eventQuery} diff --git a/queries/analytics/pageview/getPageviewParams.js b/queries/analytics/pageview/getPageviewParams.js index 8ec26dec..5cdabfa3 100644 --- a/queries/analytics/pageview/getPageviewParams.js +++ b/queries/analytics/pageview/getPageviewParams.js @@ -10,7 +10,7 @@ export async function getPageviewParams(...args) { async function relationalQuery(websiteId, start_at, end_at, column, table, filters = {}) { const { parseFilters, rawQuery } = prisma; - const params = [websiteId, start_at, end_at]; + const params = [start_at, end_at]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, column, @@ -25,7 +25,7 @@ async function relationalQuery(websiteId, start_at, end_at, column, table, filte ${` join website on ${table}.website_id = website.website_id`} ${joinSession} where website.website_uuid='${websiteId}' - and ${table}.created_at between $2 and $3 + and ${table}.created_at between $1 and $2 and ${table}.url like '%?%' ${pageviewQuery} ${joinSession && sessionQuery} diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index ceed4daf..5ec8339f 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -22,7 +22,7 @@ async function relationalQuery( }, ) { const { getDateQuery, parseFilters, rawQuery } = prisma; - const params = [websiteId, start_at, end_at]; + const params = [start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', null, @@ -38,7 +38,7 @@ async function relationalQuery( on pageview.website_id = website.website_id ${joinSession} where website.website_uuid='${websiteId}' - and pageview.created_at between $2 and $3 + and pageview.created_at between $1 and $2 ${pageviewQuery} ${sessionQuery} group by 1`, diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index cbf3ed58..020bddfb 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -11,7 +11,7 @@ export async function getSessionMetrics(...args) { async function relationalQuery(websiteId, { startDate, endDate, field, filters = {} }) { const { parseFilters, rawQuery } = prisma; - const params = [websiteId, startDate, endDate]; + const params = [startDate, endDate]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters(null, filters, params); return rawQuery( @@ -24,7 +24,7 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters = on pageview.website_id = website.website_id ${joinSession} where website.website_uuid='${websiteId}' - and pageview.created_at between $2 and $3 + and pageview.created_at between $1 and $2 ${pageviewQuery} ${sessionQuery} ) diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index 9d6b1f09..3a898d94 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -12,7 +12,7 @@ export async function getActiveVisitors(...args) { async function relationalQuery(websiteId) { const date = subMinutes(new Date(), 5); - const params = [websiteId, date]; + const params = [date]; return prisma.rawQuery( `select count(distinct session_id) x @@ -20,7 +20,7 @@ async function relationalQuery(websiteId) { join website on pageview.website_id = website.website_id where website.website_uuid = '${websiteId}' - and pageview.created_at >= $2`, + and pageview.created_at >= $1`, params, ); } diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index b6bf7b87..134e1c3e 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -11,7 +11,7 @@ export async function getWebsiteStats(...args) { async function relationalQuery(websiteId, { start_at, end_at, filters = {} }) { const { getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; - const params = [websiteId, start_at, end_at]; + const params = [start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( 'pageview', null, @@ -34,7 +34,7 @@ async function relationalQuery(websiteId, { start_at, end_at, filters = {} }) { on pageview.website_id = website.website_id ${joinSession} where website.website_uuid='${websiteId}' - and pageview.created_at between $2 and $3 + and pageview.created_at between $1 and $2 ${pageviewQuery} ${sessionQuery} group by 1, 2 diff --git a/queries/index.js b/queries/index.js index d6b4093a..abff147a 100644 --- a/queries/index.js +++ b/queries/index.js @@ -17,6 +17,7 @@ export * from './admin/website/resetWebsite'; export * from './admin/website/updateWebsite'; export * from './analytics/event/getEventMetrics'; export * from './analytics/event/getEvents'; +export * from './analytics/event/getEventData'; export * from './analytics/event/saveEvent'; export * from './analytics/pageview/getPageviewMetrics'; export * from './analytics/pageview/getPageviewParams';