From 810b0639c87a825c5c00ae343f799f3490858a6e Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 2 Apr 2024 17:06:06 -0700 Subject: [PATCH] Updated filtering logic. --- .../[reportId]/FieldFilterEditForm.tsx | 4 +- .../reports/[reportId]/FilterParameters.tsx | 6 +-- .../[websiteId]/WebsiteExpandedView.tsx | 8 +++- src/components/metrics/FilterTags.tsx | 4 +- src/lib/clickhouse.ts | 27 +++++------ src/lib/params.ts | 47 +++++++++++++++---- src/lib/prisma.ts | 29 +++++------- src/lib/{query.ts => request.ts} | 21 ++------- src/pages/api/send.ts | 8 ++-- src/pages/api/websites/[websiteId]/events.ts | 4 +- src/pages/api/websites/[websiteId]/metrics.ts | 6 +-- .../api/websites/[websiteId]/pageviews.ts | 6 +-- src/pages/api/websites/[websiteId]/stats.ts | 6 +-- src/pages/api/websites/[websiteId]/values.ts | 4 +- 14 files changed, 97 insertions(+), 83 deletions(-) rename src/lib/{query.ts => request.ts} (63%) diff --git a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx index f1ebd30d..5612c554 100644 --- a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx +++ b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx @@ -16,7 +16,7 @@ import { } from 'react-basics'; import { useMessages, useFilters, useFormat, useLocale, useWebsiteValues } from 'components/hooks'; import { OPERATORS } from 'lib/constants'; -import { operatorEquals } from 'lib/params'; +import { isEqualsOperator } from 'lib/params'; import styles from './FieldFilterEditForm.module.css'; export interface FieldFilterFormProps { @@ -50,7 +50,7 @@ export default function FieldFilterEditForm({ const [operator, setOperator] = useState(defaultOperator); const [value, setValue] = useState(defaultValue); const [showMenu, setShowMenu] = useState(false); - const isEquals = operatorEquals(operator); + const isEquals = isEqualsOperator(operator); const [search, setSearch] = useState(''); const [selected, setSelected] = useState(isEquals ? value : ''); const { filters } = useFilters(); diff --git a/src/app/(main)/reports/[reportId]/FilterParameters.tsx b/src/app/(main)/reports/[reportId]/FilterParameters.tsx index d88f785d..edc0ff4e 100644 --- a/src/app/(main)/reports/[reportId]/FilterParameters.tsx +++ b/src/app/(main)/reports/[reportId]/FilterParameters.tsx @@ -7,7 +7,7 @@ import ParameterList from '../[reportId]/ParameterList'; import PopupForm from '../[reportId]/PopupForm'; import { ReportContext } from './Report'; import FieldFilterEditForm from '../[reportId]/FieldFilterEditForm'; -import { operatorEquals } from 'lib/params'; +import { isSearchOperator } from 'lib/params'; import styles from './FilterParameters.module.css'; export function FilterParameters() { @@ -69,7 +69,7 @@ export function FilterParameters() { {filters.map( ({ name, operator, value }: { name: string; operator: string; value: string }) => { const label = fields.find(f => f.name === name)?.label; - const isEquals = operatorEquals(operator); + const isSearch = isSearchOperator(operator); return ( handleRemove(name)}> @@ -79,7 +79,7 @@ export function FilterParameters() { name={name} label={label} operator={operator} - value={isEquals ? formatValue(value, name) : value} + value={isSearch ? value : formatValue(value, name)} onChange={handleChange} /> diff --git a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx index 9fdc8fd0..ea037852 100644 --- a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx +++ b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx @@ -44,7 +44,6 @@ export default function WebsiteExpandedView({ const { router, renderUrl, - pathname, query: { view }, } = useNavigation(); @@ -122,7 +121,12 @@ export default function WebsiteExpandedView({ return (
- + diff --git a/src/components/metrics/FilterTags.tsx b/src/components/metrics/FilterTags.tsx index 24311d6e..a302575f 100644 --- a/src/components/metrics/FilterTags.tsx +++ b/src/components/metrics/FilterTags.tsx @@ -11,7 +11,7 @@ import { import PopupForm from 'app/(main)/reports/[reportId]/PopupForm'; import FieldFilterEditForm from 'app/(main)/reports/[reportId]/FieldFilterEditForm'; import { OPERATOR_PREFIXES } from 'lib/constants'; -import { operatorEquals, parseParameterValue } from 'lib/params'; +import { isSearchOperator, parseParameterValue } from 'lib/params'; import styles from './FilterTags.module.css'; export function FilterTags({ @@ -66,7 +66,7 @@ export function FilterTags({ } const label = fields.find(f => f.name === key)?.label; const { operator, value } = parseParameterValue(params[key]); - const paramValue = operatorEquals(operator) ? formatValue(value, key) : value; + const paramValue = isSearchOperator(operator) ? formatValue(value, key) : value; return ( diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 0eabd3d2..5bcf7ee0 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -3,9 +3,10 @@ import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; import { QueryFilters, QueryOptions } from './types'; -import { FILTER_COLUMNS, OPERATORS } from './constants'; +import { OPERATORS } from './constants'; import { loadWebsite } from './load'; import { maxDate } from './date'; +import { filtersToArray } from './params'; export const CLICKHOUSE_DATE_FORMATS = { minute: '%Y-%m-%d %H:%M:00', @@ -79,15 +80,11 @@ function mapFilter(column: string, operator: string, name: string, type: string } function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) { - const query = Object.keys(filters).reduce((arr, key) => { - const filter = filters[key]; - const operator = filter?.operator ?? OPERATORS.equals; - const column = filter?.column ?? FILTER_COLUMNS[key] ?? options?.columns?.[key]; + const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => { + if (column) { + arr.push(`and ${mapFilter(column, operator, name)}`); - if (filter !== undefined && column !== undefined) { - arr.push(`and ${mapFilter(column, operator, key)}`); - - if (key === 'referrer') { + if (name === 'referrer') { arr.push('and referrer_domain != {websiteDomain:String}'); } } @@ -98,11 +95,11 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) return query.join('\n'); } -function normalizeFilters(filters = {}) { - return Object.keys(filters).reduce((obj, key) => { - const value = filters[key]; - - obj[key] = value?.value ?? value; +function getFilterParams(filters: QueryFilters = {}) { + return filtersToArray(filters).reduce((obj, { name, value }) => { + if (name && value !== undefined) { + obj[name] = value; + } return obj; }, {}); @@ -114,7 +111,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio return { filterQuery: getFilterQuery(filters, options), params: { - ...normalizeFilters(filters), + ...getFilterParams(filters), websiteId, startDate: maxDate(filters.startDate, new Date(website?.resetAt)), websiteDomain: website.domain, diff --git a/src/lib/params.ts b/src/lib/params.ts index 10944c96..801c60b2 100644 --- a/src/lib/params.ts +++ b/src/lib/params.ts @@ -1,15 +1,46 @@ -import { OPERATOR_PREFIXES, OPERATORS } from 'lib/constants'; +import { FILTER_COLUMNS, OPERATOR_PREFIXES, OPERATORS } from 'lib/constants'; +import { QueryFilters, QueryOptions } from 'lib/types'; -export function parseParameterValue(param: string) { - const [, prefix, value] = param.match(/^(!~|!|~)?(.*)$/); +export function parseParameterValue(param: any) { + if (typeof param === 'string') { + const [, prefix, value] = param.match(/^(!~|!|~)?(.*)$/); - const operator = - Object.keys(OPERATOR_PREFIXES).find(key => OPERATOR_PREFIXES[key] === prefix) || - OPERATORS.equals; + const operator = + Object.keys(OPERATOR_PREFIXES).find(key => OPERATOR_PREFIXES[key] === prefix) || + OPERATORS.equals; - return { operator, value }; + return { operator, value }; + } + return { operator: OPERATORS.equals, value: param }; } -export function operatorEquals(operator: any) { +export function isEqualsOperator(operator: any) { return [OPERATORS.equals, OPERATORS.notEquals].includes(operator); } + +export function isSearchOperator(operator: any) { + return [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator); +} + +export function filtersToArray(filters: QueryFilters = {}, options: QueryOptions = {}) { + return Object.keys(filters).reduce((arr, key) => { + const filter = filters[key]; + + if (filter === undefined || filter === null) { + return arr; + } + + if (filter?.name && filter?.value !== undefined) { + return arr.concat(filter); + } + + const { operator, value } = parseParameterValue(filter); + + return arr.concat({ + name: key, + column: options?.columns?.[key] ?? FILTER_COLUMNS[key], + operator, + value, + }); + }, []); +} diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index c3dd071b..c35e0cde 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -2,10 +2,11 @@ import { Prisma } from '@prisma/client'; import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; -import { FILTER_COLUMNS, SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants'; +import { SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants'; import { loadWebsite } from './load'; import { maxDate } from './date'; import { QueryFilters, QueryOptions, SearchFilter } from './types'; +import { filtersToArray } from './params'; const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -112,15 +113,11 @@ function mapFilter(column: string, operator: string, name: string, type: string } function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string { - const query = Object.keys(filters).reduce((arr, key) => { - const filter = filters[key]; - const operator = filter?.operator ?? OPERATORS.equals; - const column = filter?.column ?? FILTER_COLUMNS[key] ?? options?.columns?.[key]; + const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => { + if (column) { + arr.push(`and ${mapFilter(column, operator, name)}`); - if (filter !== undefined && column !== undefined) { - arr.push(`and ${mapFilter(column, operator, key)}`); - - if (key === 'referrer') { + if (name === 'referrer') { arr.push( 'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)', ); @@ -133,12 +130,9 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): return query.join('\n'); } -function normalizeFilters(filters = {}) { - return Object.keys(filters).reduce((obj, key) => { - const filter = filters[key]; - const value = filter?.value ?? filter; - - obj[key] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(filter?.operator) +function getFilterParams(filters: QueryFilters = {}) { + return filtersToArray(filters).reduce((obj, { name, operator, value }) => { + obj[name] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator) ? `%${value}%` : value; @@ -152,15 +146,16 @@ async function parseFilters( options: QueryOptions = {}, ) { const website = await loadWebsite(websiteId); + const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key)); return { joinSession: - options?.joinSession || Object.keys(filters).find(key => SESSION_COLUMNS.includes(key)) + options?.joinSession || joinSession ? `inner join session on website_event.session_id = session.session_id` : '', filterQuery: getFilterQuery(filters, options), params: { - ...normalizeFilters(filters), + ...getFilterParams(filters), websiteId, startDate: maxDate(filters.startDate, website?.resetAt), websiteDomain: website.domain, diff --git a/src/lib/query.ts b/src/lib/request.ts similarity index 63% rename from src/lib/query.ts rename to src/lib/request.ts index 8b6bcee6..5e2be2fe 100644 --- a/src/lib/query.ts +++ b/src/lib/request.ts @@ -1,9 +1,9 @@ import { NextApiRequest } from 'next'; import { getAllowedUnits, getMinimumUnit } from './date'; import { getWebsiteDateRange } from '../queries'; -import { FILTER_COLUMNS, OPERATORS, OPERATOR_PREFIXES } from 'lib/constants'; +import { FILTER_COLUMNS } from 'lib/constants'; -export async function parseDateRangeQuery(req: NextApiRequest) { +export async function getRequestDateRange(req: NextApiRequest) { const { websiteId, startAt, endAt, unit } = req.query; // All-time @@ -31,27 +31,14 @@ export async function parseDateRangeQuery(req: NextApiRequest) { }; } -export function getQueryFilters(req: NextApiRequest) { +export function getRequestFilters(req: NextApiRequest) { return Object.keys(FILTER_COLUMNS).reduce((obj, key) => { const value = req.query[key]; - if (value) { + if (value !== undefined) { obj[key] = value; } - if (typeof value === 'string') { - const [, prefix, paramValue] = value.match(/^(!~|!|~)?(.*)$/); - - if (prefix && paramValue) { - obj[key] = { - name: key, - column: FILTER_COLUMNS[key], - operator: OPERATOR_PREFIXES[prefix] || OPERATORS.equals, - value: paramValue, - }; - } - } - return obj; }, {}); } diff --git a/src/pages/api/send.ts b/src/pages/api/send.ts index 72fd7a64..0b43f920 100644 --- a/src/pages/api/send.ts +++ b/src/pages/api/send.ts @@ -98,7 +98,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { } const { type, payload } = req.body; - const { url, referrer, name: eventName, data: eventData, title } = payload; + const { url, referrer, name: eventName, data, title } = payload; const pageTitle = safeDecodeURI(title); await useSession(req, res); @@ -142,7 +142,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { referrerDomain, pageTitle, eventName, - eventData, + eventData: data, ...session, sessionId: session.id, visitId: session.visitId, @@ -150,14 +150,14 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { } if (type === COLLECTION_TYPE.identify) { - if (!eventData) { + if (!data) { return badRequest(res, 'Data required.'); } await saveSessionData({ websiteId: session.websiteId, sessionId: session.id, - sessionData: eventData, + sessionData: data, }); } diff --git a/src/pages/api/websites/[websiteId]/events.ts b/src/pages/api/websites/[websiteId]/events.ts index b4d465a4..d07fd28f 100644 --- a/src/pages/api/websites/[websiteId]/events.ts +++ b/src/pages/api/websites/[websiteId]/events.ts @@ -1,6 +1,6 @@ import { canViewWebsite } from 'lib/auth'; import { useAuth, useCors, useValidate } from 'lib/middleware'; -import { parseDateRangeQuery } from 'lib/query'; +import { getRequestDateRange } from 'lib/request'; import { NextApiRequestQueryBody, WebsiteMetric } from 'lib/types'; import { TimezoneTest, UnitTypeTest } from 'lib/yup'; import { NextApiResponse } from 'next'; @@ -37,7 +37,7 @@ export default async ( await useValidate(schema, req, res); const { websiteId, timezone, url } = req.query; - const { startDate, endDate, unit } = await parseDateRangeQuery(req); + const { startDate, endDate, unit } = await getRequestDateRange(req); if (req.method === 'GET') { if (!(await canViewWebsite(req.auth, websiteId))) { diff --git a/src/pages/api/websites/[websiteId]/metrics.ts b/src/pages/api/websites/[websiteId]/metrics.ts index 2387d9c2..d1765b1c 100644 --- a/src/pages/api/websites/[websiteId]/metrics.ts +++ b/src/pages/api/websites/[websiteId]/metrics.ts @@ -5,7 +5,7 @@ import { canViewWebsite } from 'lib/auth'; import { useAuth, useCors, useValidate } from 'lib/middleware'; import { SESSION_COLUMNS, EVENT_COLUMNS, FILTER_COLUMNS, OPERATORS } from 'lib/constants'; import { getPageviewMetrics, getSessionMetrics } from 'queries'; -import { getQueryFilters, parseDateRangeQuery } from 'lib/query'; +import { getRequestFilters, getRequestDateRange } from 'lib/request'; import * as yup from 'yup'; export interface WebsiteMetricsRequestQuery { @@ -69,10 +69,10 @@ export default async ( return unauthorized(res); } - const { startDate, endDate } = await parseDateRangeQuery(req); + const { startDate, endDate } = await getRequestDateRange(req); const column = FILTER_COLUMNS[type] || type; const filters = { - ...getQueryFilters(req), + ...getRequestFilters(req), startDate, endDate, }; diff --git a/src/pages/api/websites/[websiteId]/pageviews.ts b/src/pages/api/websites/[websiteId]/pageviews.ts index 9ac4e870..19671064 100644 --- a/src/pages/api/websites/[websiteId]/pageviews.ts +++ b/src/pages/api/websites/[websiteId]/pageviews.ts @@ -1,6 +1,6 @@ import { canViewWebsite } from 'lib/auth'; import { useAuth, useCors, useValidate } from 'lib/middleware'; -import { getQueryFilters, parseDateRangeQuery } from 'lib/query'; +import { getRequestFilters, getRequestDateRange } from 'lib/request'; import { NextApiRequestQueryBody, WebsitePageviews } from 'lib/types'; import { NextApiResponse } from 'next'; import { methodNotAllowed, ok, unauthorized } from 'next-basics'; @@ -59,10 +59,10 @@ export default async ( return unauthorized(res); } - const { startDate, endDate, unit } = await parseDateRangeQuery(req); + const { startDate, endDate, unit } = await getRequestDateRange(req); const filters = { - ...getQueryFilters(req), + ...getRequestFilters(req), startDate, endDate, timezone, diff --git a/src/pages/api/websites/[websiteId]/stats.ts b/src/pages/api/websites/[websiteId]/stats.ts index dfc3df93..81a6d835 100644 --- a/src/pages/api/websites/[websiteId]/stats.ts +++ b/src/pages/api/websites/[websiteId]/stats.ts @@ -4,7 +4,7 @@ import { methodNotAllowed, ok, unauthorized } from 'next-basics'; import { canViewWebsite } from 'lib/auth'; import { useAuth, useCors, useValidate } from 'lib/middleware'; import { NextApiRequestQueryBody, WebsiteStats } from 'lib/types'; -import { getQueryFilters, parseDateRangeQuery } from 'lib/query'; +import { getRequestFilters, getRequestDateRange } from 'lib/request'; import { getWebsiteStats } from 'queries'; export interface WebsiteStatsRequestQuery { @@ -59,12 +59,12 @@ export default async ( return unauthorized(res); } - const { startDate, endDate } = await parseDateRangeQuery(req); + const { startDate, endDate } = await getRequestDateRange(req); const diff = differenceInMinutes(endDate, startDate); const prevStartDate = subMinutes(startDate, diff); const prevEndDate = subMinutes(endDate, diff); - const filters = getQueryFilters(req); + const filters = getRequestFilters(req); const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate }); diff --git a/src/pages/api/websites/[websiteId]/values.ts b/src/pages/api/websites/[websiteId]/values.ts index 36ca7948..364261d9 100644 --- a/src/pages/api/websites/[websiteId]/values.ts +++ b/src/pages/api/websites/[websiteId]/values.ts @@ -11,7 +11,7 @@ import { } from 'next-basics'; import { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; import { getValues } from 'queries'; -import { parseDateRangeQuery } from 'lib/query'; +import { getRequestDateRange } from 'lib/request'; import * as yup from 'yup'; export interface ValuesRequestQuery { @@ -38,7 +38,7 @@ export default async (req: NextApiRequestQueryBody, res: Nex await useValidate(schema, req, res); const { websiteId, type, search } = req.query; - const { startDate, endDate } = await parseDateRangeQuery(req); + const { startDate, endDate } = await getRequestDateRange(req); if (req.method === 'GET') { if (!SESSION_COLUMNS.includes(type as string) && !EVENT_COLUMNS.includes(type as string)) {