From cff2d0053655a18f6e32e62d1f1fa083f5bb58fa Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 26 Mar 2024 17:31:16 -0700 Subject: [PATCH] Refactored filter parameters. --- .../reports/[reportId]/FieldAddForm.tsx | 14 +- .../[reportId]/FieldFilterEditForm.module.css | 30 ++-- .../[reportId]/FieldFilterEditForm.tsx | 145 +++++++++++++----- .../[reportId]/FilterParameters.module.css | 2 +- .../reports/[reportId]/FilterParameters.tsx | 83 ++++++---- .../reports/[reportId]/FilterSelectForm.tsx | 6 +- .../hooks/queries/useWebsiteValues.ts | 25 ++- src/lib/clickhouse.ts | 4 +- src/lib/prisma.ts | 27 ++-- src/pages/api/reports/insights.ts | 4 +- src/pages/api/websites/[websiteId]/metrics.ts | 4 +- src/pages/api/websites/[websiteId]/values.ts | 26 +++- src/queries/analytics/getValues.ts | 44 +++++- 13 files changed, 291 insertions(+), 123 deletions(-) diff --git a/src/app/(main)/reports/[reportId]/FieldAddForm.tsx b/src/app/(main)/reports/[reportId]/FieldAddForm.tsx index 32a0b0af..9217ce4d 100644 --- a/src/app/(main)/reports/[reportId]/FieldAddForm.tsx +++ b/src/app/(main)/reports/[reportId]/FieldAddForm.tsx @@ -3,8 +3,6 @@ import { createPortal } from 'react-dom'; import { REPORT_PARAMETERS } from 'lib/constants'; import PopupForm from './PopupForm'; import FieldSelectForm from './FieldSelectForm'; -import FieldAggregateForm from './FieldAggregateForm'; -import FieldFilterEditForm from './FieldFilterEditForm'; export function FieldAddForm({ fields = [], @@ -17,7 +15,11 @@ export function FieldAddForm({ onAdd: (group: string, value: string) => void; onClose: () => void; }) { - const [selected, setSelected] = useState<{ name: string; type: string; value: string }>(); + const [selected, setSelected] = useState<{ + name: string; + type: string; + value: string; + }>(); const handleSelect = (value: any) => { const { type } = value; @@ -39,12 +41,6 @@ export function FieldAddForm({ return createPortal( {!selected && } - {selected && group === REPORT_PARAMETERS.fields && ( - - )} - {selected && group === REPORT_PARAMETERS.filters && ( - - )} , document.body, ); diff --git a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.module.css b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.module.css index ed78512d..43a34438 100644 --- a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.module.css +++ b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.module.css @@ -1,12 +1,7 @@ -.popup { - display: flex; +.menu { + position: absolute; max-width: 300px; max-height: 210px; - overflow: hidden; -} - -.popup > div { - overflow-y: auto; } .filter { @@ -16,9 +11,26 @@ } .dropdown { - min-width: 180px; + min-width: 200px; } .text { - min-width: 180px; + min-width: 200px; +} + +.selected { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 16px; + white-space: nowrap; + min-width: 200px; + font-weight: 900; + background: var(--base100); + border-radius: var(--border-radius); + cursor: pointer; +} + +.search { + position: relative; } diff --git a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx index f3b5b247..96faeb9c 100644 --- a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx +++ b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx @@ -6,14 +6,15 @@ import { Flexbox, Dropdown, Button, + SearchField, TextField, + Text, + Icon, + Icons, Menu, - Popup, - PopupTrigger, Loading, } from 'react-basics'; import { useMessages, useFilters, useFormat, useLocale, useWebsiteValues } from 'components/hooks'; -import { safeDecodeURIComponent } from 'next-basics'; import { OPERATORS } from 'lib/constants'; import styles from './FieldFilterEditForm.module.css'; @@ -22,8 +23,11 @@ export interface FieldFilterFormProps { name: string; label?: string; type: string; + startDate: Date; + endDate: Date; + operator?: string; defaultValue?: string; - onChange?: (filter: { name: string; type: string; filter: string; value: string }) => void; + onChange?: (filter: { name: string; type: string; operator: string; value: string }) => void; allowFilterSelect?: boolean; isNew?: boolean; } @@ -33,19 +37,37 @@ export default function FieldFilterEditForm({ name, label, type, - defaultValue, + startDate, + endDate, + operator: defaultOperator = 'eq', + defaultValue = '', onChange, allowFilterSelect = true, isNew, }: FieldFilterFormProps) { const { formatMessage, labels } = useMessages(); - const [filter, setFilter] = useState('eq'); - const [value, setValue] = useState(defaultValue ?? ''); + const [operator, setOperator] = useState(defaultOperator); + const [value, setValue] = useState(defaultValue); + const [showMenu, setShowMenu] = useState(false); + const isEquals = [OPERATORS.equals, OPERATORS.notEquals].includes(operator as any); + const [search, setSearch] = useState(''); + const [selected, setSelected] = useState(isEquals ? value : ''); const { getFilters } = useFilters(); const { formatValue } = useFormat(); const { locale } = useLocale(); const filters = getFilters(type); - const { data: values = [], isLoading } = useWebsiteValues(websiteId, name); + const isDisabled = !operator || (isEquals && !selected) || (!isEquals && !value); + const { + data: values = [], + isLoading, + refetch, + } = useWebsiteValues({ + websiteId, + type: name, + startDate, + endDate, + search, + }); const formattedValues = useMemo(() => { if (!values) { @@ -69,25 +91,49 @@ export default function FieldFilterEditForm({ const filteredValues = useMemo(() => { return value - ? values.filter(n => formattedValues[n].toLowerCase().includes(value.toLowerCase())) + ? values.filter((n: string | number) => + formattedValues[n].toLowerCase().includes(value.toLowerCase()), + ) : values; }, [value, formattedValues]); - const renderFilterValue = value => { - return filters.find(f => f.value === value)?.label; + const renderFilterValue = (value: any) => { + return filters.find((f: { value: any }) => f.value === value)?.label; }; const handleAdd = () => { - onChange({ name, type, filter, value }); + onChange({ name, type, operator, value: isEquals ? selected : value }); }; - const handleMenuSelect = value => { - setValue(value); + const handleMenuSelect = (close: () => void, value: string) => { + setSelected(value); + close(); }; - const showMenu = - [OPERATORS.equals, OPERATORS.notEquals].includes(filter as any) && - !(filteredValues?.length === 1 && filteredValues[0] === formattedValues[value]); + const handleSearch = (value: string) => { + setSearch(value); + }; + + const handleReset = () => { + setSelected(''); + setValue(''); + setSearch(''); + refetch(); + }; + + const handleOperatorChange = (value: any) => { + setOperator(value); + + if ([OPERATORS.equals, OPERATORS.notEquals].includes(value)) { + setValue(''); + } else { + setSelected(''); + } + }; + + const handleBlur = () => { + window.setTimeout(() => setShowMenu(false), 500); + }; return (
@@ -97,35 +143,54 @@ export default function FieldFilterEditForm({ setFilter(key)} + onChange={handleOperatorChange} > {({ value, label }) => { return {label}; }} )} - - setValue(e.target.value)} - /> - {showMenu && ( - + {selected && isEquals && ( +
+ {selected} + + + +
+ )} + {!selected && isEquals && ( +
+ setValue(e.target.value)} + onSearch={handleSearch} + delay={500} + onFocus={() => setShowMenu(true)} + onBlur={handleBlur} + /> + {showMenu && ( - - )} - + )} +
+ )} + {!selected && !isEquals && ( + setValue(e.target.value)} + /> + )} - @@ -136,17 +201,23 @@ export default function FieldFilterEditForm({ const ResultsMenu = ({ values, type, isLoading, onSelect }) => { const { formatValue } = useFormat(); if (isLoading) { - return ; + return ( + + + + + + ); } if (!values?.length) { - return null; + return

poop

; } return ( - + {values?.map(value => { - return {safeDecodeURIComponent(formatValue(value, type))}; + return {formatValue(value, type)}; })} ); diff --git a/src/app/(main)/reports/[reportId]/FilterParameters.module.css b/src/app/(main)/reports/[reportId]/FilterParameters.module.css index 022c0d7e..939d0652 100644 --- a/src/app/(main)/reports/[reportId]/FilterParameters.module.css +++ b/src/app/(main)/reports/[reportId]/FilterParameters.module.css @@ -15,7 +15,7 @@ white-space: nowrap; } -.filter { +.op { color: var(--blue900); background-color: var(--blue100); font-size: 12px; diff --git a/src/app/(main)/reports/[reportId]/FilterParameters.tsx b/src/app/(main)/reports/[reportId]/FilterParameters.tsx index cd7a555e..c712e9a4 100644 --- a/src/app/(main)/reports/[reportId]/FilterParameters.tsx +++ b/src/app/(main)/reports/[reportId]/FilterParameters.tsx @@ -1,5 +1,4 @@ import { useContext } from 'react'; -import { safeDecodeURIComponent } from 'next-basics'; import { useMessages, useFormat, useFilters, useFields } from 'components/hooks'; import Icons from 'components/icons'; import { Button, FormRow, Icon, Popup, PopupTrigger } from 'react-basics'; @@ -15,9 +14,8 @@ export function FilterParameters() { const { report, updateReport } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); const { formatValue } = useFormat(); - const { filterLabels } = useFilters(); const { parameters } = report || {}; - const { websiteId, filters } = parameters || {}; + const { websiteId, filters, dateRange } = parameters || {}; const { fields } = useFields(); const handleAdd = (value: { name: any }) => { @@ -30,7 +28,7 @@ export function FilterParameters() { updateReport({ parameters: { filters: filters.filter(f => f.name !== name) } }); }; - const handleChange = filter => { + const handleChange = (close: () => void, filter: { name: any }) => { updateReport({ parameters: { filters: filters.map(f => { @@ -41,6 +39,7 @@ export function FilterParameters() { }), }, }); + close(); }; const AddButton = () => { @@ -67,44 +66,66 @@ export function FilterParameters() { return ( }> - {filters.map(({ name, filter, value }: { name: string; filter: string; value: string }) => { - const label = fields.find(f => f.name === name)?.label; - const isEquals = [OPERATORS.equals, OPERATORS.notEquals].includes(filter as any); - return ( - handleRemove(name)}> - - - ); - })} + {filters.map( + ({ name, operator, value }: { name: string; operator: string; value: string }) => { + const label = fields.find(f => f.name === name)?.label; + const isEquals = [OPERATORS.equals, OPERATORS.notEquals].includes(operator as any); + return ( + handleRemove(name)}> + + + ); + }, + )} ); } -const FilterParameter = ({ name, label, filter, value, type = 'string', onChange }) => { +const FilterParameter = ({ + websiteId, + name, + label, + operator, + value, + type = 'string', + startDate, + endDate, + onChange, +}) => { + const { filterLabels } = useFilters(); + return (
{label}
-
{filter}
-
{safeDecodeURIComponent(value)}
+
{filterLabels[operator]}
+
{value}
- - - + {(close: any) => ( + + + + )} ); diff --git a/src/app/(main)/reports/[reportId]/FilterSelectForm.tsx b/src/app/(main)/reports/[reportId]/FilterSelectForm.tsx index f43ac5f6..b81c8576 100644 --- a/src/app/(main)/reports/[reportId]/FilterSelectForm.tsx +++ b/src/app/(main)/reports/[reportId]/FilterSelectForm.tsx @@ -1,11 +1,12 @@ import { useState } from 'react'; import FieldSelectForm from './FieldSelectForm'; import FieldFilterEditForm from './FieldFilterEditForm'; +import { useDateRange } from 'components/hooks'; export interface FilterSelectFormProps { websiteId?: string; fields: any[]; - onChange?: (filter: { name: string; type: string; filter: string; value: string }) => void; + onChange?: (filter: { name: string; type: string; operator: string; value: string }) => void; allowFilterSelect?: boolean; } @@ -16,6 +17,7 @@ export default function FilterSelectForm({ allowFilterSelect, }: FilterSelectFormProps) { const [field, setField] = useState<{ name: string; label: string; type: string }>(); + const [{ startDate, endDate }] = useDateRange(websiteId); if (!field) { return ; @@ -29,6 +31,8 @@ export default function FilterSelectForm({ name={name} label={label} type={type} + startDate={startDate} + endDate={endDate} onChange={onChange} allowFilterSelect={allowFilterSelect} isNew={true} diff --git a/src/components/hooks/queries/useWebsiteValues.ts b/src/components/hooks/queries/useWebsiteValues.ts index ab287c07..02e26fc3 100644 --- a/src/components/hooks/queries/useWebsiteValues.ts +++ b/src/components/hooks/queries/useWebsiteValues.ts @@ -1,19 +1,30 @@ import { useApi } from 'components/hooks'; -import { subDays } from 'date-fns'; -export function useWebsiteValues(websiteId: string, type: string) { - const now = Date.now(); +export function useWebsiteValues({ + websiteId, + type, + startDate, + endDate, + search, +}: { + websiteId: string; + type: string; + startDate: Date; + endDate: Date; + search?: string; +}) { const { get, useQuery } = useApi(); return useQuery({ - queryKey: ['websites:values', websiteId, type], + queryKey: ['websites:values', { websiteId, type, startDate, endDate, search }], queryFn: () => get(`/websites/${websiteId}/values`, { type, - startAt: +subDays(now, 90), - endAt: now, + startAt: +startDate, + endAt: +endDate, + search, }), - enabled: !!(websiteId && type), + enabled: !!(websiteId && type && startDate && endDate), }); } diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 4be15db2..e41d987b 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -61,8 +61,8 @@ function getDateFormat(date: Date) { return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; } -function mapFilter(column: string, filter: string, name: string, type: string = 'String') { - switch (filter) { +function mapFilter(column: string, operator: string, name: string, type: string = 'String') { + switch (operator) { case OPERATORS.equals: return `${column} = {${name}:${type}}`; case OPERATORS.notEquals: diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index b79d4249..c3dd071b 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -92,16 +92,20 @@ function getTimestampDiffQuery(field1: string, field2: string): string { } } -function mapFilter(column: string, op: string, name: string, type = 'varchar') { - switch (op) { +function mapFilter(column: string, operator: string, name: string, type: string = '') { + const db = getDatabaseType(); + const like = db === POSTGRESQL ? 'ilike' : 'like'; + const value = `{{${name}${type ? `::${type}` : ''}}}`; + + switch (operator) { case OPERATORS.equals: - return `${column} = {{${name}::${type}}}`; + return `${column} = ${value}`; case OPERATORS.notEquals: - return `${column} != {{${name}::${type}}}`; + return `${column} != ${value}`; case OPERATORS.contains: - return `${column} ilike {{${name}::${type}}}`; + return `${column} ${like} ${value}`; case OPERATORS.doesNotContain: - return `${column} not ilike {{${name}::${type}}}`; + return `${column} not ${like} ${value}`; default: return ''; } @@ -110,11 +114,11 @@ function mapFilter(column: string, op: string, name: string, type = 'varchar') { function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; - const op = filter?.op ?? OPERATORS.equals; + const operator = filter?.operator ?? OPERATORS.equals; const column = filter?.column ?? FILTER_COLUMNS[key] ?? options?.columns?.[key]; if (filter !== undefined && column !== undefined) { - arr.push(`and ${mapFilter(column, op, key)}`); + arr.push(`and ${mapFilter(column, operator, key)}`); if (key === 'referrer') { arr.push( @@ -131,9 +135,12 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): function normalizeFilters(filters = {}) { return Object.keys(filters).reduce((obj, key) => { - const value = filters[key]; + const filter = filters[key]; + const value = filter?.value ?? filter; - obj[key] = value?.value ?? value; + obj[key] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(filter?.operator) + ? `%${value}%` + : value; return obj; }, {}); diff --git a/src/pages/api/reports/insights.ts b/src/pages/api/reports/insights.ts index c70d218e..efd3a6b5 100644 --- a/src/pages/api/reports/insights.ts +++ b/src/pages/api/reports/insights.ts @@ -13,7 +13,7 @@ export interface InsightsRequestBody { endDate: string; }; fields: { name: string; type: string; label: string }[]; - filters: { name: string; type: string; filter: string; value: string }[]; + filters: { name: string; type: string; operator: string; value: string }[]; groups: { name: string; type: string }[]; } @@ -42,7 +42,7 @@ const schema = { yup.object().shape({ name: yup.string().required(), type: yup.string().required(), - filter: yup.string().required(), + operator: yup.string().required(), value: yup.string().required(), }), ), diff --git a/src/pages/api/websites/[websiteId]/metrics.ts b/src/pages/api/websites/[websiteId]/metrics.ts index beb4dd50..d78c67c8 100644 --- a/src/pages/api/websites/[websiteId]/metrics.ts +++ b/src/pages/api/websites/[websiteId]/metrics.ts @@ -109,8 +109,8 @@ export default async ( if (search) { filters[column] = { column, - op: OPERATORS.contains, - value: '%' + search + '%', + operator: OPERATORS.contains, + value: search, }; } diff --git a/src/pages/api/websites/[websiteId]/values.ts b/src/pages/api/websites/[websiteId]/values.ts index 9cf2dba8..36ca7948 100644 --- a/src/pages/api/websites/[websiteId]/values.ts +++ b/src/pages/api/websites/[websiteId]/values.ts @@ -2,23 +2,33 @@ import { NextApiRequestQueryBody } from 'lib/types'; import { canViewWebsite } from 'lib/auth'; import { useAuth, useCors, useValidate } from 'lib/middleware'; import { NextApiResponse } from 'next'; -import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { + badRequest, + methodNotAllowed, + ok, + safeDecodeURIComponent, + unauthorized, +} from 'next-basics'; import { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; import { getValues } from 'queries'; import { parseDateRangeQuery } from 'lib/query'; +import * as yup from 'yup'; export interface ValuesRequestQuery { websiteId: string; + type: string; startAt: number; endAt: number; + search?: string; } -import * as yup from 'yup'; const schema = { GET: yup.object().shape({ websiteId: yup.string().uuid().required(), + type: yup.string().required(), startAt: yup.number().required(), endAt: yup.number().required(), + search: yup.string(), }), }; @@ -27,7 +37,7 @@ export default async (req: NextApiRequestQueryBody, res: Nex await useAuth(req, res); await useValidate(schema, req, res); - const { websiteId, type } = req.query; + const { websiteId, type, search } = req.query; const { startDate, endDate } = await parseDateRangeQuery(req); if (req.method === 'GET') { @@ -39,12 +49,18 @@ export default async (req: NextApiRequestQueryBody, res: Nex return unauthorized(res); } - const values = await getValues(websiteId, FILTER_COLUMNS[type as string], startDate, endDate); + const values = await getValues( + websiteId, + FILTER_COLUMNS[type as string], + startDate, + endDate, + search, + ); return ok( res, values - .map(({ value }) => value) + .map(({ value }) => safeDecodeURIComponent(value)) .filter(n => n) .sort(), ); diff --git a/src/queries/analytics/getValues.ts b/src/queries/analytics/getValues.ts index 44955423..572fbac2 100644 --- a/src/queries/analytics/getValues.ts +++ b/src/queries/analytics/getValues.ts @@ -3,7 +3,7 @@ import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; export async function getValues( - ...args: [websiteId: string, column: string, startDate: Date, endDate: Date] + ...args: [websiteId: string, column: string, startDate: Date, endDate: Date, search: string] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -11,42 +11,72 @@ export async function getValues( }); } -async function relationalQuery(websiteId: string, column: string, startDate: Date, endDate: Date) { +async function relationalQuery( + websiteId: string, + column: string, + startDate: Date, + endDate: Date, + search: string, +) { const { rawQuery } = prisma; + let searchQuery = ''; + + if (search) { + searchQuery = `and ${column} LIKE {{search}}`; + } return rawQuery( ` - select distinct ${column} as "value" + select ${column} as "value", count(*) from website_event inner join session on session.session_id = website_event.session_id where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} - limit 500 + ${searchQuery} + group by 1 + order by 2 desc + limit 10 `, { websiteId, startDate, endDate, + search: `%${search}%`, }, ); } -async function clickhouseQuery(websiteId: string, column: string, startDate: Date, endDate: Date) { +async function clickhouseQuery( + websiteId: string, + column: string, + startDate: Date, + endDate: Date, + search: string, +) { const { rawQuery } = clickhouse; + let searchQuery = ''; + + if (search) { + searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`; + } return rawQuery( ` - select distinct ${column} as value + select ${column} as value, count(*) from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - limit 500 + ${searchQuery} + group by 1 + order by 2 desc + limit 10 `, { websiteId, startDate, endDate, + search, }, ); }