Refactored filter parameters.

src/app/(main)/reports/[reportId]/FieldAddForm.tsx

@@ -3,8 +3,6 @@ import { createPortal } from 'react-dom';
 import { REPORT_PARAMETERS } from 'lib/constants';
 import PopupForm from './PopupForm';
 import FieldSelectForm from './FieldSelectForm';
-import FieldAggregateForm from './FieldAggregateForm';
-import FieldFilterEditForm from './FieldFilterEditForm';
 
 export function FieldAddForm({
   fields = [],
@@ -17,7 +15,11 @@ export function FieldAddForm({
   onAdd: (group: string, value: string) => void;
   onClose: () => void;
 }) {
-  const [selected, setSelected] = useState<{ name: string; type: string; value: string }>();
+  const [selected, setSelected] = useState<{
+    name: string;
+    type: string;
+    value: string;
+  }>();
 
   const handleSelect = (value: any) => {
     const { type } = value;
@@ -39,12 +41,6 @@ export function FieldAddForm({
   return createPortal(
     <PopupForm>
       {!selected && <FieldSelectForm fields={fields} onSelect={handleSelect} />}
-      {selected && group === REPORT_PARAMETERS.fields && (
-        <FieldAggregateForm {...selected} onSelect={handleSave} />
-      )}
-      {selected && group === REPORT_PARAMETERS.filters && (
-        <FieldFilterEditForm {...selected} onChange={handleSave} />
-      )}
     </PopupForm>,
     document.body,
   );

src/app/(main)/reports/[reportId]/FieldFilterEditForm.module.css

@@ -1,12 +1,7 @@
-.popup {
+.menu {
-  display: flex;
+  position: absolute;
   max-width: 300px;
   max-height: 210px;
-  overflow: hidden;
-}
-
-.popup > div {
-  overflow-y: auto;
 }
 
 .filter {
@@ -16,9 +11,26 @@
 }
 
 .dropdown {
-  min-width: 180px;
+  min-width: 200px;
 }
 
 .text {
-  min-width: 180px;
+  min-width: 200px;
+}
+
+.selected {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 8px 16px;
+  white-space: nowrap;
+  min-width: 200px;
+  font-weight: 900;
+  background: var(--base100);
+  border-radius: var(--border-radius);
+  cursor: pointer;
+}
+
+.search {
+  position: relative;
 }

src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx

@@ -6,14 +6,15 @@ import {
   Flexbox,
   Dropdown,
   Button,
+  SearchField,
   TextField,
+  Text,
+  Icon,
+  Icons,
   Menu,
-  Popup,
-  PopupTrigger,
   Loading,
 } from 'react-basics';
 import { useMessages, useFilters, useFormat, useLocale, useWebsiteValues } from 'components/hooks';
-import { safeDecodeURIComponent } from 'next-basics';
 import { OPERATORS } from 'lib/constants';
 import styles from './FieldFilterEditForm.module.css';
 
@@ -22,8 +23,11 @@ export interface FieldFilterFormProps {
   name: string;
   label?: string;
   type: string;
+  startDate: Date;
+  endDate: Date;
+  operator?: string;
   defaultValue?: string;
-  onChange?: (filter: { name: string; type: string; filter: string; value: string }) => void;
+  onChange?: (filter: { name: string; type: string; operator: string; value: string }) => void;
   allowFilterSelect?: boolean;
   isNew?: boolean;
 }
@@ -33,19 +37,37 @@ export default function FieldFilterEditForm({
   name,
   label,
   type,
-  defaultValue,
+  startDate,
+  endDate,
+  operator: defaultOperator = 'eq',
+  defaultValue = '',
   onChange,
   allowFilterSelect = true,
   isNew,
 }: FieldFilterFormProps) {
   const { formatMessage, labels } = useMessages();
-  const [filter, setFilter] = useState('eq');
+  const [operator, setOperator] = useState(defaultOperator);
-  const [value, setValue] = useState(defaultValue ?? '');
+  const [value, setValue] = useState(defaultValue);
+  const [showMenu, setShowMenu] = useState(false);
+  const isEquals = [OPERATORS.equals, OPERATORS.notEquals].includes(operator as any);
+  const [search, setSearch] = useState('');
+  const [selected, setSelected] = useState(isEquals ? value : '');
   const { getFilters } = useFilters();
   const { formatValue } = useFormat();
   const { locale } = useLocale();
   const filters = getFilters(type);
-  const { data: values = [], isLoading } = useWebsiteValues(websiteId, name);
+  const isDisabled = !operator || (isEquals && !selected) || (!isEquals && !value);
+  const {
+    data: values = [],
+    isLoading,
+    refetch,
+  } = useWebsiteValues({
+    websiteId,
+    type: name,
+    startDate,
+    endDate,
+    search,
+  });
 
   const formattedValues = useMemo(() => {
     if (!values) {
@@ -69,25 +91,49 @@ export default function FieldFilterEditForm({
 
   const filteredValues = useMemo(() => {
     return value
-      ? values.filter(n => formattedValues[n].toLowerCase().includes(value.toLowerCase()))
+      ? values.filter((n: string | number) =>
+          formattedValues[n].toLowerCase().includes(value.toLowerCase()),
+        )
       : values;
   }, [value, formattedValues]);
 
-  const renderFilterValue = value => {
+  const renderFilterValue = (value: any) => {
-    return filters.find(f => f.value === value)?.label;
+    return filters.find((f: { value: any }) => f.value === value)?.label;
   };
 
   const handleAdd = () => {
-    onChange({ name, type, filter, value });
+    onChange({ name, type, operator, value: isEquals ? selected : value });
   };
 
-  const handleMenuSelect = value => {
+  const handleMenuSelect = (close: () => void, value: string) => {
-    setValue(value);
+    setSelected(value);
+    close();
   };
 
-  const showMenu =
+  const handleSearch = (value: string) => {
-    [OPERATORS.equals, OPERATORS.notEquals].includes(filter as any) &&
+    setSearch(value);
-    !(filteredValues?.length === 1 && filteredValues[0] === formattedValues[value]);
+  };
+
+  const handleReset = () => {
+    setSelected('');
+    setValue('');
+    setSearch('');
+    refetch();
+  };
+
+  const handleOperatorChange = (value: any) => {
+    setOperator(value);
+
+    if ([OPERATORS.equals, OPERATORS.notEquals].includes(value)) {
+      setValue('');
+    } else {
+      setSelected('');
+    }
+  };
+
+  const handleBlur = () => {
+    window.setTimeout(() => setShowMenu(false), 500);
+  };
 
   return (
     <Form>
@@ -97,35 +143,54 @@ export default function FieldFilterEditForm({
             <Dropdown
               className={styles.dropdown}
               items={filters}
-              value={filter}
+              value={operator}
               renderValue={renderFilterValue}
-              onChange={(key: any) => setFilter(key)}
+              onChange={handleOperatorChange}
             >
               {({ value, label }) => {
                 return <Item key={value}>{label}</Item>;
               }}
             </Dropdown>
           )}
-          <PopupTrigger>
+          {selected && isEquals && (
-            <TextField
+            <div className={styles.selected} onClick={handleReset}>
-              className={styles.text}
+              <Text>{selected}</Text>
-              value={decodeURIComponent(value)}
+              <Icon>
-              placeholder={formatMessage(labels.enter)}
+                <Icons.Close />
-              onChange={e => setValue(e.target.value)}
+              </Icon>
-            />
+            </div>
-            {showMenu && (
+          )}
-              <Popup className={styles.popup} alignment="start">
+          {!selected && isEquals && (
+            <div className={styles.search}>
+              <SearchField
+                className={styles.text}
+                value={value}
+                placeholder={formatMessage(labels.enter)}
+                onChange={e => setValue(e.target.value)}
+                onSearch={handleSearch}
+                delay={500}
+                onFocus={() => setShowMenu(true)}
+                onBlur={handleBlur}
+              />
+              {showMenu && (
                 <ResultsMenu
                   values={filteredValues}
                   type={name}
                   isLoading={isLoading}
-                  onSelect={handleMenuSelect}
+                  onSelect={handleMenuSelect.bind(null, close)}
                 />
-              </Popup>
+              )}
-            )}
+            </div>
-          </PopupTrigger>
+          )}
+          {!selected && !isEquals && (
+            <TextField
+              className={styles.text}
+              value={value}
+              onChange={e => setValue(e.target.value)}
+            />
+          )}
         </Flexbox>
-        <Button variant="primary" onClick={handleAdd} disabled={!filter || !value}>
+        <Button variant="primary" onClick={handleAdd} disabled={isDisabled}>
           {isNew ? formatMessage(labels.add) : formatMessage(labels.update)}
         </Button>
       </FormRow>
@@ -136,17 +201,23 @@ export default function FieldFilterEditForm({
 const ResultsMenu = ({ values, type, isLoading, onSelect }) => {
   const { formatValue } = useFormat();
   if (isLoading) {
-    return <Loading icon="dots" position="center" />;
+    return (
+      <Menu>
+        <Item>
+          <Loading icon="dots" position="center" />
+        </Item>
+      </Menu>
+    );
   }
 
   if (!values?.length) {
-    return null;
+    return <h1>poop</h1>;
   }
 
   return (
-    <Menu variant="popup" onSelect={onSelect}>
+    <Menu className={styles.menu} variant="popup" onSelect={onSelect}>
       {values?.map(value => {
-        return <Item key={value}>{safeDecodeURIComponent(formatValue(value, type))}</Item>;
+        return <Item key={value}>{formatValue(value, type)}</Item>;
       })}
     </Menu>
   );

src/app/(main)/reports/[reportId]/FilterParameters.module.css

@@ -15,7 +15,7 @@
   white-space: nowrap;
 }
 
-.filter {
+.op {
   color: var(--blue900);
   background-color: var(--blue100);
   font-size: 12px;

src/app/(main)/reports/[reportId]/FilterParameters.tsx

@@ -1,5 +1,4 @@
 import { useContext } from 'react';
-import { safeDecodeURIComponent } from 'next-basics';
 import { useMessages, useFormat, useFilters, useFields } from 'components/hooks';
 import Icons from 'components/icons';
 import { Button, FormRow, Icon, Popup, PopupTrigger } from 'react-basics';
@@ -15,9 +14,8 @@ export function FilterParameters() {
   const { report, updateReport } = useContext(ReportContext);
   const { formatMessage, labels } = useMessages();
   const { formatValue } = useFormat();
-  const { filterLabels } = useFilters();
   const { parameters } = report || {};
-  const { websiteId, filters } = parameters || {};
+  const { websiteId, filters, dateRange } = parameters || {};
   const { fields } = useFields();
 
   const handleAdd = (value: { name: any }) => {
@@ -30,7 +28,7 @@ export function FilterParameters() {
     updateReport({ parameters: { filters: filters.filter(f => f.name !== name) } });
   };
 
-  const handleChange = filter => {
+  const handleChange = (close: () => void, filter: { name: any }) => {
     updateReport({
       parameters: {
         filters: filters.map(f => {
@@ -41,6 +39,7 @@ export function FilterParameters() {
         }),
       },
     });
+    close();
   };
 
   const AddButton = () => {
@@ -67,44 +66,66 @@ export function FilterParameters() {
   return (
     <FormRow label={formatMessage(labels.filters)} action={<AddButton />}>
       <ParameterList>
-        {filters.map(({ name, filter, value }: { name: string; filter: string; value: string }) => {
+        {filters.map(
-          const label = fields.find(f => f.name === name)?.label;
+          ({ name, operator, value }: { name: string; operator: string; value: string }) => {
-          const isEquals = [OPERATORS.equals, OPERATORS.notEquals].includes(filter as any);
+            const label = fields.find(f => f.name === name)?.label;
-          return (
+            const isEquals = [OPERATORS.equals, OPERATORS.notEquals].includes(operator as any);
-            <ParameterList.Item key={name} onRemove={() => handleRemove(name)}>
+            return (
-              <FilterParameter
+              <ParameterList.Item key={name} onRemove={() => handleRemove(name)}>
-                name={name}
+                <FilterParameter
-                label={label}
+                  {...dateRange}
-                filter={filterLabels[filter]}
+                  websiteId={websiteId}
-                value={isEquals ? formatValue(value, name) : value}
+                  name={name}
-                onChange={handleChange}
+                  label={label}
-              />
+                  operator={operator}
-            </ParameterList.Item>
+                  value={isEquals ? formatValue(value, name) : value}
-          );
+                  onChange={handleChange}
-        })}
+                />
+              </ParameterList.Item>
+            );
+          },
+        )}
       </ParameterList>
     </FormRow>
   );
 }
 
-const FilterParameter = ({ name, label, filter, value, type = 'string', onChange }) => {
+const FilterParameter = ({
+  websiteId,
+  name,
+  label,
+  operator,
+  value,
+  type = 'string',
+  startDate,
+  endDate,
+  onChange,
+}) => {
+  const { filterLabels } = useFilters();
+
   return (
     <PopupTrigger>
       <div className={styles.item}>
         <div className={styles.label}>{label}</div>
-        <div className={styles.filter}>{filter}</div>
+        <div className={styles.op}>{filterLabels[operator]}</div>
-        <div className={styles.value}>{safeDecodeURIComponent(value)}</div>
+        <div className={styles.value}>{value}</div>
       </div>
       <Popup className={styles.edit} alignment="start">
-        <PopupForm>
+        {(close: any) => (
-          <FieldFilterEditForm
+          <PopupForm>
-            name={name}
+            <FieldFilterEditForm
-            label={label}
+              websiteId={websiteId}
-            type={type}
+              name={name}
-            defaultValue={value}
+              label={label}
-            onChange={onChange}
+              type={type}
-          />
+              startDate={startDate}
-        </PopupForm>
+              endDate={endDate}
+              operator={operator}
+              defaultValue={value}
+              onChange={onChange.bind(null, close)}
+            />
+          </PopupForm>
+        )}
       </Popup>
     </PopupTrigger>
   );

src/app/(main)/reports/[reportId]/FilterSelectForm.tsx

@@ -1,11 +1,12 @@
 import { useState } from 'react';
 import FieldSelectForm from './FieldSelectForm';
 import FieldFilterEditForm from './FieldFilterEditForm';
+import { useDateRange } from 'components/hooks';
 
 export interface FilterSelectFormProps {
   websiteId?: string;
   fields: any[];
-  onChange?: (filter: { name: string; type: string; filter: string; value: string }) => void;
+  onChange?: (filter: { name: string; type: string; operator: string; value: string }) => void;
   allowFilterSelect?: boolean;
 }
 
@@ -16,6 +17,7 @@ export default function FilterSelectForm({
   allowFilterSelect,
 }: FilterSelectFormProps) {
   const [field, setField] = useState<{ name: string; label: string; type: string }>();
+  const [{ startDate, endDate }] = useDateRange(websiteId);
 
   if (!field) {
     return <FieldSelectForm fields={fields} onSelect={setField} showType={false} />;
@@ -29,6 +31,8 @@ export default function FilterSelectForm({
       name={name}
       label={label}
       type={type}
+      startDate={startDate}
+      endDate={endDate}
       onChange={onChange}
       allowFilterSelect={allowFilterSelect}
       isNew={true}

src/components/hooks/queries/useWebsiteValues.ts

@@ -1,19 +1,30 @@
 import { useApi } from 'components/hooks';
-import { subDays } from 'date-fns';
 
-export function useWebsiteValues(websiteId: string, type: string) {
+export function useWebsiteValues({
-  const now = Date.now();
+  websiteId,
+  type,
+  startDate,
+  endDate,
+  search,
+}: {
+  websiteId: string;
+  type: string;
+  startDate: Date;
+  endDate: Date;
+  search?: string;
+}) {
   const { get, useQuery } = useApi();
 
   return useQuery({
-    queryKey: ['websites:values', websiteId, type],
+    queryKey: ['websites:values', { websiteId, type, startDate, endDate, search }],
     queryFn: () =>
       get(`/websites/${websiteId}/values`, {
         type,
-        startAt: +subDays(now, 90),
+        startAt: +startDate,
-        endAt: now,
+        endAt: +endDate,
+        search,
       }),
-    enabled: !!(websiteId && type),
+    enabled: !!(websiteId && type && startDate && endDate),
   });
 }
 

src/lib/clickhouse.ts

@@ -61,8 +61,8 @@ function getDateFormat(date: Date) {
   return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
 }
 
-function mapFilter(column: string, filter: string, name: string, type: string = 'String') {
+function mapFilter(column: string, operator: string, name: string, type: string = 'String') {
-  switch (filter) {
+  switch (operator) {
     case OPERATORS.equals:
       return `${column} = {${name}:${type}}`;
     case OPERATORS.notEquals:

src/lib/prisma.ts

@@ -92,16 +92,20 @@ function getTimestampDiffQuery(field1: string, field2: string): string {
   }
 }
 
-function mapFilter(column: string, op: string, name: string, type = 'varchar') {
+function mapFilter(column: string, operator: string, name: string, type: string = '') {
-  switch (op) {
+  const db = getDatabaseType();
+  const like = db === POSTGRESQL ? 'ilike' : 'like';
+  const value = `{{${name}${type ? `::${type}` : ''}}}`;
+
+  switch (operator) {
     case OPERATORS.equals:
-      return `${column} = {{${name}::${type}}}`;
+      return `${column} = ${value}`;
     case OPERATORS.notEquals:
-      return `${column} != {{${name}::${type}}}`;
+      return `${column} != ${value}`;
     case OPERATORS.contains:
-      return `${column} ilike {{${name}::${type}}}`;
+      return `${column} ${like} ${value}`;
     case OPERATORS.doesNotContain:
-      return `${column} not ilike {{${name}::${type}}}`;
+      return `${column} not ${like} ${value}`;
     default:
       return '';
   }
@@ -110,11 +114,11 @@ function mapFilter(column: string, op: string, name: string, type = 'varchar') {
 function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {
   const query = Object.keys(filters).reduce((arr, key) => {
     const filter = filters[key];
-    const op = filter?.op ?? OPERATORS.equals;
+    const operator = filter?.operator ?? OPERATORS.equals;
     const column = filter?.column ?? FILTER_COLUMNS[key] ?? options?.columns?.[key];
 
     if (filter !== undefined && column !== undefined) {
-      arr.push(`and ${mapFilter(column, op, key)}`);
+      arr.push(`and ${mapFilter(column, operator, key)}`);
 
       if (key === 'referrer') {
         arr.push(
@@ -131,9 +135,12 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}):
 
 function normalizeFilters(filters = {}) {
   return Object.keys(filters).reduce((obj, key) => {
-    const value = filters[key];
+    const filter = filters[key];
+    const value = filter?.value ?? filter;
 
-    obj[key] = value?.value ?? value;
+    obj[key] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(filter?.operator)
+      ? `%${value}%`
+      : value;
 
     return obj;
   }, {});

src/pages/api/reports/insights.ts

@@ -13,7 +13,7 @@ export interface InsightsRequestBody {
     endDate: string;
   };
   fields: { name: string; type: string; label: string }[];
-  filters: { name: string; type: string; filter: string; value: string }[];
+  filters: { name: string; type: string; operator: string; value: string }[];
   groups: { name: string; type: string }[];
 }
 
@@ -42,7 +42,7 @@ const schema = {
       yup.object().shape({
         name: yup.string().required(),
         type: yup.string().required(),
-        filter: yup.string().required(),
+        operator: yup.string().required(),
         value: yup.string().required(),
       }),
     ),

src/pages/api/websites/[websiteId]/metrics.ts

@@ -109,8 +109,8 @@ export default async (
     if (search) {
       filters[column] = {
         column,
-        op: OPERATORS.contains,
+        operator: OPERATORS.contains,
-        value: '%' + search + '%',
+        value: search,
       };
     }
 

src/pages/api/websites/[websiteId]/values.ts

@@ -2,23 +2,33 @@ import { NextApiRequestQueryBody } from 'lib/types';
 import { canViewWebsite } from 'lib/auth';
 import { useAuth, useCors, useValidate } from 'lib/middleware';
 import { NextApiResponse } from 'next';
-import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';
+import {
+  badRequest,
+  methodNotAllowed,
+  ok,
+  safeDecodeURIComponent,
+  unauthorized,
+} from 'next-basics';
 import { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
 import { getValues } from 'queries';
 import { parseDateRangeQuery } from 'lib/query';
+import * as yup from 'yup';
 
 export interface ValuesRequestQuery {
   websiteId: string;
+  type: string;
   startAt: number;
   endAt: number;
+  search?: string;
 }
 
-import * as yup from 'yup';
 const schema = {
   GET: yup.object().shape({
     websiteId: yup.string().uuid().required(),
+    type: yup.string().required(),
     startAt: yup.number().required(),
     endAt: yup.number().required(),
+    search: yup.string(),
   }),
 };
 
@@ -27,7 +37,7 @@ export default async (req: NextApiRequestQueryBody<ValuesRequestQuery>, res: Nex
   await useAuth(req, res);
   await useValidate(schema, req, res);
 
-  const { websiteId, type } = req.query;
+  const { websiteId, type, search } = req.query;
   const { startDate, endDate } = await parseDateRangeQuery(req);
 
   if (req.method === 'GET') {
@@ -39,12 +49,18 @@ export default async (req: NextApiRequestQueryBody<ValuesRequestQuery>, res: Nex
       return unauthorized(res);
     }
 
-    const values = await getValues(websiteId, FILTER_COLUMNS[type as string], startDate, endDate);
+    const values = await getValues(
+      websiteId,
+      FILTER_COLUMNS[type as string],
+      startDate,
+      endDate,
+      search,
+    );
 
     return ok(
       res,
       values
-        .map(({ value }) => value)
+        .map(({ value }) => safeDecodeURIComponent(value))
         .filter(n => n)
         .sort(),
     );

src/queries/analytics/getValues.ts

@@ -3,7 +3,7 @@ import clickhouse from 'lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
 
 export async function getValues(
-  ...args: [websiteId: string, column: string, startDate: Date, endDate: Date]
+  ...args: [websiteId: string, column: string, startDate: Date, endDate: Date, search: string]
 ) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -11,42 +11,72 @@ export async function getValues(
   });
 }
 
-async function relationalQuery(websiteId: string, column: string, startDate: Date, endDate: Date) {
+async function relationalQuery(
+  websiteId: string,
+  column: string,
+  startDate: Date,
+  endDate: Date,
+  search: string,
+) {
   const { rawQuery } = prisma;
+  let searchQuery = '';
+
+  if (search) {
+    searchQuery = `and ${column} LIKE {{search}}`;
+  }
 
   return rawQuery(
     `
-    select distinct ${column} as "value"
+    select ${column} as "value", count(*)
     from website_event
     inner join session
       on session.session_id = website_event.session_id
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
-    limit 500
+      ${searchQuery}
+    group by 1
+    order by 2 desc
+    limit 10
     `,
     {
       websiteId,
       startDate,
       endDate,
+      search: `%${search}%`,
     },
   );
 }
 
-async function clickhouseQuery(websiteId: string, column: string, startDate: Date, endDate: Date) {
+async function clickhouseQuery(
+  websiteId: string,
+  column: string,
+  startDate: Date,
+  endDate: Date,
+  search: string,
+) {
   const { rawQuery } = clickhouse;
+  let searchQuery = '';
+
+  if (search) {
+    searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;
+  }
 
   return rawQuery(
     `
-    select distinct ${column} as value
+    select ${column} as value, count(*)
     from website_event
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-    limit 500
+      ${searchQuery}
+    group by 1
+    order by 2 desc
+    limit 10
     `,
     {
       websiteId,
       startDate,
       endDate,
+      search,
     },
   );
 }