Refactored query filters.

components/pages/reports/FieldSelectForm.js

@@ -13,7 +13,7 @@ export default function FieldSelectForm({ fields, onSelect }) {
             return (
               <Item key={index} className={styles.item}>
                 <div>{label || name}</div>
-                <div className={styles.type}>{type}</div>
+                {type && <div className={styles.type}>{type}</div>}
               </Item>
             );
           })}

components/pages/reports/funnel/FunnelParameters.js

@@ -16,6 +16,7 @@ import UrlAddForm from './UrlAddForm';
 import { ReportContext } from 'components/pages/reports/Report';
 import BaseParameters from '../BaseParameters';
 import ParameterList from '../ParameterList';
+import PopupForm from '../PopupForm';
 
 export function FunnelParameters() {
   const { report, runReport, updateReport, isRunning } = useContext(ReportContext);
@@ -53,7 +54,11 @@ export function FunnelParameters() {
         </Icon>
         <Popup position="bottom" alignment="start">
           {(close, element) => {
-            return <UrlAddForm element={element} onAdd={handleAddUrl} onClose={close} />;
+            return (
+              <PopupForm element={element} onClose={close}>
+                <UrlAddForm onAdd={handleAddUrl} />
+              </PopupForm>
+            );
           }}
         </Popup>
       </PopupTrigger>

components/pages/reports/funnel/UrlAddForm.js

@@ -2,16 +2,14 @@ import { useState } from 'react';
 import { useMessages } from 'hooks';
 import { Button, Form, FormRow, TextField, Flexbox } from 'react-basics';
 import styles from './UrlAddForm.module.css';
-import PopupForm from '../PopupForm';
 
-export function UrlAddForm({ defaultValue = '', element, onAdd, onClose }) {
+export function UrlAddForm({ defaultValue = '', onAdd }) {
   const [url, setUrl] = useState(defaultValue);
   const { formatMessage, labels } = useMessages();
 
   const handleSave = () => {
     onAdd(url);
     setUrl('');
-    onClose();
   };
 
   const handleChange = e => {
@@ -26,25 +24,23 @@ export function UrlAddForm({ defaultValue = '', element, onAdd, onClose }) {
   };
 
   return (
-    <PopupForm element={element}>
-      <Form>
-        <FormRow label={formatMessage(labels.url)}>
-          <Flexbox gap={10}>
-            <TextField
-              className={styles.input}
-              value={url}
-              onChange={handleChange}
-              autoFocus={true}
-              autoComplete="off"
-              onKeyDown={handleKeyDown}
-            />
-            <Button variant="primary" onClick={handleSave}>
-              {formatMessage(labels.add)}
-            </Button>
-          </Flexbox>
-        </FormRow>
-      </Form>
-    </PopupForm>
+    <Form>
+      <FormRow label={formatMessage(labels.url)}>
+        <Flexbox gap={10}>
+          <TextField
+            className={styles.input}
+            value={url}
+            onChange={handleChange}
+            autoFocus={true}
+            autoComplete="off"
+            onKeyDown={handleKeyDown}
+          />
+          <Button variant="primary" onClick={handleSave}>
+            {formatMessage(labels.add)}
+          </Button>
+        </Flexbox>
+      </FormRow>
+    </Form>
   );
 }
 

components/pages/reports/insights/InsightsParameters.js

@@ -2,12 +2,28 @@ import { useContext, useRef } from 'react';
 import { useMessages } from 'hooks';
 import { Form, FormRow, FormButtons, SubmitButton, PopupTrigger, Icon, Popup } from 'react-basics';
 import { ReportContext } from 'components/pages/reports/Report';
-import { REPORT_PARAMETERS, WEBSITE_EVENT_FIELDS } from 'lib/constants';
+import { REPORT_PARAMETERS } from 'lib/constants';
 import Icons from 'components/icons';
 import BaseParameters from '../BaseParameters';
-import FieldAddForm from '../FieldAddForm';
 import ParameterList from '../ParameterList';
 import styles from './InsightsParameters.module.css';
+import FieldSelectForm from '../FieldSelectForm';
+import PopupForm from '../PopupForm';
+import FieldFilterForm from '../FieldFilterForm';
+
+const fieldOptions = [
+  { name: 'url', type: 'string' },
+  { name: 'title', type: 'string' },
+  { name: 'referrer', type: 'string' },
+  { name: 'query', type: 'string' },
+  { name: 'browser', type: 'string' },
+  { name: 'os', type: 'string' },
+  { name: 'device', type: 'string' },
+  { name: 'country', type: 'string' },
+  { name: 'region', type: 'string' },
+  { name: 'city', type: 'string' },
+  { name: 'language', type: 'string' },
+];
 
 export function InsightsParameters() {
   const { report, runReport, updateReport, isRunning } = useContext(ReportContext);
@@ -16,7 +32,6 @@ export function InsightsParameters() {
   const { parameters } = report || {};
   const { websiteId, dateRange, fields, filters, groups } = parameters || {};
   const queryEnabled = websiteId && dateRange && fields?.length;
-  const fieldOptions = Object.keys(WEBSITE_EVENT_FIELDS).map(key => WEBSITE_EVENT_FIELDS[key]);
 
   const parameterGroups = [
     { label: formatMessage(labels.fields), group: REPORT_PARAMETERS.fields },
@@ -57,13 +72,14 @@ export function InsightsParameters() {
         <Popup position="bottom" alignment="start">
           {(close, element) => {
             return (
-              <FieldAddForm
-                fields={fieldOptions}
-                group={group}
-                element={element}
-                onAdd={handleAdd}
-                onClose={close}
-              />
+              <PopupForm element={element} onClose={close}>
+                {group === REPORT_PARAMETERS.fields && (
+                  <FieldSelectForm fields={fieldOptions} onSelect={handleAdd.bind(null, group)} />
+                )}
+                {group === REPORT_PARAMETERS.filters && (
+                  <FieldFilterForm fields={fieldOptions} onSelect={handleAdd.bind(null, group)} />
+                )}
+              </PopupForm>
             );
           }}
         </Popup>

lib/clickhouse.ts

@@ -70,6 +70,10 @@ function getFilterQuery(filters = {}) {
       arr.push(`and ${column} = {${key}:String}`);
     }
 
+    if (key === 'referrer') {
+      arr.push('and referrer_domain != {domain:String}');
+    }
+
     return arr;
   }, []);
 

lib/constants.ts

@@ -43,11 +43,6 @@ export const SESSION_COLUMNS = [
   'city',
 ];
 
-export const COLLECTION_TYPE = {
-  event: 'event',
-  identify: 'identify',
-};
-
 export const FILTER_COLUMNS = {
   url: 'url_path',
   referrer: 'referrer_domain',
@@ -57,6 +52,11 @@ export const FILTER_COLUMNS = {
   region: 'subdivision1',
 };
 
+export const COLLECTION_TYPE = {
+  event: 'event',
+  identify: 'identify',
+};
+
 export const EVENT_TYPE = {
   pageView: 1,
   customEvent: 2,
@@ -120,37 +120,6 @@ export const ROLE_PERMISSIONS = {
   [ROLES.teamMember]: [],
 } as const;
 
-export const WEBSITE_EVENT_FIELDS = {
-  eventId: { name: 'event_id', type: 'uuid', label: 'Event ID' },
-  websiteId: { name: 'website_id', type: 'uuid', label: 'Website ID' },
-  sessionId: { name: 'session_id', type: 'uuid', label: 'Session ID' },
-  createdAt: { name: 'created_at', type: 'date', label: 'Created date' },
-  urlPath: { name: 'url_path', type: 'string', label: 'URL path' },
-  urlQuery: { name: 'url_query', type: 'string', label: 'URL query' },
-  referrerPath: { name: 'referrer_path', type: 'string', label: 'Referrer path' },
-  referrerQuery: { name: 'referrer_query', type: 'string', label: 'Referrer query' },
-  referrerDomain: { name: 'referrer_domain', type: 'string', label: 'Referrer domain' },
-  pageTitle: { name: 'page_title', type: 'string', label: 'Page title' },
-  eventType: { name: 'event_type', type: 'string', label: 'Event type' },
-  eventName: { name: 'event_name', type: 'string', label: 'Event name' },
-};
-
-export const SESSION_FIELDS = {
-  sessionId: { name: 'session_id', type: 'uuid' },
-  websiteId: { name: 'website_id', type: 'uuid' },
-  hostname: { name: 'hostname', type: 'string' },
-  browser: { name: 'browser', type: 'string' },
-  os: { name: 'os', type: 'string' },
-  device: { name: 'device', type: 'string' },
-  screen: { name: 'screen', type: 'string' },
-  language: { name: 'language', type: 'string' },
-  country: { name: 'country', type: 'string' },
-  subdivision1: { name: 'subdivision1', type: 'string' },
-  subdivision2: { name: 'subdivision2', type: 'string' },
-  city: { name: 'city', type: 'string' },
-  createdAt: { name: 'created_at', type: 'date' },
-};
-
 export const THEME_COLORS = {
   light: {
     primary: '#2680eb',

lib/prisma.ts

@@ -73,6 +73,12 @@ function getFilterQuery(filters = {}): string {
       arr.push(`and ${column}={{${key}}}`);
     }
 
+    if (key === 'referrer') {
+      arr.push(
+        'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)',
+      );
+    }
+
     return arr;
   }, []);
 

pages/api/reports/insights.ts

@@ -11,7 +11,7 @@ export interface InsightsRequestBody {
     startDate: string;
     endDate: string;
   };
-  fields: string[];
+  fields: { name: string; type: string; value: string }[];
   filters: string[];
   groups: string[];
 }

pages/api/websites/[id]/metrics.ts

@@ -46,6 +46,7 @@ export default async (
     country,
     region,
     city,
+    language,
   } = req.query;
 
   if (req.method === 'GET') {
@@ -55,19 +56,26 @@ export default async (
 
     const { startDate, endDate } = await parseDateRangeQuery(req);
 
+    const filters = {
+      url,
+      referrer,
+      title,
+      query,
+      event,
+      os,
+      browser,
+      device,
+      country,
+      region,
+      city,
+      language,
+    };
+
+    filters[type] = undefined;
+
+    const column = FILTER_COLUMNS[type] || type;
+
     if (SESSION_COLUMNS.includes(type)) {
-      const column = FILTER_COLUMNS[type] || type;
-      const filters = {
-        os,
-        browser,
-        device,
-        country,
-        region,
-        city,
-      };
-
-      filters[type] = undefined;
-
       const data = await getSessionMetrics(websiteId, {
         startDate,
         endDate,
@@ -95,23 +103,6 @@ export default async (
     }
 
     if (EVENT_COLUMNS.includes(type)) {
-      const column = FILTER_COLUMNS[type] || type;
-      const filters = {
-        url,
-        referrer,
-        title,
-        query,
-        event,
-        os,
-        browser,
-        device,
-        country,
-        region,
-        city,
-      };
-
-      filters[type] = undefined;
-
       const data = await getPageviewMetrics(websiteId, {
         startDate,
         endDate,

queries/analytics/events/getEventMetrics.ts

@@ -47,11 +47,12 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite
     order by 2
     `,
     {
+      ...filters,
       websiteId,
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.customEvent,
-      ...filters,
+      domain: website.domain,
     },
   );
 }
@@ -82,6 +83,7 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.customEvent,
+      domain: website.domain,
     },
   );
 }

queries/analytics/pageviews/getPageviewMetrics.ts

@@ -34,22 +34,6 @@ async function relationalQuery(
   const { startDate, endDate, filters = {}, column } = criteria;
   const { rawQuery, parseFilters } = prisma;
   const website = await loadWebsite(websiteId);
-  const params: any = {
-    websiteId,
-    startDate: maxDate(startDate, website.resetAt),
-    endDate,
-    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
-    ...filters,
-  };
-
-  let excludeDomain = '';
-
-  if (column === 'referrer_domain') {
-    excludeDomain =
-      'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)';
-
-    params.domain = website.domain;
-  }
 
   const { filterQuery, joinSession } = parseFilters(filters);
 
@@ -61,13 +45,19 @@ async function relationalQuery(
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
       and event_type = {{eventType}}
-      ${excludeDomain}
       ${filterQuery}
     group by 1
     order by 2 desc
     limit 100
     `,
-    params,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+      domain: website.domain,
+    },
   );
 }
 
@@ -83,21 +73,6 @@ async function clickhouseQuery(
   const { startDate, endDate, filters = {}, column } = criteria;
   const { rawQuery, parseFilters } = clickhouse;
   const website = await loadWebsite(websiteId);
-  const params = {
-    ...filters,
-    websiteId,
-    startDate: maxDate(startDate, website.resetAt),
-    endDate,
-    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
-    domain: undefined,
-  };
-
-  let excludeDomain = '';
-
-  if (column === 'referrer_domain') {
-    excludeDomain = 'and referrer_domain != {domain:String}';
-    params.domain = website.domain;
-  }
 
   const { filterQuery } = parseFilters(filters);
 
@@ -108,12 +83,18 @@ async function clickhouseQuery(
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime} and {endDate:DateTime}
       and event_type = {eventType:UInt32}
-      ${excludeDomain}
       ${filterQuery}
     group by x
     order by y desc
     limit 100
     `,
-    params,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+      domain: website.domain,
+    },
   );
 }

queries/analytics/pageviews/getPageviewStats.ts

@@ -59,6 +59,7 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }
@@ -93,6 +94,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }

queries/analytics/reports/getInsights.ts

@@ -29,7 +29,7 @@ async function relationalQuery(
     y: number;
   }[]
 > {
-  const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria;
+  const { startDate, endDate, filters = [] } = criteria;
   const { parseFilters, rawQuery } = prisma;
   const website = await loadWebsite(websiteId);
   const params = {};
@@ -107,7 +107,7 @@ function parseFields(fields) {
 
     if (!count && value === 'total') {
       count = true;
-      arr = arr.concat(`count(*) as total`);
+      arr = arr.concat(`count(*) as views`);
     } else if (!distinct && value === 'unique') {
       distinct = true;
       //arr = arr.concat(`count(distinct ${name})`);

queries/analytics/sessions/getSessionMetrics.ts

@@ -1,7 +1,7 @@
 import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
-import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants';
+import { EVENT_TYPE } from 'lib/constants';
 import { loadWebsite } from 'lib/load';
 import { maxDate } from 'lib/date';
 
@@ -28,14 +28,9 @@ async function relationalQuery(
 
   return rawQuery(
     `select ${column} x, count(*) y
-    from session as s
-    where s.session_id in (
-      select website_event.session_id
       from website_event
-        join website 
-          on website_event.website_id = website.website_id
-        ${joinSession}
-      where website.website_id = {{websiteId::uuid}}
+      ${joinSession}
+      where website_event.website_id = {{websiteId::uuid}}
         and website_event.created_at between {{startDate}} and {{endDate}}
       ${filterQuery}
     ) as t

queries/analytics/sessions/getSessionStats.ts

@@ -59,6 +59,7 @@ async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }
@@ -93,6 +94,7 @@ async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }

queries/analytics/stats/getWebsiteStats.ts

@@ -51,11 +51,12 @@ async function relationalQuery(
     ) as t
     `,
     {
+      ...filters,
       websiteId,
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
-      ...filters,
+      domain: website.domain,
     },
   );
 }
@@ -97,6 +98,7 @@ async function clickhouseQuery(
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }