Merge branch 'dev' of https://github.com/umami-software/umami into feat/um-366-event-data-migration

components/metrics/MetricsBar.js

@@ -1,13 +1,27 @@
-import { Loading } from 'react-basics';
+import { useState } from 'react';
+import { Loading, cloneChildren } from 'react-basics';
 import ErrorMessage from 'components/common/ErrorMessage';
 import styles from './MetricsBar.module.css';
+import { formatLongNumber, formatNumber } from 'lib/format';
+
+export function MetricsBar({ children, isLoading, isFetched, error }) {
+  const [format, setFormat] = useState(true);
+
+  const formatFunc = format
+    ? n => (n >= 0 ? formatLongNumber(n) : `-${formatLongNumber(Math.abs(n))}`)
+    : formatNumber;
+
+  const handleSetFormat = () => {
+    setFormat(state => !state);
+  };
 
-export function MetricsBar({ children, onClick, isLoading, isFetched, error }) {
   return (
-    <div className={styles.bar} onClick={onClick}>
+    <div className={styles.bar} onClick={handleSetFormat}>
       {isLoading && !isFetched && <Loading icon="dots" />}
       {error && <ErrorMessage />}
-      {children}
+      {cloneChildren(children, child => {
+        return { format: child.props.format || formatFunc };
+      })}
     </div>
   );
 }

components/pages/event-data/EventDataMetricsBar.js

@@ -13,9 +13,9 @@ export function EventDataMetricsBar({ websiteId }) {
   const { startDate, endDate, modified } = dateRange;
 
   const { data, error, isLoading, isFetched } = useQuery(
-    ['event-data:fields', { websiteId, startDate, endDate, modified }],
+    ['event-data:stats', { websiteId, startDate, endDate, modified }],
     () =>
-      get(`/event-data/fields`, {
+      get(`/event-data/stats`, {
         websiteId,
         startAt: +startDate,
         endAt: +endDate,
@@ -27,11 +27,18 @@ export function EventDataMetricsBar({ websiteId }) {
       <Column defaultSize={12} xl={8}>
         <MetricsBar isLoading={isLoading} isFetched={isFetched} error={error}>
           {!error && isFetched && (
-            <MetricCard
-              className={styles.card}
-              label={formatMessage(labels.fields)}
-              value={data?.length}
-            />
+            <>
+              <MetricCard
+                className={styles.card}
+                label={formatMessage(labels.fields)}
+                value={data?.fields}
+              />
+              <MetricCard
+                className={styles.card}
+                label={formatMessage(labels.totalRecords)}
+                value={data?.records}
+              />
+            </>
           )}
         </MetricsBar>
       </Column>

components/pages/event-data/EventDataTable.js

@@ -3,7 +3,7 @@ import { GridTable, GridColumn } from 'react-basics';
 import { useMessages, usePageQuery } from 'hooks';
 import Empty from 'components/common/Empty';
 
-export function EventDataTable({ data = [], showValue }) {
+export function EventDataTable({ data = [] }) {
   const { formatMessage, labels } = useMessages();
   const { resolveUrl } = usePageQuery();
 

components/pages/websites/WebsiteEventData.js

@@ -3,6 +3,7 @@ import EventDataTable from 'components/pages/event-data/EventDataTable';
 import EventDataValueTable from 'components/pages/event-data/EventDataValueTable';
 import { EventDataMetricsBar } from 'components/pages/event-data/EventDataMetricsBar';
 import { useDateRange, useApi, usePageQuery } from 'hooks';
+import styles from './WebsiteEventData.module.css';
 
 function useFields(websiteId, field) {
   const [dateRange] = useDateRange(websiteId);
@@ -11,7 +12,7 @@ function useFields(websiteId, field) {
   const { data, error, isLoading } = useQuery(
     ['event-data:fields', { websiteId, startDate, endDate, field }],
     () =>
-      get('/event-data', {
+      get('/event-data/fields', {
         websiteId,
         startAt: +startDate,
         endAt: +endDate,
@@ -30,7 +31,7 @@ export default function WebsiteEventData({ websiteId }) {
   const { data } = useFields(websiteId, view);
 
   return (
-    <Flexbox direction="column" gap={20}>
+    <Flexbox className={styles.container} direction="column" gap={20}>
       <EventDataMetricsBar websiteId={websiteId} />
       {!view && <EventDataTable data={data} />}
       {view && <EventDataValueTable field={view} data={data} />}

components/pages/websites/WebsiteEventData.module.css

@@ -1,4 +1,7 @@
-.container {
-  display: flex;
-  flex-direction: column;
+.container a {
+  color: var(--font-color100);
+}
+
+.container a:hover {
+  color: var(--primary400);
 }

components/pages/websites/WebsiteHeader.js

@@ -48,10 +48,7 @@ export function WebsiteHeader({ websiteId, showLinks = true, children }) {
         {showLinks && (
           <Flexbox alignItems="center">
             {links.map(({ label, icon, path }) => {
-              const query = path.indexOf('?');
-              const selected = path
-                ? asPath.endsWith(query >= 0 ? path.substring(0, query) : path)
-                : pathname === '/websites/[id]';
+              const selected = path ? pathname.endsWith(path) : pathname === '/websites/[id]';
 
               return (
                 <Link key={label} href={`/websites/${websiteId}${path}`} shallow={true}>

components/pages/websites/WebsiteMetricsBar.js

@@ -14,7 +14,6 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
   const { get, useQuery } = useApi();
   const [dateRange] = useDateRange(websiteId);
   const { startDate, endDate, modified } = dateRange;
-  const [format, setFormat] = useState(true);
   const { ref, isSticky } = useSticky({ enabled: sticky });
   const {
     query: { url, referrer, title, os, browser, device, country, region, city },
@@ -41,14 +40,6 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
       }),
   );
 
-  const formatFunc = format
-    ? n => (n >= 0 ? formatLongNumber(n) : `-${formatLongNumber(Math.abs(n))}`)
-    : formatNumber;
-
-  function handleSetFormat() {
-    setFormat(state => !state);
-  }
-
   const { pageviews, uniques, bounces, totaltime } = data || {};
   const num = Math.min(data && uniques.value, data && bounces.value);
   const diffs = data && {
@@ -67,12 +58,7 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
       })}
     >
       <Column defaultSize={12} xl={8}>
-        <MetricsBar
-          isLoading={isLoading}
-          isFetched={isFetched}
-          error={error}
-          onClick={handleSetFormat}
-        >
+        <MetricsBar isLoading={isLoading} isFetched={isFetched} error={error}>
           {!error && isFetched && (
             <>
               <MetricCard
@@ -80,14 +66,12 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
                 label={formatMessage(labels.views)}
                 value={pageviews.value}
                 change={pageviews.change}
-                format={formatFunc}
               />
               <MetricCard
                 className={styles.card}
                 label={formatMessage(labels.visitors)}
                 value={uniques.value}
                 change={uniques.change}
-                format={formatFunc}
               />
               <MetricCard
                 className={styles.card}

db/clickhouse/migrations/01_edit_keys.sql

@@ -11,10 +11,8 @@ ALTER TABLE "event_data" ADD COLUMN "job_id" UUID AFTER "created_at";
 -- update event_data string
 alter table umami.event_data
 update string_value = number_value
-where number_value is not null
-    and string_value is null;
+where data_type = 2
 
 alter table umami.event_data
 update string_value = replaceOne(concat(CAST(toDateTime(date_value, 'UTC'), 'String'),'Z'), ' ', 'T')
-where date_value is not null
-    and string_value is null;
+where data_type = 4

lib/types.ts

@@ -94,11 +94,17 @@ export interface WebsiteEventMetric {
   y: number;
 }
 
-export interface WebsiteEventDataMetric {
-  x: string;
-  t: string;
-  eventName?: string;
-  urlPath?: string;
+export interface WebsiteEventDataStats {
+  field: string;
+  type: number;
+  total: number;
+}
+
+export interface WebsiteEventDataFields {
+  field: string;
+  type: number;
+  value?: string;
+  total: number;
 }
 
 export interface WebsitePageviews {

package.json

@@ -79,7 +79,6 @@
     "del": "^6.0.0",
     "detect-browser": "^5.2.0",
     "dotenv": "^10.0.0",
-    "formik": "^2.2.9",
     "fs-extra": "^10.0.1",
     "immer": "^9.0.12",
     "ipaddr.js": "^2.0.1",

pages/api/event-data/fields.ts

@@ -21,13 +21,13 @@ export default async (
   await useAuth(req, res);
 
   if (req.method === 'GET') {
-    const { websiteId, startAt, endAt } = req.query;
+    const { websiteId, startAt, endAt, field } = req.query;
 
     if (!(await canViewWebsite(req.auth, websiteId))) {
       return unauthorized(res);
     }
 
-    const data = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt));
+    const data = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt), field);
 
     return ok(res, data);
   }

pages/api/event-data/stats.ts

@@ -3,7 +3,7 @@ import { useCors, useAuth } from 'lib/middleware';
 import { NextApiRequestQueryBody } from 'lib/types';
 import { NextApiResponse } from 'next';
 import { ok, methodNotAllowed, unauthorized } from 'next-basics';
-import { getEventData } from 'queries';
+import { getEventDataFields } from 'queries';
 
 export interface EventDataRequestBody {
   websiteId: string;
@@ -22,13 +22,21 @@ export default async (
   await useAuth(req, res);
 
   if (req.method === 'GET') {
-    const { websiteId, startAt, endAt, field } = req.query;
+    const { websiteId, startAt, endAt } = req.query;
 
     if (!(await canViewWebsite(req.auth, websiteId))) {
       return unauthorized(res);
     }
 
-    const data = await getEventData(websiteId, new Date(+startAt), new Date(+endAt), field);
+    const results = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt));
+
+    const data = results.reduce(
+      (obj, row) => {
+        obj.records += row.total;
+        return obj;
+      },
+      { fields: results.length, records: 0 },
+    );
 
     return ok(res, data);
   }

queries/analytics/eventData/getEventData.ts

@@ -1,94 +0,0 @@
-import prisma from 'lib/prisma';
-import clickhouse from 'lib/clickhouse';
-import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
-import { WebsiteEventDataMetric } from 'lib/types';
-import { loadWebsite } from 'lib/query';
-import { DEFAULT_CREATED_AT } from 'lib/constants';
-
-export async function getEventData(
-  ...args: [websiteId: string, startDate: Date, endDate: Date, field?: string]
-): Promise<WebsiteEventDataMetric[]> {
-  return runQuery({
-    [PRISMA]: () => relationalQuery(...args),
-    [CLICKHOUSE]: () => clickhouseQuery(...args),
-  });
-}
-
-async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
-  const { toUuid, rawQuery } = prisma;
-  const website = await loadWebsite(websiteId);
-  const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
-
-  if (field) {
-    return rawQuery(
-      `select event_key as field,
-              string_value as value,
-              count(*)  as total
-       from event_data
-       where website_id = $1${toUuid()}
-          and event_key = $2
-         and created_at >= $3
-         and created_at between $4 and $5
-       group by event_key, string_value
-       order by 3 desc, 2 desc, 1 asc
-           limit 100
-      `,
-      [websiteId, field, resetDate, startDate, endDate] as any,
-    );
-  }
-
-  return rawQuery(
-    `select
-        event_key as field,
-        count(*) as total
-        from event_data
-        where website_id = $1${toUuid()}
-          and created_at >= $2
-          and created_at between $3 and $4
-        group by event_key
-        order by 2 desc, 1 asc
-        limit 100
-    `,
-    [websiteId, resetDate, startDate, endDate] as any,
-  );
-}
-
-async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
-  const { rawQuery, getDateFormat, getBetweenDates } = clickhouse;
-  const website = await loadWebsite(websiteId);
-  const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
-
-  if (field) {
-    return rawQuery(
-      `select
-        event_key as field,
-        string_value as value,
-        count(*) as total
-        from event_data
-        where website_id = {websiteId:UUID}
-          and event_key = {field:String}
-          and created_at >= ${getDateFormat(resetDate)}
-          and ${getBetweenDates('created_at', startDate, endDate)}
-        group by event_key, string_value
-        order by 3 desc, 2 desc, 1 asc
-        limit 100
-    `,
-      { websiteId, field },
-    );
-  }
-
-  return rawQuery(
-    `select
-        event_key as field,
-        count(*) as total
-        from event_data
-        where website_id = {websiteId:UUID}
-          and created_at >= ${getDateFormat(resetDate)}
-          and ${getBetweenDates('created_at', startDate, endDate)}
-        group by event_key
-        order by 2 desc, 1 asc
-        limit 100
-    `,
-    { websiteId },
-  );
-}

queries/analytics/eventData/getEventDataFields.ts

@@ -1,51 +1,96 @@
+import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
-import prisma from 'lib/prisma';
-import { WebsiteEventDataMetric } from 'lib/types';
+import { WebsiteEventDataFields } from 'lib/types';
 import { loadWebsite } from 'lib/query';
 import { DEFAULT_CREATED_AT } from 'lib/constants';
 
 export async function getEventDataFields(
-  ...args: [websiteId: string, startDate: Date, endDate: Date]
-): Promise<WebsiteEventDataMetric[]> {
+  ...args: [websiteId: string, startDate: Date, endDate: Date, field?: string]
+): Promise<WebsiteEventDataFields[]> {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
     [CLICKHOUSE]: () => clickhouseQuery(...args),
   });
 }
 
-async function relationalQuery(websiteId: string, startDate: Date, endDate: Date) {
+async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
   const { toUuid, rawQuery } = prisma;
   const website = await loadWebsite(websiteId);
   const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
-  const params: any = [websiteId, resetDate, startDate, endDate];
+
+  if (field) {
+    return rawQuery(
+      `select event_key as field,
+              string_value as value,
+              count(*) as total
+       from event_data
+       where website_id = $1${toUuid()}
+         and event_key = $2
+         and created_at >= $3
+         and created_at between $4 and $5
+       group by event_key, string_value
+       order by 3 desc, 2 desc, 1 asc
+           limit 100
+      `,
+      [websiteId, field, resetDate, startDate, endDate] as any,
+    );
+  }
 
   return rawQuery(
     `select
-        distinct event_key as eventKey, data_type as eventDataType
-    from event_data
-    where website_id = $1${toUuid()}
-      and created_at >= $2
-      and created_at between $3 and $4
-    order by event_key asc`,
-    params,
+        event_key as field,
+        data_type as type,
+        count(*) as total
+        from event_data
+        where website_id = $1${toUuid()}
+          and created_at >= $2
+          and created_at between $3 and $4
+        group by event_key, data_type
+        order by 3 desc, 2 asc, 1 asc
+        limit 100
+    `,
+    [websiteId, resetDate, startDate, endDate] as any,
   );
 }
 
-async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date) {
+async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
   const { rawQuery, getDateFormat, getBetweenDates } = clickhouse;
   const website = await loadWebsite(websiteId);
   const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
-  const params = { websiteId };
+
+  if (field) {
+    return rawQuery(
+      `select
+        event_key as field,
+        string_value as value,
+        count(*) as total
+        from event_data
+        where website_id = {websiteId:UUID}
+          and event_key = {field:String}
+          and created_at >= ${getDateFormat(resetDate)}
+          and ${getBetweenDates('created_at', startDate, endDate)}
+        group by event_key, string_value
+        order by 3 desc, 2 desc, 1 asc
+        limit 100
+    `,
+      { websiteId, field },
+    );
+  }
 
   return rawQuery(
     `select
-        distinct event_key as eventKey, data_type as eventDataType
-    from event_data
-    where website_id = {websiteId:UUID}
-      and created_at >= ${getDateFormat(resetDate)}
-      and ${getBetweenDates('created_at', startDate, endDate)}
-    order by event_key asc`,
-    params,
+        event_key as field,
+        data_type as type,
+        count(*) as total
+        from event_data
+        where website_id = {websiteId:UUID}
+          and created_at >= ${getDateFormat(resetDate)}
+          and ${getBetweenDates('created_at', startDate, endDate)}
+        group by event_key, data_type
+        order by 3 desc, 2 asc, 1 asc
+        limit 100
+    `,
+    { websiteId },
   );
 }

queries/analytics/pageview/getPageviewFunnel.ts

@@ -91,7 +91,7 @@ async function clickhouseQuery(
         count(*) AS count
     FROM (
     SELECT session_id,
-            windowFunnel({window:UInt32})
+            windowFunnel({window:UInt32}, 'strict_order')
             (
                 created_at
                 ${columnsQuery}

queries/index.js

@@ -6,7 +6,6 @@ export * from './admin/website';
 export * from './analytics/event/getEventMetrics';
 export * from './analytics/event/getEventUsage';
 export * from './analytics/event/getEvents';
-export * from './analytics/eventData/getEventData';
 export * from './analytics/eventData/getEventDataFields';
 export * from './analytics/eventData/getEventDataUsage';
 export * from './analytics/event/saveEvent';