Fixed realtime chart display.

components/metrics/RealtimeChart.js

@@ -1,5 +1,5 @@
 import { useMemo, useRef } from 'react';
-import { format, parseISO, startOfMinute, subMinutes, isBefore } from 'date-fns';
+import { format, startOfMinute, subMinutes, isBefore } from 'date-fns';
 import PageviewsChart from './PageviewsChart';
 import { getDateArray } from 'lib/date';
 import { DEFAULT_ANIMATION_DURATION, REALTIME_RANGE } from 'lib/constants';
@@ -8,13 +8,12 @@ function mapData(data) {
   let last = 0;
   const arr = [];
 
-  data.reduce((obj, val) => {
-    const { createdAt } = val;
-    const t = startOfMinute(parseISO(createdAt));
+  data.reduce((obj, { timestamp }) => {
+    const t = startOfMinute(new Date(timestamp));
     if (t.getTime() > last) {
       obj = { t: format(t, 'yyyy-LL-dd HH:mm:00'), y: 1 };
       arr.push(obj);
-      last = t;
+      last = t.getTime();
     } else {
       obj.y += 1;
     }
@@ -30,14 +29,15 @@ export default function RealtimeChart({ data, unit, ...props }) {
   const prevEndDate = useRef(endDate);
 
   const chartData = useMemo(() => {
-    if (data) {
-      return {
-        pageviews: getDateArray(mapData(data.pageviews), startDate, endDate, unit),
-        sessions: getDateArray(mapData(data.sessions), startDate, endDate, unit),
-      };
+    if (!data) {
+      return { pageviews: [], sessions: [] };
     }
-    return { pageviews: [], sessions: [] };
-  }, [data]);
+
+    return {
+      pageviews: getDateArray(mapData(data.pageviews), startDate, endDate, unit),
+      sessions: getDateArray(mapData(data.sessions), startDate, endDate, unit),
+    };
+  }, [data, startDate, endDate, unit]);
 
   // Don't animate the bars shifting over because it looks weird
   const animationDuration = useMemo(() => {
@@ -46,7 +46,7 @@ export default function RealtimeChart({ data, unit, ...props }) {
       return 0;
     }
     return DEFAULT_ANIMATION_DURATION;
-  }, [data]);
+  }, [data, endDate]);
 
   return (
     <PageviewsChart

components/pages/realtime/RealtimeCountries.js

@@ -0,0 +1,26 @@
+import { useCallback } from 'react';
+import { useIntl } from 'react-intl';
+import { labels } from 'components/messages';
+import DataTable from 'components/metrics/DataTable';
+import useLocale from 'hooks/useLocale';
+import useCountryNames from 'hooks/useCountryNames';
+
+export default function RealtimeCountries({ data }) {
+  const { formatMessage } = useIntl();
+  const { locale } = useLocale();
+  const countryNames = useCountryNames(locale);
+
+  const renderCountryName = useCallback(
+    ({ x }) => <span className={locale}>{countryNames[x]}</span>,
+    [countryNames, locale],
+  );
+
+  return (
+    <DataTable
+      title={formatMessage(labels.countries)}
+      metric={formatMessage(labels.visitors)}
+      data={data}
+      renderLabel={renderCountryName}
+    />
+  );
+}

components/pages/realtime/RealtimeDashboard.js

@@ -1,66 +1,57 @@
-import { useState, useEffect, useMemo, useCallback } from 'react';
+import { useState, useEffect, useMemo } from 'react';
 import { useIntl } from 'react-intl';
-import { subMinutes, startOfMinute, differenceInMinutes } from 'date-fns';
+import { subMinutes, startOfMinute } from 'date-fns';
+import { useRouter } from 'next/router';
 import firstBy from 'thenby';
 import { GridRow, GridColumn } from 'components/layout/Grid';
 import Page from 'components/layout/Page';
 import RealtimeChart from 'components/metrics/RealtimeChart';
-import RealtimeLog from 'components/pages/realtime/RealtimeLog';
-import RealtimeHeader from 'components/pages/realtime/RealtimeHeader';
 import StickyHeader from 'components/helpers/StickyHeader';
 import PageHeader from 'components/layout/PageHeader';
 import WorldMap from 'components/common/WorldMap';
-import DataTable from 'components/metrics/DataTable';
+import RealtimeLog from 'components/pages/realtime/RealtimeLog';
+import RealtimeHeader from 'components/pages/realtime/RealtimeHeader';
 import RealtimeUrls from 'components/pages/realtime/RealtimeUrls';
+import RealtimeCountries from 'components/pages/realtime/RealtimeCountries';
+import WebsiteSelect from 'components/input/WebsiteSelect';
 import useApi from 'hooks/useApi';
-import useLocale from 'hooks/useLocale';
-import useCountryNames from 'hooks/useCountryNames';
 import { percentFilter } from 'lib/filters';
 import { labels } from 'components/messages';
 import { REALTIME_RANGE, REALTIME_INTERVAL } from 'lib/constants';
 import styles from './RealtimeDashboard.module.css';
-import WebsiteSelect from '../../input/WebsiteSelect';
-import { useRouter } from 'next/router';
 
-function mergeData(state = [], data, time) {
+function mergeData(state = [], data = [], time) {
   const ids = state.map(({ __id }) => __id);
   return state
     .concat(data.filter(({ __id }) => !ids.includes(__id)))
-    .filter(({ createdAt }) => new Date(createdAt).getTime() >= time);
+    .filter(({ timestamp }) => timestamp >= time);
 }
 
 export default function RealtimeDashboard({ websiteId }) {
   const { formatMessage } = useIntl();
-  const { locale } = useLocale();
   const router = useRouter();
-  const countryNames = useCountryNames(locale);
   const [currentData, setCurrentData] = useState();
   const { get, useQuery } = useApi();
   const { data, isLoading, error } = useQuery(
     ['realtime', websiteId],
-    () => get(`/realtime/${websiteId}`, { startAt: currentData?.timestamp }),
+    () => get(`/realtime/${websiteId}`, { startAt: currentData?.timestamp || 0 }),
     {
       enabled: !!websiteId,
-      retryInterval: REALTIME_INTERVAL,
+      refetchInterval: REALTIME_INTERVAL,
+      cache: false,
     },
   );
 
-  const renderCountryName = useCallback(
-    ({ x }) => <span className={locale}>{countryNames[x]}</span>,
-    [countryNames],
-  );
-
   useEffect(() => {
     if (data) {
-      const { pageviews, sessions, events, timestamp } = data;
-      const time = subMinutes(startOfMinute(new Date()), REALTIME_RANGE).getTime();
+      const date = subMinutes(startOfMinute(new Date()), REALTIME_RANGE);
+      const time = date.getTime();
 
       setCurrentData(state => ({
-        ...state,
-        pageviews: mergeData(state?.pageviews, pageviews, time),
-        sessions: mergeData(state?.sessions, sessions, time),
-        events: mergeData(state?.events, events, time),
-        timestamp,
+        pageviews: mergeData(state?.pageviews, data.pageviews, time),
+        sessions: mergeData(state?.sessions, data.sessions, time),
+        events: mergeData(state?.events, data.events, time),
+        timestamp: data.timestamp,
       }));
     }
   }, [data]);
@@ -72,6 +63,12 @@ export default function RealtimeDashboard({ websiteId }) {
 
     currentData.countries = percentFilter(
       currentData.sessions
+        .reduce((arr, data) => {
+          if (!arr.find(({ sessionId }) => sessionId === data.sessionId)) {
+            return arr.concat(data);
+          }
+          return arr;
+        }, [])
         .reduce((arr, { country }) => {
           if (country) {
             const row = arr.find(({ x }) => x === country);
@@ -115,12 +112,7 @@ export default function RealtimeDashboard({ websiteId }) {
       </GridRow>
       <GridRow>
         <GridColumn xs={12} lg={4}>
-          <DataTable
-            title={formatMessage(labels.countries)}
-            metric={formatMessage(labels.visitors)}
-            data={realtimeData?.countries}
-            renderLabel={renderCountryName}
-          />
+          <RealtimeCountries data={realtimeData?.countries} />
         </GridColumn>
         <GridColumn xs={12} lg={8}>
           <WorldMap data={realtimeData?.countries} />

components/pages/realtime/RealtimeHeader.js

@@ -7,13 +7,20 @@ export default function RealtimeHeader({ data = {} }) {
   const { formatMessage } = useIntl();
   const { pageviews, sessions, events, countries } = data;
 
+  const visitors = sessions?.reduce((arr, { sessionId }) => {
+    if (sessionId && !arr.includes(sessionId)) {
+      return arr.concat(sessionId);
+    }
+    return arr;
+  }, []);
+
   return (
     <div className={styles.header}>
       <div className={styles.metrics}>
         <MetricCard label={formatMessage(labels.views)} value={pageviews?.length} hideComparison />
         <MetricCard
           label={formatMessage(labels.visitors)}
-          value={sessions?.length}
+          value={visitors?.length}
           hideComparison
         />
         <MetricCard label={formatMessage(labels.events)} value={events?.length} hideComparison />

lib/constants.ts

@@ -21,7 +21,7 @@ export const DEFAULT_DATE_RANGE = '24hour';
 export const DEFAULT_WEBSITE_LIMIT = 10;
 
 export const REALTIME_RANGE = 30;
-export const REALTIME_INTERVAL = 3000;
+export const REALTIME_INTERVAL = 5000;
 
 export const UI_LAYOUT_BODY = 'ui-layout-body';
 

lib/crypto.js

@@ -1,3 +1,4 @@
+import crypto from 'crypto';
 import { v4, v5 } from 'uuid';
 import { startOfMonth } from 'date-fns';
 import { hash } from 'next-basics';
@@ -17,3 +18,7 @@ export function uuid(...args) {
 
   return v5(hash(...args, salt()), v5.DNS);
 }
+
+export function md5(...args) {
+  return crypto.createHash('md5').update(args.join('')).digest('hex');
+}

pages/_app.js

@@ -14,6 +14,7 @@ import '@fontsource/inter/600.css';
 const client = new QueryClient({
   defaultOptions: {
     queries: {
+      retry: false,
       refetchOnWindowFocus: false,
     },
   },

pages/api/realtime/[id].ts

@@ -9,9 +9,14 @@ export default async (req: NextApiRequestAuth, res: NextApiResponse<RealtimeInit
   await useAuth(req, res);
 
   if (req.method === 'GET') {
-    const { id } = req.query;
+    const { id, startAt } = req.query;
+    let startTime = subMinutes(new Date(), 30);
 
-    const data = await getRealtimeData(id, subMinutes(new Date(), 30));
+    if (+startAt > startTime.getTime()) {
+      startTime = new Date(+startAt);
+    }
+
+    const data = await getRealtimeData(id, startTime);
 
     return ok(res, data);
   }

pages/realtime/[id]/index.js

@@ -12,7 +12,7 @@ export default function RealtimeDetailsPage() {
 
   return (
     <AppLayout>
-      <RealtimeDashboard websiteId={websiteId} />
+      <RealtimeDashboard key={websiteId} websiteId={websiteId} />
     </AppLayout>
   );
 }

queries/analytics/event/getEvents.ts

@@ -26,12 +26,13 @@ function clickhouseQuery(websiteId: string, startAt: Date) {
 
   return rawQuery(
     `select
-      event_id,
-      website_id, 
-      session_id,
-      created_at,
+      event_id as id,
+      website_id as websiteId, 
+      session_id as sessionId,
+      created_at as createdAt,
+      toUnixTimestamp(created_at) as timestamp,
       url,
-      event_name
+      event_name as eventName
     from event
     where event_type = ${EVENT_TYPE.customEvent}
       and website_id = {websiteId:UUID}

queries/analytics/pageview/getPageviews.ts

@@ -29,6 +29,7 @@ async function clickhouseQuery(websiteId: string, startAt: Date) {
         website_id as websiteId,
         session_id as sessionId,
         created_at as createdAt,
+        toUnixTimestamp(created_at) as timestamp,
         url
       from event
       where event_type = ${EVENT_TYPE.pageView}

queries/analytics/session/getSessions.ts

@@ -25,9 +25,10 @@ async function clickhouseQuery(websiteId: string, startAt: Date) {
 
   return rawQuery(
     `select distinct
-      session_id,
-      website_id,
-      created_at,
+      session_id as sessionId,
+      website_id as websiteId,
+      created_at as createdAt,
+      toUnixTimestamp(created_at) as timestamp,
       hostname,
       browser,
       os,

queries/analytics/stats/getRealtimeData.ts

@@ -1,3 +1,4 @@
+import { md5 } from 'lib/crypto';
 import { getPageviews } from '../pageview/getPageviews';
 import { getSessions } from '../session/getSessions';
 import { getEvents } from '../event/getEvents';
@@ -9,22 +10,19 @@ export async function getRealtimeData(websiteId, time) {
     getEvents(websiteId, time),
   ]);
 
+  const decorate = (id, data) => {
+    return data.map(props => ({
+      ...props,
+      __id: md5(id, ...Object.values(props)),
+      timestamp: props.timestamp * 1000,
+      timestampCompare: new Date(props.createdAt).getTime(),
+    }));
+  };
+
   return {
-    pageviews: pageviews.map(({ id, ...props }) => ({
-      __id: `p${id}`,
-      pageviewId: id,
-      ...props,
-    })),
-    sessions: sessions.map(({ id, ...props }) => ({
-      __id: `s${id}`,
-      sessionId: id,
-      ...props,
-    })),
-    events: events.map(({ id, ...props }) => ({
-      __id: `e${id}`,
-      eventId: id,
-      ...props,
-    })),
+    pageviews: decorate('pageviews', pageviews),
+    sessions: decorate('sessions', sessions),
+    events: decorate('events', events),
     timestamp: Date.now(),
   };
 }

scripts/download-country-names.js

@@ -1,4 +1,4 @@
-/* eslint-disable no-console */
+/* eslint-disable no-console, @typescript-eslint/no-var-requires */
 const fs = require('fs-extra');
 const path = require('path');
 const https = require('https');