Merge branch 'dev' into analytics

src/app/(main)/websites/[websiteId]/sessions/SessionsPage.tsx

@@ -8,6 +8,7 @@ import { GridRow } from 'components/layout/Grid';
 import { Item, Tabs } from 'react-basics';
 import { useState } from 'react';
 import { useMessages } from 'components/hooks';
+import SessionsWeekly from './SessionsWeekly';
 
 export function SessionsPage({ websiteId }) {
   const [tab, setTab] = useState('activity');
@@ -17,8 +18,9 @@ export function SessionsPage({ websiteId }) {
     <>
       <WebsiteHeader websiteId={websiteId} />
       <SessionsMetricsBar websiteId={websiteId} />
-      <GridRow columns="one">
+      <GridRow columns="two-one">
-        <WorldMap websiteId={websiteId} style={{ width: 800, margin: '0 auto' }} />
+        <WorldMap websiteId={websiteId} />
+        <SessionsWeekly websiteId={websiteId} />
       </GridRow>
       <Tabs selectedKey={tab} onSelect={(value: any) => setTab(value)} style={{ marginBottom: 30 }}>
         <Item key="activity">{formatMessage(labels.activity)}</Item>

src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.module.css

@@ -0,0 +1,40 @@
+.week {
+  display: flex;
+  justify-content: space-between;
+  position: relative;
+}
+
+.header {
+  text-align: center;
+  font-weight: 700;
+  margin-bottom: 10px;
+}
+
+.day {
+  display: flex;
+  flex-direction: column;
+  gap: 5px;
+  position: relative;
+}
+
+.cell {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  background-color: var(--base100);
+  width: 20px;
+  height: 20px;
+}
+
+.hour {
+  font-weight: 700;
+  color: var(--font-color300);
+  background-color: transparent;
+}
+
+.block {
+  background-color: var(--primary400);
+  width: 20px;
+  height: 20px;
+  border-radius: 3px;
+}

src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx

@@ -0,0 +1,78 @@
+import { format } from 'date-fns';
+import { useLocale, useMessages, useWebsiteSessionsWeekly } from 'components/hooks';
+import { LoadingPanel } from 'components/common/LoadingPanel';
+import { getDayOfWeekAsDate } from 'lib/date';
+import styles from './SessionsWeekly.module.css';
+import classNames from 'classnames';
+import { TooltipPopup } from 'react-basics';
+
+export function SessionsWeekly({ websiteId }: { websiteId: string }) {
+  const { data, ...props } = useWebsiteSessionsWeekly(websiteId);
+  const { dateLocale } = useLocale();
+  const { labels, formatMessage } = useMessages();
+
+  const [, max] = data
+    ? data.reduce((arr: number[], hours: number[], index: number) => {
+        const min = Math.min(...hours);
+        const max = Math.max(...hours);
+
+        if (index === 0) {
+          return [min, max];
+        }
+
+        if (min < arr[0]) {
+          arr[0] = min;
+        }
+
+        if (max > arr[1]) {
+          arr[1] = max;
+        }
+
+        return arr;
+      }, [])
+    : [];
+
+  return (
+    <LoadingPanel {...(props as any)} data={data}>
+      <div className={styles.week}>
+        <div className={styles.day}>
+          <div className={styles.header}>&nbsp;</div>
+          {Array(24)
+            .fill(null)
+            .map((_, i) => {
+              return (
+                <div key={i} className={classNames(styles.cell, styles.hour)}>
+                  {i.toString().padStart(2, '0')}
+                </div>
+              );
+            })}
+        </div>
+        {data?.map((day: number[], index: number) => {
+          return (
+            <div className={styles.day} key={index}>
+              <div className={styles.header}>
+                {format(getDayOfWeekAsDate(index), 'EEE', { locale: dateLocale })}
+              </div>
+              {day?.map((hour: number) => {
+                return (
+                  <div key={hour} className={classNames(styles.cell)}>
+                    {hour > 0 && (
+                      <TooltipPopup
+                        label={`${formatMessage(labels.visitors)}: ${hour}`}
+                        position="right"
+                      >
+                        <div className={styles.block} style={{ opacity: hour / max }} />
+                      </TooltipPopup>
+                    )}
+                  </div>
+                );
+              })}
+            </div>
+          );
+        })}
+      </div>
+    </LoadingPanel>
+  );
+}
+
+export default SessionsWeekly;

src/app/(main)/websites/[websiteId]/sessions/[sessionId]/SessionActivity.tsx

@@ -12,8 +12,8 @@ export function SessionActivity({
 }: {
   websiteId: string;
   sessionId: string;
-  startDate: string;
+  startDate: Date;
-  endDate: string;
+  endDate: Date;
 }) {
   const { formatDate } = useTimezone();
   const { data, isLoading } = useSessionActivity(websiteId, sessionId, startDate, endDate);

src/components/charts/BubbleChart.tsx

@@ -0,0 +1,27 @@
+import { Chart, ChartProps } from 'components/charts/Chart';
+import { useState } from 'react';
+import { StatusLight } from 'react-basics';
+import { formatLongNumber } from 'lib/format';
+
+export interface BubbleChartProps extends ChartProps {
+  type?: 'bubble';
+}
+
+export default function BubbleChart(props: BubbleChartProps) {
+  const [tooltip, setTooltip] = useState(null);
+  const { type = 'bubble' } = props;
+
+  const handleTooltip = ({ tooltip }) => {
+    const { labelColors, dataPoints } = tooltip;
+
+    setTooltip(
+      tooltip.opacity ? (
+        <StatusLight color={labelColors?.[0]?.backgroundColor}>
+          {formatLongNumber(dataPoints?.[0]?.raw)} {dataPoints?.[0]?.label}
+        </StatusLight>
+      ) : null,
+    );
+  };
+
+  return <Chart {...props} type={type} tooltip={tooltip} onTooltip={handleTooltip} />;
+}

src/components/common/LoadingPanel.module.css

@@ -3,4 +3,14 @@
   flex-direction: column;
   position: relative;
   flex: 1;
+  height: 100%;
+}
+
+.loading {
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  margin: auto;
 }

src/components/common/LoadingPanel.tsx

@@ -1,9 +1,9 @@
 import { ReactNode } from 'react';
-import styles from './LoadingPanel.module.css';
 import classNames from 'classnames';
-import ErrorMessage from 'components/common/ErrorMessage';
 import { Loading } from 'react-basics';
+import ErrorMessage from 'components/common/ErrorMessage';
 import Empty from 'components/common/Empty';
+import styles from './LoadingPanel.module.css';
 
 export function LoadingPanel({
   data,
@@ -27,7 +27,7 @@ export function LoadingPanel({
 
   return (
     <div className={classNames(styles.panel, className)}>
-      {isLoading && !isFetched && <Loading icon={loadingIcon} />}
+      {isLoading && !isFetched && <Loading className={styles.loading} icon={loadingIcon} />}
       {error && <ErrorMessage />}
       {!error && isEmpty && <Empty />}
       {!error && !isEmpty && data && children}

src/components/hooks/index.ts

@@ -14,6 +14,7 @@ export * from './queries/useSessionDataProperties';
 export * from './queries/useSessionDataValues';
 export * from './queries/useWebsiteSession';
 export * from './queries/useWebsiteSessions';
+export * from './queries/useWebsiteSessionsWeekly';
 export * from './queries/useShareToken';
 export * from './queries/useTeam';
 export * from './queries/useTeams';

src/components/hooks/queries/useSessionActivity.ts

@@ -3,15 +3,19 @@ import { useApi } from './useApi';
 export function useSessionActivity(
   websiteId: string,
   sessionId: string,
-  startDate: string,
+  startDate: Date,
-  endDate: string,
+  endDate: Date,
 ) {
   const { get, useQuery } = useApi();
 
   return useQuery({
-    queryKey: ['session:activity', { websiteId, sessionId }],
+    queryKey: ['session:activity', { websiteId, sessionId, startDate, endDate }],
     queryFn: () => {
-      return get(`/websites/${websiteId}/sessions/${sessionId}/activity`, { startDate, endDate });
+      return get(`/websites/${websiteId}/sessions/${sessionId}/activity`, {
+        startAt: +new Date(startDate),
+        endAt: +new Date(endDate),
+      });
     },
+    enabled: Boolean(websiteId && sessionId && startDate && endDate),
   });
 }

src/components/hooks/queries/useWebsiteSessionsWeekly.ts

@@ -0,0 +1,24 @@
+import { useApi } from './useApi';
+import useModified from '../useModified';
+import { useFilterParams } from 'components/hooks/useFilterParams';
+
+export function useWebsiteSessionsWeekly(
+  websiteId: string,
+  params?: { [key: string]: string | number },
+) {
+  const { get, useQuery } = useApi();
+  const { modified } = useModified(`sessions`);
+  const filters = useFilterParams(websiteId);
+
+  return useQuery({
+    queryKey: ['sessions', { websiteId, modified, ...params, ...filters }],
+    queryFn: () => {
+      return get(`/websites/${websiteId}/sessions/weekly`, {
+        ...params,
+        ...filters,
+      });
+    },
+  });
+}
+
+export default useWebsiteSessionsWeekly;

src/components/hooks/useFilterParams.ts

@@ -1,19 +1,18 @@
 import { useNavigation } from './useNavigation';
 import { useDateRange } from './useDateRange';
 import { useTimezone } from './useTimezone';
-import { zonedTimeToUtc } from 'date-fns-tz';
 
 export function useFilterParams(websiteId: string) {
   const { dateRange } = useDateRange(websiteId);
   const { startDate, endDate, unit } = dateRange;
-  const { timezone } = useTimezone();
+  const { timezone, toUtc } = useTimezone();
   const {
     query: { url, referrer, title, query, host, os, browser, device, country, region, city, event },
   } = useNavigation();
 
   return {
-    startAt: +zonedTimeToUtc(startDate, timezone),
+    startAt: +toUtc(startDate),
-    endAt: +zonedTimeToUtc(endDate, timezone),
+    endAt: +toUtc(endDate),
     unit,
     timezone,
     url,

src/components/hooks/useTimezone.ts

@@ -1,6 +1,6 @@
 import { setItem } from 'next-basics';
 import { TIMEZONE_CONFIG } from 'lib/constants';
-import { formatInTimeZone } from 'date-fns-tz';
+import { formatInTimeZone, zonedTimeToUtc, utcToZonedTime } from 'date-fns-tz';
 import useStore, { setTimezone } from 'store/app';
 
 const selector = (state: { timezone: string }) => state.timezone;
@@ -23,7 +23,15 @@ export function useTimezone() {
     );
   };
 
-  return { timezone, saveTimezone, formatDate };
+  const toUtc = (date: Date | string | number) => {
+    return zonedTimeToUtc(date, timezone);
+  };
+
+  const fromUtc = (date: Date | string | number) => {
+    return utcToZonedTime(date, timezone);
+  };
+
+  return { timezone, saveTimezone, formatDate, toUtc, fromUtc };
 }
 
 export default useTimezone;

src/declaration.d.ts

@@ -1,5 +1,4 @@
 declare module 'cors';
-declare module 'dateformat';
 declare module 'debug';
 declare module 'chartjs-adapter-date-fns';
 declare module 'md5';

src/lib/clickhouse.ts

@@ -1,4 +1,5 @@
 import { ClickHouseClient, createClient } from '@clickhouse/client';
+import { formatInTimeZone } from 'date-fns-tz';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
 import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
@@ -8,6 +9,7 @@ import { filtersToArray } from './params';
 import { PageParams, QueryFilters, QueryOptions } from './types';
 
 export const CLICKHOUSE_DATE_FORMATS = {
+  utc: '%Y-%m-%dT%H:%i:%SZ',
   second: '%Y-%m-%d %H:%i:%S',
   minute: '%Y-%m-%d %H:%i:00',
   hour: '%Y-%m-%d %H:00:00',
@@ -47,7 +49,11 @@ function getClient() {
   return client;
 }
 
-function getDateStringSQL(data: any, unit: string | number, timezone?: string) {
+function getUTCString(date?: Date) {
+  return formatInTimeZone(date || new Date(), 'UTC', 'yyyy-MM-dd HH:mm:ss');
+}
+
+function getDateStringSQL(data: any, unit: string = 'utc', timezone?: string) {
   if (timezone) {
     return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}', '${timezone}')`;
   }
@@ -220,6 +226,7 @@ export default {
   getDateStringSQL,
   getDateSQL,
   getFilterQuery,
+  getUTCString,
   parseFilters,
   pagedQuery,
   findUnique,

src/lib/date.ts

@@ -34,6 +34,7 @@ import {
   addWeeks,
   subWeeks,
   endOfMinute,
+  isSameDay,
 } from 'date-fns';
 import { getDateLocale } from 'lib/lang';
 import { DateRange } from 'lib/types';
@@ -336,3 +337,16 @@ export function getCompareDate(compare: string, startDate: Date, endDate: Date)
 
   return { startDate: subMinutes(startDate, diff), endDate: subMinutes(endDate, diff) };
 }
+
+export function getDayOfWeekAsDate(dayOfWeek: number) {
+  const startOfWeekDay = startOfWeek(new Date());
+  const daysToAdd = [0, 1, 2, 3, 4, 5, 6].indexOf(dayOfWeek);
+  let currentDate = addDays(startOfWeekDay, daysToAdd);
+
+  // Ensure we're not returning a past date
+  if (isSameDay(currentDate, startOfWeekDay)) {
+    currentDate = addDays(currentDate, 7);
+  }
+
+  return currentDate;
+}

src/lib/prisma.ts

@@ -81,6 +81,18 @@ function getDateSQL(field: string, unit: string, timezone?: string): string {
   }
 }
 
+function getDateWeeklySQL(field: string) {
+  const db = getDatabaseType();
+
+  if (db === POSTGRESQL) {
+    return `EXTRACT(DOW FROM ${field})`;
+  }
+
+  if (db === MYSQL) {
+    return `DAYOFWEEK(${field})-1`;
+  }
+}
+
 export function getTimestampSQL(field: string) {
   const db = getDatabaseType();
 
@@ -284,6 +296,7 @@ export default {
   getCastColumnQuery,
   getDayDiffQuery,
   getDateSQL,
+  getDateWeeklySQL,
   getFilterQuery,
   getSearchParameters,
   getTimestampDiffSQL,

src/pages/api/websites/[websiteId]/session-data/properties.ts

@@ -6,7 +6,7 @@ import { methodNotAllowed, ok, unauthorized } from 'next-basics';
 import { getSessionDataProperties } from 'queries';
 import * as yup from 'yup';
 
-export interface EventDataFieldsRequestQuery {
+export interface SessionDataFieldsRequestQuery {
   websiteId: string;
   startAt: string;
   endAt: string;
@@ -23,7 +23,7 @@ const schema = {
 };
 
 export default async (
-  req: NextApiRequestQueryBody<EventDataFieldsRequestQuery>,
+  req: NextApiRequestQueryBody<SessionDataFieldsRequestQuery>,
   res: NextApiResponse<any>,
 ) => {
   await useCors(req, res);

src/pages/api/websites/[websiteId]/sessions/[sessionId]/activity.ts

@@ -9,16 +9,16 @@ import { getSessionActivity } from 'queries';
 export interface SessionActivityRequestQuery extends PageParams {
   websiteId: string;
   sessionId: string;
-  startDate: string;
+  startAt: number;
-  endDate: string;
+  endAt: number;
 }
 
 const schema = {
   GET: yup.object().shape({
     websiteId: yup.string().uuid().required(),
     sessionId: yup.string().uuid().required(),
-    startDate: yup.string().required(),
+    startAt: yup.number().integer(),
-    endDate: yup.string().required(),
+    endAt: yup.number().integer(),
   }),
 };
 
@@ -30,19 +30,17 @@ export default async (
   await useAuth(req, res);
   await useValidate(schema, req, res);
 
-  const { websiteId, sessionId, startDate, endDate } = req.query;
+  const { websiteId, sessionId, startAt, endAt } = req.query;
 
   if (req.method === 'GET') {
     if (!(await canViewWebsite(req.auth, websiteId))) {
       return unauthorized(res);
     }
 
-    const data = await getSessionActivity(
+    const startDate = new Date(+startAt);
-      websiteId,
+    const endDate = new Date(+endAt);
-      sessionId,
+
-      new Date(startDate + 'Z'),
+    const data = await getSessionActivity(websiteId, sessionId, startDate, endDate);
-      new Date(endDate + 'Z'),
-    );
 
     return ok(res, data);
   }

src/pages/api/websites/[websiteId]/sessions/weekly.ts

@@ -0,0 +1,47 @@
+import * as yup from 'yup';
+import { canViewWebsite } from 'lib/auth';
+import { useAuth, useCors, useValidate } from 'lib/middleware';
+import { NextApiRequestQueryBody, PageParams } from 'lib/types';
+import { NextApiResponse } from 'next';
+import { methodNotAllowed, ok, unauthorized } from 'next-basics';
+import { pageInfo } from 'lib/schema';
+import { getWebsiteSessionsWeekly } from 'queries';
+
+export interface ReportsRequestQuery extends PageParams {
+  websiteId: string;
+}
+
+const schema = {
+  GET: yup.object().shape({
+    websiteId: yup.string().uuid().required(),
+    startAt: yup.number().integer().required(),
+    endAt: yup.number().integer().min(yup.ref('startAt')).required(),
+    ...pageInfo,
+  }),
+};
+
+export default async (
+  req: NextApiRequestQueryBody<ReportsRequestQuery, any>,
+  res: NextApiResponse,
+) => {
+  await useCors(req, res);
+  await useAuth(req, res);
+  await useValidate(schema, req, res);
+
+  const { websiteId, startAt, endAt } = req.query;
+
+  if (req.method === 'GET') {
+    if (!(await canViewWebsite(req.auth, websiteId))) {
+      return unauthorized(res);
+    }
+
+    const startDate = new Date(+startAt);
+    const endDate = new Date(+endAt);
+
+    const data = await getWebsiteSessionsWeekly(websiteId, { startDate, endDate });
+
+    return ok(res, data);
+  }
+
+  return methodNotAllowed(res);
+};

src/queries/analytics/events/saveEvent.ts

@@ -135,10 +135,10 @@ async function clickhouseQuery(data: {
     city,
     ...args
   } = data;
-  const { insert } = clickhouse;
+  const { insert, getUTCString } = clickhouse;
   const { sendMessage } = kafka;
   const eventId = uuid();
-  const createdAt = new Date().toISOString();
+  const createdAt = getUTCString();
 
   const message = {
     ...args,

src/queries/analytics/sessions/getSessionActivity.ts

@@ -23,6 +23,7 @@ async function relationalQuery(
       websiteId,
       createdAt: { gte: startDate, lte: endDate },
     },
+    take: 500,
   });
 }
 
@@ -37,8 +38,6 @@ async function clickhouseQuery(
   return rawQuery(
     `
     select
-      session_id as id,
-      website_id as websiteId,
       created_at as createdAt,
       url_path as urlPath,
       url_query as urlQuery,
@@ -52,6 +51,7 @@ async function clickhouseQuery(
       and session_id = {sessionId:UUID} 
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
     order by created_at desc
+    limit 500
     `,
     { websiteId, sessionId, startDate, endDate },
   );

src/queries/analytics/sessions/getWebsiteSession.ts

@@ -19,7 +19,7 @@ async function relationalQuery(websiteId: string, sessionId: string) {
 }
 
 async function clickhouseQuery(websiteId: string, sessionId: string) {
-  const { rawQuery } = clickhouse;
+  const { rawQuery, getDateStringSQL } = clickhouse;
 
   return rawQuery(
     `
@@ -34,8 +34,8 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
       country,
       subdivision1,
       city,
-      min(min_time) as firstAt,
+      ${getDateStringSQL('min(min_time)')} as firstAt,
-      max(max_time) as lastAt,
+      ${getDateStringSQL('max(max_time)')} as lastAt,
       uniq(visit_id) visits,
       sum(views) as views,
       sum(events) as events,

src/queries/analytics/sessions/getWebsiteSessions.ts

@@ -24,7 +24,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
 }
 
 async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) {
-  const { pagedQuery, parseFilters } = clickhouse;
+  const { pagedQuery, parseFilters, getDateStringSQL } = clickhouse;
   const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters);
 
   return pagedQuery(
@@ -42,8 +42,8 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
       country,
       subdivision1,
       city,
-      min(min_time) as firstAt,
+      ${getDateStringSQL('min(min_time)')} as firstAt,
-      max(max_time) as lastAt,
+      ${getDateStringSQL('max(max_time)')} as lastAt,
       uniq(visit_id) as visits,
       sumIf(views, event_type = 1) as views
     from website_event_stats_hourly

src/queries/analytics/sessions/getWebsiteSessionsWeekly.ts

@@ -0,0 +1,69 @@
+import prisma from 'lib/prisma';
+import clickhouse from 'lib/clickhouse';
+import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';
+import { QueryFilters } from 'lib/types';
+
+export async function getWebsiteSessionsWeekly(
+  ...args: [websiteId: string, filters?: QueryFilters]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(websiteId: string, filters: QueryFilters) {
+  const { rawQuery, getDateWeeklySQL, parseFilters } = prisma;
+  const { params } = await parseFilters(websiteId, filters);
+
+  return rawQuery(
+    `
+    select
+      ${getDateWeeklySQL('created_at')} as time,
+      count(distinct session_id) as value
+    from website_event_stats_hourly
+    where website_id = {{websiteId::uuid}}
+      and created_at between {{startDate}} and {{endDate}}
+    group by time
+    order by 2
+    `,
+    params,
+  ).then(formatResults);
+}
+
+async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+  const { rawQuery } = clickhouse;
+  const { startDate, endDate } = filters;
+
+  return rawQuery(
+    `
+    select
+      formatDateTime(created_at, '%w:%H') as time,
+      count(distinct session_id) as value
+    from website_event_stats_hourly
+    where website_id = {websiteId:UUID}
+      and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    group by time
+    order by time
+    `,
+    { websiteId, startDate, endDate },
+  ).then(formatResults);
+}
+
+function formatResults(data: any) {
+  const days = [];
+
+  for (let i = 0; i < 7; i++) {
+    days.push([]);
+
+    for (let j = 0; j < 24; j++) {
+      days[i].push(
+        Number(
+          data.find(({ time }) => time === `${i}:${j.toString().padStart(2, '0')}`)?.value || 0,
+        ),
+      );
+    }
+  }
+
+  return days;
+}

src/queries/analytics/sessions/saveSessionData.ts

@@ -80,9 +80,9 @@ async function clickhouseQuery(data: {
 }) {
   const { websiteId, sessionId, sessionData } = data;
 
-  const { insert } = clickhouse;
+  const { insert, getUTCString } = clickhouse;
   const { sendMessages } = kafka;
-  const createdAt = new Date().toISOString();
+  const createdAt = getUTCString();
 
   const jsonKeys = flattenJSON(sessionData);
 

src/queries/index.ts

@@ -26,6 +26,7 @@ export * from './analytics/sessions/getSessionDataProperties';
 export * from './analytics/sessions/getSessionDataValues';
 export * from './analytics/sessions/getSessionMetrics';
 export * from './analytics/sessions/getWebsiteSessions';
+export * from './analytics/sessions/getWebsiteSessionsWeekly';
 export * from './analytics/sessions/getSessionActivity';
 export * from './analytics/sessions/getSessionStats';
 export * from './analytics/sessions/saveSessionData';