MySQL query optimization. Added loading component.

components/common/Loading.js

@@ -0,0 +1,13 @@
+import React from 'react';
+import classNames from 'classnames';
+import styles from './Loading.module.css';
+
+export default function Loading({ className }) {
+  return (
+    <div className={classNames(styles.loading, className)}>
+      <div />
+      <div />
+      <div />
+    </div>
+  );
+}

components/common/Loading.module.css

@@ -0,0 +1,41 @@
+@keyframes blink {
+  0% {
+    opacity: 0.2;
+  }
+  20% {
+    opacity: 1;
+  }
+  100% {
+    opacity: 0.2;
+  }
+}
+
+.loading {
+  display: flex;
+  position: absolute;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  margin: 0;
+}
+
+.loading div {
+  width: 10px;
+  height: 10px;
+  border-radius: 100%;
+  background: var(--gray400);
+  animation: blink 1.4s infinite;
+  animation-fill-mode: both;
+}
+
+.loading div + div {
+  margin-left: 10px;
+}
+
+.loading div:nth-child(2) {
+  animation-delay: 0.2s;
+}
+
+.loading div:nth-child(3) {
+  animation-delay: 0.4s;
+}

components/layout/MenuLayout.module.css

@@ -10,6 +10,7 @@
 }
 
 .container .content {
+  position: relative;
   border-left: 1px solid var(--gray300);
   padding-left: 30px;
 }

components/metrics/BarChart.js

@@ -120,6 +120,7 @@ export default function BarChart({
 
     options.scales.xAxes[0].time.unit = unit;
     options.scales.xAxes[0].ticks.callback = renderLabel;
+    options.animation.duration = animationDuration;
 
     onUpdate(chart.current);
   };
@@ -133,7 +134,7 @@ export default function BarChart({
         updateChart();
       }
     }
-  }, [datasets]);
+  }, [datasets, unit, animationDuration]);
 
   return (
     <>

components/metrics/EventsChart.js

@@ -42,7 +42,7 @@ export default function EventsChart({ websiteId, startDate, endDate, unit }) {
       unit,
       tz: getTimezone(),
     });
-    console.log({ data });
+
     const map = data.reduce((obj, { x, t, y }) => {
       if (!obj[x]) {
         obj[x] = [];

components/metrics/MetricsTable.js

@@ -8,6 +8,7 @@ import { get } from 'lib/web';
 import { percentFilter } from 'lib/filters';
 import { formatNumber, formatLongNumber } from 'lib/format';
 import styles from './MetricsTable.module.css';
+import Loading from '../common/Loading';
 
 export default function MetricsTable({
   title,
@@ -79,35 +80,37 @@ export default function MetricsTable({
     }
   }, [websiteId, startDate, endDate, type]);
 
-  if (!data) {
-    return null;
-  }
-
   return (
     <div className={classNames(styles.container, className)}>
-      <div className={styles.header}>
-        <div className={styles.title}>{title}</div>
-        {headerComponent}
-        <div className={styles.metric} onClick={handleSetFormat}>
-          {metric}
-        </div>
-      </div>
-      <div className={styles.body}>
-        {limit
-          ? rankings.map(row => getRow(row))
-          : data?.length > 0 && (
-              <FixedSizeList height={500} itemCount={rankings.length} itemSize={30}>
-                {Row}
-              </FixedSizeList>
+      {data ? (
+        <>
+          <div className={styles.header}>
+            <div className={styles.title}>{title}</div>
+            {headerComponent}
+            <div className={styles.metric} onClick={handleSetFormat}>
+              {metric}
+            </div>
+          </div>
+          <div className={styles.body}>
+            {limit
+              ? rankings.map(row => getRow(row))
+              : data?.length > 0 && (
+                  <FixedSizeList height={500} itemCount={rankings.length} itemSize={30}>
+                    {Row}
+                  </FixedSizeList>
+                )}
+          </div>
+          <div className={styles.footer}>
+            {limit && data.length > limit && (
+              <Button icon={<Arrow />} size="xsmall" onClick={() => onExpand(type)}>
+                <div>More</div>
+              </Button>
             )}
-      </div>
-      <div className={styles.footer}>
-        {limit && data.length > limit && (
-          <Button icon={<Arrow />} size="xsmall" onClick={() => onExpand(type)}>
-            <div>More</div>
-          </Button>
-        )}
-      </div>
+          </div>
+        </>
+      ) : (
+        <Loading />
+      )}
     </div>
   );
 }

components/metrics/PageviewsChart.js

@@ -1,16 +1,15 @@
 import React from 'react';
+import CheckVisible from 'components/helpers/CheckVisible';
 import BarChart from './BarChart';
 
-export default function PageviewsChart({ websiteId, data, unit, className, animationDuration }) {
+export default function PageviewsChart({ websiteId, data, unit, className }) {
   const handleUpdate = chart => {
     const {
       data: { datasets },
-      options,
     } = chart;
 
     datasets[0].data = data.uniques;
     datasets[1].data = data.pageviews;
-    options.animation.duration = animationDuration;
 
     chart.update();
   };
@@ -20,30 +19,35 @@ export default function PageviewsChart({ websiteId, data, unit, className, anima
   }
 
   return (
-    <BarChart
-      className={className}
-      chartId={websiteId}
-      datasets={[
-        {
-          label: 'unique visitors',
-          data: data.uniques,
-          lineTension: 0,
-          backgroundColor: 'rgb(38, 128, 235, 0.4)',
-          borderColor: 'rgb(13, 102, 208, 0.4)',
-          borderWidth: 1,
-        },
-        {
-          label: 'page views',
-          data: data.pageviews,
-          lineTension: 0,
-          backgroundColor: 'rgb(38, 128, 235, 0.2)',
-          borderColor: 'rgb(13, 102, 208, 0.2)',
-          borderWidth: 1,
-        },
-      ]}
-      unit={unit}
-      records={data.pageviews.length}
-      onUpdate={handleUpdate}
-    />
+    <CheckVisible>
+      {visible => (
+        <BarChart
+          className={className}
+          chartId={websiteId}
+          datasets={[
+            {
+              label: 'unique visitors',
+              data: data.uniques,
+              lineTension: 0,
+              backgroundColor: 'rgb(38, 128, 235, 0.4)',
+              borderColor: 'rgb(13, 102, 208, 0.4)',
+              borderWidth: 1,
+            },
+            {
+              label: 'page views',
+              data: data.pageviews,
+              lineTension: 0,
+              backgroundColor: 'rgb(38, 128, 235, 0.2)',
+              borderColor: 'rgb(13, 102, 208, 0.2)',
+              borderWidth: 1,
+            },
+          ]}
+          unit={unit}
+          records={data.pageviews.length}
+          animationDuration={visible ? 300 : 0}
+          onUpdate={handleUpdate}
+        />
+      )}
+    </CheckVisible>
   );
 }

components/metrics/WebsiteChart.js

@@ -1,7 +1,6 @@
-import React, { useState, useEffect, useMemo, useRef } from 'react';
+import React, { useState, useEffect, useMemo } from 'react';
 import classNames from 'classnames';
 import PageviewsChart from './PageviewsChart';
-import CheckVisible from '../helpers/CheckVisible';
 import MetricsBar from './MetricsBar';
 import QuickButtons from './QuickButtons';
 import DateFilter from '../common/DateFilter';
@@ -74,19 +73,10 @@ export default function WebsiteChart({
         </StickyHeader>
       </div>
       <div className="row">
-        <CheckVisible className="col">
-          {visible => (
-            <>
-              <PageviewsChart
-                websiteId={websiteId}
-                data={{ pageviews, uniques }}
-                unit={unit}
-                animationDuration={visible ? 300 : 0}
-              />
-              <QuickButtons value={value} onChange={handleDateChange} />
-            </>
-          )}
-        </CheckVisible>
+        <div className="col">
+          <PageviewsChart websiteId={websiteId} data={{ pageviews, uniques }} unit={unit} />
+          <QuickButtons value={value} onChange={handleDateChange} />
+        </div>
       </div>
     </>
   );

lib/date.js

@@ -98,6 +98,12 @@ export function getDateArray(data, startDate, endDate, unit) {
 
   function findData(t) {
     const x = data.find(e => {
+      console.log(
+        new Date(e.t),
+        getLocalTime(new Date(e.t)),
+        getLocalTime(new Date(e.t)).getTime(),
+        normalize(new Date(t)).getTime(),
+      );
       return getLocalTime(new Date(e.t)).getTime() === normalize(new Date(t)).getTime();
     });
 

lib/filters.js

@@ -49,12 +49,13 @@ export const urlFilter = (data, { domain, raw }) => {
 };
 
 export const refFilter = (data, { domain, domainOnly, raw }) => {
+  const regex = new RegExp(domain.startsWith('http') ? domain : `http[s]?://${domain}`);
+
   const isValidRef = ref => {
     return ref !== '' && !ref.startsWith('/') && !ref.startsWith('#');
   };
 
   if (raw) {
-    const regex = new RegExp(`http[s]?://${domain}`);
     return data.filter(({ x }) => isValidRef(x) && !regex.test(x));
   }
 
@@ -62,7 +63,7 @@ export const refFilter = (data, { domain, domainOnly, raw }) => {
     try {
       const { hostname, origin, pathname, searchParams, protocol } = new URL(url);
 
-      if (hostname === domain) {
+      if (hostname === domain || regex.test(url)) {
         return null;
       }
 

lib/queries.js

@@ -5,10 +5,28 @@ import { subMinutes } from 'date-fns';
 const POSTGRESQL = 'postgresql';
 const MYSQL = 'mysql';
 
+const DATE_FORMATS = {
+  minute: '%Y-%m-%d %H:%i:00',
+  hour: '%Y-%m-%d %H:00:00',
+  day: '%Y-%m-%d',
+  month: '%Y-%m-01',
+  year: '%Y-01-01',
+};
+
 export function getDatabase() {
   return process.env.DATABASE_TYPE || process.env.DATABASE_URL.split(':')[0];
 }
 
+export function getDateQuery(field, unit, timezone) {
+  if (timezone) {
+    const tz = moment.tz(timezone).format('Z');
+
+    return `DATE_FORMAT(convert_tz(${field},'+00:00','${tz}'), '${DATE_FORMATS[unit]}')`;
+  }
+
+  return `DATE_FORMAT(${field}, '${DATE_FORMATS[unit]}')`;
+}
+
 export async function getWebsiteById(website_id) {
   return runQuery(
     prisma.website.findOne({
@@ -264,7 +282,7 @@ export function getMetrics(website_id, start_at, end_at) {
         sum(t.time) as "totaltime"
       from (
          select session_id,
-           date_trunc('hour', created_at),
+           ${getDateQuery('created_at', 'hour')},
            count(*) c,
            floor(unix_timestamp(max(created_at)) - unix_timestamp(min(created_at))) as "time"
          from pageview
@@ -296,7 +314,7 @@ export function getPageviews(
     return prisma.$queryRaw(
       `
       select date_trunc('${unit}', created_at at time zone '${timezone}') t,
-      count(${count}) y
+        count(${count}) y
       from pageview
       where website_id=$1
       and created_at between $2 and $3
@@ -310,11 +328,10 @@ export function getPageviews(
   }
 
   if (db === MYSQL) {
-    const tz = moment.tz(timezone).format('Z');
     return prisma.$queryRaw(
       `
-      select date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t,
-      count(${count}) y
+      select ${getDateQuery('created_at', unit, timezone)} t,
+        count(${count}) y
       from pageview
       where website_id=?
       and created_at between ? and ?
@@ -424,12 +441,11 @@ export function getEvents(website_id, start_at, end_at, timezone = 'utc', unit =
   }
 
   if (db === MYSQL) {
-    const tz = moment.tz(timezone).format('Z');
     return prisma.$queryRaw(
       `
       select
         event_value x,
-        date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t,
+        ${getDateQuery('created_at', unit, timezone)} t,
         count(*) y
       from event
       where website_id=?

sql/schema.mysql.sql

@@ -3,7 +3,6 @@ drop table if exists pageview;
 drop table if exists session;
 drop table if exists website;
 drop table if exists account;
-drop function if exists date_trunc;
 
 create table account (
     user_id int unsigned not null auto_increment primary key,
@@ -72,37 +71,10 @@ create index pageview_created_at_idx on pageview(created_at);
 create index pageview_website_id_idx on pageview(website_id);
 create index pageview_session_id_idx on pageview(session_id);
 create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
+create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
 
 create index event_created_at_idx on event(created_at);
 create index event_website_id_idx on event(website_id);
 create index event_session_id_idx on event(session_id);
 
-delimiter $$
-
-create function date_trunc(
-  in_granularity enum('minute', 'hour', 'day', 'month', 'year'),
-  in_datetime datetime(6)
-)
-returns datetime(6)
-deterministic 
-begin
-  if (in_granularity = 'minute') then
-    return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:%i:00.0000');
-  end if;
-  if (in_granularity = 'hour') then
-    return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:00:00.0000');
-  end if;
-  if (in_granularity = 'day') then
-    return DATE_FORMAT(in_datetime, '%Y-%m-%d 00:00:00.0000');
-  end if;
-  if (in_granularity = 'month') then
-    return DATE_FORMAT(in_datetime, '%Y-%m-01 00:00:00.0000');
-  end if;
-  if (in_granularity = 'year') then
-    return DATE_FORMAT(in_datetime, '%Y-01-01 00:00:00.0000');
-  end if;
-end;
-
-$$
-
 insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);

sql/schema.postgresql.sql

@@ -65,6 +65,7 @@ create index pageview_created_at_idx on pageview(created_at);
 create index pageview_website_id_idx on pageview(website_id);
 create index pageview_session_id_idx on pageview(session_id);
 create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
+create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
 
 create index event_created_at_idx on event(created_at);
 create index event_website_id_idx on event(website_id);