From ccb98f836fef74dfa7f056c238af5f3e0ea455eb Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 27 Aug 2020 23:45:37 -0700 Subject: [PATCH] MySQL query optimization. Added loading component. --- components/common/Loading.js | 13 ++++++ components/common/Loading.module.css | 41 +++++++++++++++++ components/layout/MenuLayout.module.css | 1 + components/metrics/BarChart.js | 3 +- components/metrics/EventsChart.js | 2 +- components/metrics/MetricsTable.js | 55 ++++++++++++----------- components/metrics/PageviewsChart.js | 60 +++++++++++++------------ components/metrics/WebsiteChart.js | 20 +++------ lib/date.js | 6 +++ lib/filters.js | 5 ++- lib/queries.js | 30 ++++++++++--- sql/schema.mysql.sql | 30 +------------ sql/schema.postgresql.sql | 1 + 13 files changed, 158 insertions(+), 109 deletions(-) create mode 100644 components/common/Loading.js create mode 100644 components/common/Loading.module.css diff --git a/components/common/Loading.js b/components/common/Loading.js new file mode 100644 index 00000000..46e72f17 --- /dev/null +++ b/components/common/Loading.js @@ -0,0 +1,13 @@ +import React from 'react'; +import classNames from 'classnames'; +import styles from './Loading.module.css'; + +export default function Loading({ className }) { + return ( +
+
+
+
+
+ ); +} diff --git a/components/common/Loading.module.css b/components/common/Loading.module.css new file mode 100644 index 00000000..2a210078 --- /dev/null +++ b/components/common/Loading.module.css @@ -0,0 +1,41 @@ +@keyframes blink { + 0% { + opacity: 0.2; + } + 20% { + opacity: 1; + } + 100% { + opacity: 0.2; + } +} + +.loading { + display: flex; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + margin: 0; +} + +.loading div { + width: 10px; + height: 10px; + border-radius: 100%; + background: var(--gray400); + animation: blink 1.4s infinite; + animation-fill-mode: both; +} + +.loading div + div { + margin-left: 10px; +} + +.loading div:nth-child(2) { + animation-delay: 0.2s; +} + +.loading div:nth-child(3) { + animation-delay: 0.4s; +} diff --git a/components/layout/MenuLayout.module.css b/components/layout/MenuLayout.module.css index 62233fd6..1bdd7473 100644 --- a/components/layout/MenuLayout.module.css +++ b/components/layout/MenuLayout.module.css @@ -10,6 +10,7 @@ } .container .content { + position: relative; border-left: 1px solid var(--gray300); padding-left: 30px; } diff --git a/components/metrics/BarChart.js b/components/metrics/BarChart.js index fdce6330..25c14fe5 100644 --- a/components/metrics/BarChart.js +++ b/components/metrics/BarChart.js @@ -120,6 +120,7 @@ export default function BarChart({ options.scales.xAxes[0].time.unit = unit; options.scales.xAxes[0].ticks.callback = renderLabel; + options.animation.duration = animationDuration; onUpdate(chart.current); }; @@ -133,7 +134,7 @@ export default function BarChart({ updateChart(); } } - }, [datasets]); + }, [datasets, unit, animationDuration]); return ( <> diff --git a/components/metrics/EventsChart.js b/components/metrics/EventsChart.js index 1825116f..7a751b5b 100644 --- a/components/metrics/EventsChart.js +++ b/components/metrics/EventsChart.js @@ -42,7 +42,7 @@ export default function EventsChart({ websiteId, startDate, endDate, unit }) { unit, tz: getTimezone(), }); - console.log({ data }); + const map = data.reduce((obj, { x, t, y }) => { if (!obj[x]) { obj[x] = []; diff --git a/components/metrics/MetricsTable.js b/components/metrics/MetricsTable.js index bd66cb85..566ceed3 100644 --- a/components/metrics/MetricsTable.js +++ b/components/metrics/MetricsTable.js @@ -8,6 +8,7 @@ import { get } from 'lib/web'; import { percentFilter } from 'lib/filters'; import { formatNumber, formatLongNumber } from 'lib/format'; import styles from './MetricsTable.module.css'; +import Loading from '../common/Loading'; export default function MetricsTable({ title, @@ -79,35 +80,37 @@ export default function MetricsTable({ } }, [websiteId, startDate, endDate, type]); - if (!data) { - return null; - } - return (
-
-
{title}
- {headerComponent} -
- {metric} -
-
-
- {limit - ? rankings.map(row => getRow(row)) - : data?.length > 0 && ( - - {Row} - + {data ? ( + <> +
+
{title}
+ {headerComponent} +
+ {metric} +
+
+
+ {limit + ? rankings.map(row => getRow(row)) + : data?.length > 0 && ( + + {Row} + + )} +
+
+ {limit && data.length > limit && ( + )} -
-
- {limit && data.length > limit && ( - - )} -
+
+ + ) : ( + + )}
); } diff --git a/components/metrics/PageviewsChart.js b/components/metrics/PageviewsChart.js index 1052452b..f8aa0cdc 100644 --- a/components/metrics/PageviewsChart.js +++ b/components/metrics/PageviewsChart.js @@ -1,16 +1,15 @@ import React from 'react'; +import CheckVisible from 'components/helpers/CheckVisible'; import BarChart from './BarChart'; -export default function PageviewsChart({ websiteId, data, unit, className, animationDuration }) { +export default function PageviewsChart({ websiteId, data, unit, className }) { const handleUpdate = chart => { const { data: { datasets }, - options, } = chart; datasets[0].data = data.uniques; datasets[1].data = data.pageviews; - options.animation.duration = animationDuration; chart.update(); }; @@ -20,30 +19,35 @@ export default function PageviewsChart({ websiteId, data, unit, className, anima } return ( - + + {visible => ( + + )} + ); } diff --git a/components/metrics/WebsiteChart.js b/components/metrics/WebsiteChart.js index f93a985a..9c53a560 100644 --- a/components/metrics/WebsiteChart.js +++ b/components/metrics/WebsiteChart.js @@ -1,7 +1,6 @@ -import React, { useState, useEffect, useMemo, useRef } from 'react'; +import React, { useState, useEffect, useMemo } from 'react'; import classNames from 'classnames'; import PageviewsChart from './PageviewsChart'; -import CheckVisible from '../helpers/CheckVisible'; import MetricsBar from './MetricsBar'; import QuickButtons from './QuickButtons'; import DateFilter from '../common/DateFilter'; @@ -74,19 +73,10 @@ export default function WebsiteChart({
- - {visible => ( - <> - - - - )} - +
+ + +
); diff --git a/lib/date.js b/lib/date.js index 6e10f610..80980e27 100644 --- a/lib/date.js +++ b/lib/date.js @@ -98,6 +98,12 @@ export function getDateArray(data, startDate, endDate, unit) { function findData(t) { const x = data.find(e => { + console.log( + new Date(e.t), + getLocalTime(new Date(e.t)), + getLocalTime(new Date(e.t)).getTime(), + normalize(new Date(t)).getTime(), + ); return getLocalTime(new Date(e.t)).getTime() === normalize(new Date(t)).getTime(); }); diff --git a/lib/filters.js b/lib/filters.js index a1bf1b96..09f27e70 100644 --- a/lib/filters.js +++ b/lib/filters.js @@ -49,12 +49,13 @@ export const urlFilter = (data, { domain, raw }) => { }; export const refFilter = (data, { domain, domainOnly, raw }) => { + const regex = new RegExp(domain.startsWith('http') ? domain : `http[s]?://${domain}`); + const isValidRef = ref => { return ref !== '' && !ref.startsWith('/') && !ref.startsWith('#'); }; if (raw) { - const regex = new RegExp(`http[s]?://${domain}`); return data.filter(({ x }) => isValidRef(x) && !regex.test(x)); } @@ -62,7 +63,7 @@ export const refFilter = (data, { domain, domainOnly, raw }) => { try { const { hostname, origin, pathname, searchParams, protocol } = new URL(url); - if (hostname === domain) { + if (hostname === domain || regex.test(url)) { return null; } diff --git a/lib/queries.js b/lib/queries.js index 1907a954..7c8a9875 100644 --- a/lib/queries.js +++ b/lib/queries.js @@ -5,10 +5,28 @@ import { subMinutes } from 'date-fns'; const POSTGRESQL = 'postgresql'; const MYSQL = 'mysql'; +const DATE_FORMATS = { + minute: '%Y-%m-%d %H:%i:00', + hour: '%Y-%m-%d %H:00:00', + day: '%Y-%m-%d', + month: '%Y-%m-01', + year: '%Y-01-01', +}; + export function getDatabase() { return process.env.DATABASE_TYPE || process.env.DATABASE_URL.split(':')[0]; } +export function getDateQuery(field, unit, timezone) { + if (timezone) { + const tz = moment.tz(timezone).format('Z'); + + return `DATE_FORMAT(convert_tz(${field},'+00:00','${tz}'), '${DATE_FORMATS[unit]}')`; + } + + return `DATE_FORMAT(${field}, '${DATE_FORMATS[unit]}')`; +} + export async function getWebsiteById(website_id) { return runQuery( prisma.website.findOne({ @@ -264,7 +282,7 @@ export function getMetrics(website_id, start_at, end_at) { sum(t.time) as "totaltime" from ( select session_id, - date_trunc('hour', created_at), + ${getDateQuery('created_at', 'hour')}, count(*) c, floor(unix_timestamp(max(created_at)) - unix_timestamp(min(created_at))) as "time" from pageview @@ -296,7 +314,7 @@ export function getPageviews( return prisma.$queryRaw( ` select date_trunc('${unit}', created_at at time zone '${timezone}') t, - count(${count}) y + count(${count}) y from pageview where website_id=$1 and created_at between $2 and $3 @@ -310,11 +328,10 @@ export function getPageviews( } if (db === MYSQL) { - const tz = moment.tz(timezone).format('Z'); return prisma.$queryRaw( ` - select date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t, - count(${count}) y + select ${getDateQuery('created_at', unit, timezone)} t, + count(${count}) y from pageview where website_id=? and created_at between ? and ? @@ -424,12 +441,11 @@ export function getEvents(website_id, start_at, end_at, timezone = 'utc', unit = } if (db === MYSQL) { - const tz = moment.tz(timezone).format('Z'); return prisma.$queryRaw( ` select event_value x, - date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t, + ${getDateQuery('created_at', unit, timezone)} t, count(*) y from event where website_id=? diff --git a/sql/schema.mysql.sql b/sql/schema.mysql.sql index 106a6748..a4fd87f0 100644 --- a/sql/schema.mysql.sql +++ b/sql/schema.mysql.sql @@ -3,7 +3,6 @@ drop table if exists pageview; drop table if exists session; drop table if exists website; drop table if exists account; -drop function if exists date_trunc; create table account ( user_id int unsigned not null auto_increment primary key, @@ -72,37 +71,10 @@ create index pageview_created_at_idx on pageview(created_at); create index pageview_website_id_idx on pageview(website_id); create index pageview_session_id_idx on pageview(session_id); create index pageview_website_id_created_at_idx on pageview(website_id, created_at); +create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at); create index event_created_at_idx on event(created_at); create index event_website_id_idx on event(website_id); create index event_session_id_idx on event(session_id); -delimiter $$ - -create function date_trunc( - in_granularity enum('minute', 'hour', 'day', 'month', 'year'), - in_datetime datetime(6) -) -returns datetime(6) -deterministic -begin - if (in_granularity = 'minute') then - return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:%i:00.0000'); - end if; - if (in_granularity = 'hour') then - return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:00:00.0000'); - end if; - if (in_granularity = 'day') then - return DATE_FORMAT(in_datetime, '%Y-%m-%d 00:00:00.0000'); - end if; - if (in_granularity = 'month') then - return DATE_FORMAT(in_datetime, '%Y-%m-01 00:00:00.0000'); - end if; - if (in_granularity = 'year') then - return DATE_FORMAT(in_datetime, '%Y-01-01 00:00:00.0000'); - end if; -end; - -$$ - insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true); \ No newline at end of file diff --git a/sql/schema.postgresql.sql b/sql/schema.postgresql.sql index 43533349..28297c4a 100644 --- a/sql/schema.postgresql.sql +++ b/sql/schema.postgresql.sql @@ -65,6 +65,7 @@ create index pageview_created_at_idx on pageview(created_at); create index pageview_website_id_idx on pageview(website_id); create index pageview_session_id_idx on pageview(session_id); create index pageview_website_id_created_at_idx on pageview(website_id, created_at); +create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at); create index event_created_at_idx on event(created_at); create index event_website_id_idx on event(website_id);