MySQL query optimization. Added loading component.

This commit is contained in:
Mike Cao 2020-08-27 23:45:37 -07:00
parent a7e7469d22
commit ccb98f836f
13 changed files with 158 additions and 109 deletions

View File

@ -0,0 +1,13 @@
import React from 'react';
import classNames from 'classnames';
import styles from './Loading.module.css';
export default function Loading({ className }) {
return (
<div className={classNames(styles.loading, className)}>
<div />
<div />
<div />
</div>
);
}

View File

@ -0,0 +1,41 @@
@keyframes blink {
0% {
opacity: 0.2;
}
20% {
opacity: 1;
}
100% {
opacity: 0.2;
}
}
.loading {
display: flex;
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
margin: 0;
}
.loading div {
width: 10px;
height: 10px;
border-radius: 100%;
background: var(--gray400);
animation: blink 1.4s infinite;
animation-fill-mode: both;
}
.loading div + div {
margin-left: 10px;
}
.loading div:nth-child(2) {
animation-delay: 0.2s;
}
.loading div:nth-child(3) {
animation-delay: 0.4s;
}

View File

@ -10,6 +10,7 @@
} }
.container .content { .container .content {
position: relative;
border-left: 1px solid var(--gray300); border-left: 1px solid var(--gray300);
padding-left: 30px; padding-left: 30px;
} }

View File

@ -120,6 +120,7 @@ export default function BarChart({
options.scales.xAxes[0].time.unit = unit; options.scales.xAxes[0].time.unit = unit;
options.scales.xAxes[0].ticks.callback = renderLabel; options.scales.xAxes[0].ticks.callback = renderLabel;
options.animation.duration = animationDuration;
onUpdate(chart.current); onUpdate(chart.current);
}; };
@ -133,7 +134,7 @@ export default function BarChart({
updateChart(); updateChart();
} }
} }
}, [datasets]); }, [datasets, unit, animationDuration]);
return ( return (
<> <>

View File

@ -42,7 +42,7 @@ export default function EventsChart({ websiteId, startDate, endDate, unit }) {
unit, unit,
tz: getTimezone(), tz: getTimezone(),
}); });
console.log({ data });
const map = data.reduce((obj, { x, t, y }) => { const map = data.reduce((obj, { x, t, y }) => {
if (!obj[x]) { if (!obj[x]) {
obj[x] = []; obj[x] = [];

View File

@ -8,6 +8,7 @@ import { get } from 'lib/web';
import { percentFilter } from 'lib/filters'; import { percentFilter } from 'lib/filters';
import { formatNumber, formatLongNumber } from 'lib/format'; import { formatNumber, formatLongNumber } from 'lib/format';
import styles from './MetricsTable.module.css'; import styles from './MetricsTable.module.css';
import Loading from '../common/Loading';
export default function MetricsTable({ export default function MetricsTable({
title, title,
@ -79,35 +80,37 @@ export default function MetricsTable({
} }
}, [websiteId, startDate, endDate, type]); }, [websiteId, startDate, endDate, type]);
if (!data) {
return null;
}
return ( return (
<div className={classNames(styles.container, className)}> <div className={classNames(styles.container, className)}>
<div className={styles.header}> {data ? (
<div className={styles.title}>{title}</div> <>
{headerComponent} <div className={styles.header}>
<div className={styles.metric} onClick={handleSetFormat}> <div className={styles.title}>{title}</div>
{metric} {headerComponent}
</div> <div className={styles.metric} onClick={handleSetFormat}>
</div> {metric}
<div className={styles.body}> </div>
{limit </div>
? rankings.map(row => getRow(row)) <div className={styles.body}>
: data?.length > 0 && ( {limit
<FixedSizeList height={500} itemCount={rankings.length} itemSize={30}> ? rankings.map(row => getRow(row))
{Row} : data?.length > 0 && (
</FixedSizeList> <FixedSizeList height={500} itemCount={rankings.length} itemSize={30}>
{Row}
</FixedSizeList>
)}
</div>
<div className={styles.footer}>
{limit && data.length > limit && (
<Button icon={<Arrow />} size="xsmall" onClick={() => onExpand(type)}>
<div>More</div>
</Button>
)} )}
</div> </div>
<div className={styles.footer}> </>
{limit && data.length > limit && ( ) : (
<Button icon={<Arrow />} size="xsmall" onClick={() => onExpand(type)}> <Loading />
<div>More</div> )}
</Button>
)}
</div>
</div> </div>
); );
} }

View File

@ -1,16 +1,15 @@
import React from 'react'; import React from 'react';
import CheckVisible from 'components/helpers/CheckVisible';
import BarChart from './BarChart'; import BarChart from './BarChart';
export default function PageviewsChart({ websiteId, data, unit, className, animationDuration }) { export default function PageviewsChart({ websiteId, data, unit, className }) {
const handleUpdate = chart => { const handleUpdate = chart => {
const { const {
data: { datasets }, data: { datasets },
options,
} = chart; } = chart;
datasets[0].data = data.uniques; datasets[0].data = data.uniques;
datasets[1].data = data.pageviews; datasets[1].data = data.pageviews;
options.animation.duration = animationDuration;
chart.update(); chart.update();
}; };
@ -20,30 +19,35 @@ export default function PageviewsChart({ websiteId, data, unit, className, anima
} }
return ( return (
<BarChart <CheckVisible>
className={className} {visible => (
chartId={websiteId} <BarChart
datasets={[ className={className}
{ chartId={websiteId}
label: 'unique visitors', datasets={[
data: data.uniques, {
lineTension: 0, label: 'unique visitors',
backgroundColor: 'rgb(38, 128, 235, 0.4)', data: data.uniques,
borderColor: 'rgb(13, 102, 208, 0.4)', lineTension: 0,
borderWidth: 1, backgroundColor: 'rgb(38, 128, 235, 0.4)',
}, borderColor: 'rgb(13, 102, 208, 0.4)',
{ borderWidth: 1,
label: 'page views', },
data: data.pageviews, {
lineTension: 0, label: 'page views',
backgroundColor: 'rgb(38, 128, 235, 0.2)', data: data.pageviews,
borderColor: 'rgb(13, 102, 208, 0.2)', lineTension: 0,
borderWidth: 1, backgroundColor: 'rgb(38, 128, 235, 0.2)',
}, borderColor: 'rgb(13, 102, 208, 0.2)',
]} borderWidth: 1,
unit={unit} },
records={data.pageviews.length} ]}
onUpdate={handleUpdate} unit={unit}
/> records={data.pageviews.length}
animationDuration={visible ? 300 : 0}
onUpdate={handleUpdate}
/>
)}
</CheckVisible>
); );
} }

View File

@ -1,7 +1,6 @@
import React, { useState, useEffect, useMemo, useRef } from 'react'; import React, { useState, useEffect, useMemo } from 'react';
import classNames from 'classnames'; import classNames from 'classnames';
import PageviewsChart from './PageviewsChart'; import PageviewsChart from './PageviewsChart';
import CheckVisible from '../helpers/CheckVisible';
import MetricsBar from './MetricsBar'; import MetricsBar from './MetricsBar';
import QuickButtons from './QuickButtons'; import QuickButtons from './QuickButtons';
import DateFilter from '../common/DateFilter'; import DateFilter from '../common/DateFilter';
@ -74,19 +73,10 @@ export default function WebsiteChart({
</StickyHeader> </StickyHeader>
</div> </div>
<div className="row"> <div className="row">
<CheckVisible className="col"> <div className="col">
{visible => ( <PageviewsChart websiteId={websiteId} data={{ pageviews, uniques }} unit={unit} />
<> <QuickButtons value={value} onChange={handleDateChange} />
<PageviewsChart </div>
websiteId={websiteId}
data={{ pageviews, uniques }}
unit={unit}
animationDuration={visible ? 300 : 0}
/>
<QuickButtons value={value} onChange={handleDateChange} />
</>
)}
</CheckVisible>
</div> </div>
</> </>
); );

View File

@ -98,6 +98,12 @@ export function getDateArray(data, startDate, endDate, unit) {
function findData(t) { function findData(t) {
const x = data.find(e => { const x = data.find(e => {
console.log(
new Date(e.t),
getLocalTime(new Date(e.t)),
getLocalTime(new Date(e.t)).getTime(),
normalize(new Date(t)).getTime(),
);
return getLocalTime(new Date(e.t)).getTime() === normalize(new Date(t)).getTime(); return getLocalTime(new Date(e.t)).getTime() === normalize(new Date(t)).getTime();
}); });

View File

@ -49,12 +49,13 @@ export const urlFilter = (data, { domain, raw }) => {
}; };
export const refFilter = (data, { domain, domainOnly, raw }) => { export const refFilter = (data, { domain, domainOnly, raw }) => {
const regex = new RegExp(domain.startsWith('http') ? domain : `http[s]?://${domain}`);
const isValidRef = ref => { const isValidRef = ref => {
return ref !== '' && !ref.startsWith('/') && !ref.startsWith('#'); return ref !== '' && !ref.startsWith('/') && !ref.startsWith('#');
}; };
if (raw) { if (raw) {
const regex = new RegExp(`http[s]?://${domain}`);
return data.filter(({ x }) => isValidRef(x) && !regex.test(x)); return data.filter(({ x }) => isValidRef(x) && !regex.test(x));
} }
@ -62,7 +63,7 @@ export const refFilter = (data, { domain, domainOnly, raw }) => {
try { try {
const { hostname, origin, pathname, searchParams, protocol } = new URL(url); const { hostname, origin, pathname, searchParams, protocol } = new URL(url);
if (hostname === domain) { if (hostname === domain || regex.test(url)) {
return null; return null;
} }

View File

@ -5,10 +5,28 @@ import { subMinutes } from 'date-fns';
const POSTGRESQL = 'postgresql'; const POSTGRESQL = 'postgresql';
const MYSQL = 'mysql'; const MYSQL = 'mysql';
const DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d',
month: '%Y-%m-01',
year: '%Y-01-01',
};
export function getDatabase() { export function getDatabase() {
return process.env.DATABASE_TYPE || process.env.DATABASE_URL.split(':')[0]; return process.env.DATABASE_TYPE || process.env.DATABASE_URL.split(':')[0];
} }
export function getDateQuery(field, unit, timezone) {
if (timezone) {
const tz = moment.tz(timezone).format('Z');
return `DATE_FORMAT(convert_tz(${field},'+00:00','${tz}'), '${DATE_FORMATS[unit]}')`;
}
return `DATE_FORMAT(${field}, '${DATE_FORMATS[unit]}')`;
}
export async function getWebsiteById(website_id) { export async function getWebsiteById(website_id) {
return runQuery( return runQuery(
prisma.website.findOne({ prisma.website.findOne({
@ -264,7 +282,7 @@ export function getMetrics(website_id, start_at, end_at) {
sum(t.time) as "totaltime" sum(t.time) as "totaltime"
from ( from (
select session_id, select session_id,
date_trunc('hour', created_at), ${getDateQuery('created_at', 'hour')},
count(*) c, count(*) c,
floor(unix_timestamp(max(created_at)) - unix_timestamp(min(created_at))) as "time" floor(unix_timestamp(max(created_at)) - unix_timestamp(min(created_at))) as "time"
from pageview from pageview
@ -296,7 +314,7 @@ export function getPageviews(
return prisma.$queryRaw( return prisma.$queryRaw(
` `
select date_trunc('${unit}', created_at at time zone '${timezone}') t, select date_trunc('${unit}', created_at at time zone '${timezone}') t,
count(${count}) y count(${count}) y
from pageview from pageview
where website_id=$1 where website_id=$1
and created_at between $2 and $3 and created_at between $2 and $3
@ -310,11 +328,10 @@ export function getPageviews(
} }
if (db === MYSQL) { if (db === MYSQL) {
const tz = moment.tz(timezone).format('Z');
return prisma.$queryRaw( return prisma.$queryRaw(
` `
select date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t, select ${getDateQuery('created_at', unit, timezone)} t,
count(${count}) y count(${count}) y
from pageview from pageview
where website_id=? where website_id=?
and created_at between ? and ? and created_at between ? and ?
@ -424,12 +441,11 @@ export function getEvents(website_id, start_at, end_at, timezone = 'utc', unit =
} }
if (db === MYSQL) { if (db === MYSQL) {
const tz = moment.tz(timezone).format('Z');
return prisma.$queryRaw( return prisma.$queryRaw(
` `
select select
event_value x, event_value x,
date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t, ${getDateQuery('created_at', unit, timezone)} t,
count(*) y count(*) y
from event from event
where website_id=? where website_id=?

View File

@ -3,7 +3,6 @@ drop table if exists pageview;
drop table if exists session; drop table if exists session;
drop table if exists website; drop table if exists website;
drop table if exists account; drop table if exists account;
drop function if exists date_trunc;
create table account ( create table account (
user_id int unsigned not null auto_increment primary key, user_id int unsigned not null auto_increment primary key,
@ -72,37 +71,10 @@ create index pageview_created_at_idx on pageview(created_at);
create index pageview_website_id_idx on pageview(website_id); create index pageview_website_id_idx on pageview(website_id);
create index pageview_session_id_idx on pageview(session_id); create index pageview_session_id_idx on pageview(session_id);
create index pageview_website_id_created_at_idx on pageview(website_id, created_at); create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
create index event_created_at_idx on event(created_at); create index event_created_at_idx on event(created_at);
create index event_website_id_idx on event(website_id); create index event_website_id_idx on event(website_id);
create index event_session_id_idx on event(session_id); create index event_session_id_idx on event(session_id);
delimiter $$
create function date_trunc(
in_granularity enum('minute', 'hour', 'day', 'month', 'year'),
in_datetime datetime(6)
)
returns datetime(6)
deterministic
begin
if (in_granularity = 'minute') then
return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:%i:00.0000');
end if;
if (in_granularity = 'hour') then
return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:00:00.0000');
end if;
if (in_granularity = 'day') then
return DATE_FORMAT(in_datetime, '%Y-%m-%d 00:00:00.0000');
end if;
if (in_granularity = 'month') then
return DATE_FORMAT(in_datetime, '%Y-%m-01 00:00:00.0000');
end if;
if (in_granularity = 'year') then
return DATE_FORMAT(in_datetime, '%Y-01-01 00:00:00.0000');
end if;
end;
$$
insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true); insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);

View File

@ -65,6 +65,7 @@ create index pageview_created_at_idx on pageview(created_at);
create index pageview_website_id_idx on pageview(website_id); create index pageview_website_id_idx on pageview(website_id);
create index pageview_session_id_idx on pageview(session_id); create index pageview_session_id_idx on pageview(session_id);
create index pageview_website_id_created_at_idx on pageview(website_id, created_at); create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
create index event_created_at_idx on event(created_at); create index event_created_at_idx on event(created_at);
create index event_website_id_idx on event(website_id); create index event_website_id_idx on event(website_id);