Refactored filter logic.

This commit is contained in:
Mike Cao 2023-04-01 15:44:30 -07:00
parent 57c3d03cc8
commit 74192cd695
18 changed files with 205 additions and 296 deletions

View File

@ -5,9 +5,9 @@
transform: translate(-50%, -50%);
margin: auto;
display: flex;
z-index: var(--z-index-above);
background-color: var(--base50);
padding: 10px;
z-index: 1;
}
.icon {

View File

@ -14,7 +14,7 @@ export default function FilterLink({ id, value, label, externalUrl }) {
return (
<div className={styles.row}>
{!value && <span className={styles.empty}>{label || formatMessage(labels.unknown)}</span>}
{!value && `(${label || formatMessage(labels.unknown)})`}
{value && (
<Link
href={resolveUrl({ [id]: value })}

View File

@ -30,7 +30,3 @@
.icon {
cursor: pointer;
}
.empty {
color: var(--base400);
}

View File

@ -1,10 +1,13 @@
import classNames from 'classnames';
import { Banner, Loading } from 'react-basics';
import useMessages from 'hooks/useMessages';
import styles from './Page.module.css';
export default function Page({ className, error, loading, children }) {
const { formatMessage, messages } = useMessages();
if (error) {
return <Banner variant="error">Something went wrong.</Banner>;
return <Banner variant="error">{formatMessage(messages.error)}</Banner>;
}
if (loading) {

View File

@ -28,7 +28,7 @@ export default function PagesTable({ websiteId, showFilters, ...props }) {
];
const renderLink = ({ x }) => {
return <FilterLink id={view} value={x || `(${formatMessage(labels.none)})`} />;
return <FilterLink id={view} value={x || formatMessage(labels.none)} />;
};
return (

View File

@ -5,13 +5,13 @@ import useMessages from 'hooks/useMessages';
export default function ReferrersTable({ websiteId, ...props }) {
const { formatMessage, labels } = useMessages();
const renderLink = ({ w: link, x: referrer }) => {
const renderLink = ({ x: referrer }) => {
return (
<FilterLink
id="referrer"
value={referrer}
externalUrl={link}
label={!referrer && `(${formatMessage(labels.none)})`}
externalUrl={`https://${referrer}`}
label={!referrer && formatMessage(labels.none)}
/>
);
};

View File

@ -1,9 +1,10 @@
import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import debug from 'debug';
import { FILTER_IGNORED } from 'lib/constants';
import { CLICKHOUSE } from 'lib/db';
import { getEventDataType } from './eventData';
import { WebsiteMetricFilter } from './types';
import { FILTER_COLUMNS } from './constants';
export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00',
@ -104,55 +105,14 @@ function getEventDataFilterQuery(
return query.join('\n');
}
function getFilterQuery(filters = {}, params = {}, field: string) {
function getFilterQuery(filters = {}, params = {}) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter === undefined || filter === FILTER_IGNORED) {
return arr;
}
if (key === field) {
return arr;
}
switch (key) {
case 'url':
arr.push(`and url_path = {${key}:String}`);
params[key] = filter;
break;
case 'pageTitle':
case 'os':
case 'browser':
case 'device':
case 'subdivision1':
case 'subdivision2':
case 'city':
case 'country':
arr.push(`and ${key} = {${key}:String}`);
params[key] = filter;
break;
case 'eventName':
arr.push(`and event_name = {${key}:String}`);
params[key] = filter;
break;
case 'referrer':
arr.push(`and referrer_domain= {${key}:String}`);
params[key] = filter;
break;
case 'domain':
arr.push(`and referrer_domain NOT ILIKE {${key}:String}`);
arr.push(`and referrer_domain NOT ILIKE '/%'`);
params[key] = `%://${filter}/%`;
break;
case 'query':
arr.push(`and url_query= {${key}:String}`);
params[key] = filter;
break;
if (filter !== undefined) {
const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column} = {${key}:String}`);
params[key] = decodeURIComponent(filter);
}
return arr;
@ -161,34 +121,9 @@ function getFilterQuery(filters = {}, params = {}, field: string) {
return query.join('\n');
}
function parseFilters(filters: any = {}, params: any = {}, field?: string) {
const {
domain,
url,
eventUrl,
referrer,
pageTitle,
os,
browser,
device,
country,
subdivision1,
subdivision2,
city,
eventName,
query,
} = filters;
const pageviewFilters = { domain, url, referrer, query, pageTitle };
const sessionFilters = { os, browser, device, country, subdivision1, subdivision2, city };
const eventFilters = { url: eventUrl, eventName };
function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
return {
pageviewFilters,
sessionFilters,
eventFilters,
event: { eventName },
filterQuery: getFilterQuery(filters, params, field),
filterQuery: getFilterQuery(filters, params),
};
}

View File

@ -27,12 +27,33 @@ export const UI_LAYOUT_BODY = 'ui-layout-body';
export const FILTER_COMBINED = 'filter-combined';
export const FILTER_RAW = 'filter-raw';
export const FILTER_IGNORED = 'filter-ignored';
export const FILTER_DAY = 'filter-day';
export const FILTER_RANGE = 'filter-range';
export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const EVENT_COLUMNS = ['url', 'referrer', 'title', 'query', 'event'];
export const SESSION_COLUMNS = [
'browser',
'os',
'device',
'screen',
'country',
'language',
'subdivision1',
'subdivision2',
'city',
];
export const FILTER_COLUMNS = {
url: 'url_path',
referrer: 'referrer_domain',
title: 'page_title',
query: 'url_query',
event: 'event_name',
};
export const EVENT_TYPE = {
pageView: 1,
customEvent: 2,

View File

@ -1,8 +1,8 @@
import prisma from '@umami/prisma-client';
import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { FILTER_IGNORED } from 'lib/constants';
import { getEventDataType } from './eventData';
import { FILTER_COLUMNS } from './constants';
const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',
@ -106,55 +106,14 @@ function getEventDataFilterQuery(
return query.join('\n');
}
function getFilterQuery(filters = {}, params = [], field: string): string {
function getFilterQuery(filters = {}, params = []): string {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter === undefined || filter === FILTER_IGNORED) {
return arr;
}
if (key === field) {
return arr;
}
switch (key) {
case 'url':
arr.push(`and url_path=$${params.length + 1}`);
if (filter !== undefined) {
const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
break;
case 'os':
case 'pageTitle':
case 'browser':
case 'device':
case 'subdivision1':
case 'subdivision2':
case 'city':
case 'country':
arr.push(`and ${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
break;
case 'eventName':
arr.push(`and event_name=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
break;
case 'referrer':
arr.push(`and referrer_domain=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
break;
case 'domain':
arr.push(`and referrer_domain not like $${params.length + 1}`);
arr.push(`and referrer_domain not like '/%'`);
params.push(`%://${filter}/%`);
break;
case 'query':
arr.push(`and url_query=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
break;
}
return arr;
@ -166,40 +125,16 @@ function getFilterQuery(filters = {}, params = [], field: string): string {
function parseFilters(
filters: { [key: string]: any } = {},
params = [],
field?: string,
sessionKey = 'session_id',
) {
const {
domain,
url,
eventUrl,
referrer,
pageTitle,
os,
browser,
device,
country,
subdivision1,
subdivision2,
city,
eventName,
query,
} = filters;
const pageviewFilters = { domain, url, referrer, query, pageTitle };
const sessionFilters = { os, browser, device, country, subdivision1, subdivision2, city };
const eventFilters = { url: eventUrl, eventName };
const { os, browser, device, country, subdivision1, subdivision2, city } = filters;
return {
pageviewFilters,
sessionFilters,
eventFilters,
event: { eventName },
joinSession:
os || browser || device || country || subdivision1 || subdivision2 || city
? `inner join session on website_event.${sessionKey} = session.${sessionKey}`
: '',
filterQuery: getFilterQuery(filters, params, field),
filterQuery: getFilterQuery(filters, params),
};
}

19
lib/query.ts Normal file
View File

@ -0,0 +1,19 @@
import cache from 'lib/cache';
import { getWebsite } from 'queries';
import { Website } from './types';
export async function loadWebsite(websiteId: string): Promise<Website> {
let website;
if (cache.enabled) {
website = await cache.fetchWebsite(websiteId);
} else {
website = await getWebsite({ id: websiteId });
}
if (!website || website.deletedAt) {
return null;
}
return website;
}

View File

@ -69,6 +69,22 @@ export interface WebsiteMetric {
y: number;
}
export interface WebsiteMetricFilter {
domain?: string;
url?: string;
referrer?: string;
title?: string;
query?: string;
event?: string;
os?: string;
browser?: string;
device?: string;
country?: string;
subdivision1?: string;
subdivision2?: string;
city?: string;
}
export interface WebsiteEventMetric {
x: string;
t: string;

View File

@ -1,56 +1,10 @@
import { NextApiResponse } from 'next';
import { methodNotAllowed, ok, unauthorized } from 'next-basics';
import { WebsiteMetric, NextApiRequestQueryBody } from 'lib/types';
import { canViewWebsite } from 'lib/auth';
import { FILTER_IGNORED } from 'lib/constants';
import { useAuth, useCors } from 'lib/middleware';
import { NextApiResponse } from 'next';
import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';
import { getPageviewMetrics, getSessionMetrics, getWebsite } from 'queries';
const sessionColumns = [
'browser',
'os',
'device',
'screen',
'country',
'language',
'subdivision1',
'subdivision2',
'city',
];
const pageviewColumns = ['url', 'referrer', 'query', 'title'];
function getTable(type) {
if (type === 'event') {
return 'event';
}
if (sessionColumns.includes(type)) {
return 'session';
}
if (pageviewColumns.includes(type)) {
return 'pageview';
}
throw new Error('Invalid type');
}
function getColumn(type) {
switch (type) {
case 'url':
return 'url_path';
case 'referrer':
return 'referrer_domain';
case 'event':
return 'event_name';
case 'query':
return 'url_query';
case 'title':
return 'page_title';
}
return type;
}
import { SESSION_COLUMNS, EVENT_COLUMNS, FILTER_COLUMNS } from 'lib/constants';
import { getPageviewMetrics, getSessionMetrics } from 'queries';
export interface WebsiteMetricsRequestQuery {
id: string;
@ -60,6 +14,8 @@ export interface WebsiteMetricsRequestQuery {
url: string;
referrer: string;
title: string;
query: string;
event: string;
os: string;
browser: string;
device: string;
@ -84,6 +40,8 @@ export default async (
url,
referrer,
title,
query,
event,
os,
browser,
device,
@ -91,7 +49,6 @@ export default async (
subdivision1,
subdivision2,
city,
query,
} = req.query;
if (req.method === 'GET') {
@ -102,12 +59,9 @@ export default async (
const startDate = new Date(+startAt);
const endDate = new Date(+endAt);
if (sessionColumns.includes(type)) {
let data = await getSessionMetrics(websiteId, {
startDate,
endDate,
field: type,
filters: {
if (SESSION_COLUMNS.includes(type)) {
const column = FILTER_COLUMNS[type] || type;
const filters = {
os,
browser,
device,
@ -115,7 +69,15 @@ export default async (
subdivision1,
subdivision2,
city,
},
};
filters[type] = undefined;
let data = await getSessionMetrics(websiteId, {
startDate,
endDate,
column,
filters,
});
if (type === 'language') {
@ -124,8 +86,8 @@ export default async (
for (const { x, y } of data) {
const key = String(x).toLowerCase().split('-')[0];
if (!combined[key]) {
combined[key] = { x, y };
if (combined[key] === undefined) {
combined[key] = { x: key, y };
} else {
combined[key].y += y;
}
@ -137,43 +99,30 @@ export default async (
return ok(res, data);
}
if (pageviewColumns.includes(type) || type === 'event') {
let domain;
if (type === 'referrer') {
const website = await getWebsite({ id: websiteId });
if (!website) {
return badRequest(res);
}
domain = website.domain;
}
const column = getColumn(type);
const table = getTable(type);
if (EVENT_COLUMNS.includes(type)) {
const column = FILTER_COLUMNS[type] || type;
const filters = {
domain,
url: type !== 'url' && table !== 'event' ? url : undefined,
referrer: type !== 'referrer' && table !== 'event' ? referrer : FILTER_IGNORED,
title: type !== 'title' && table !== 'event' ? title : undefined,
os: type !== 'os' ? os : undefined,
browser: type !== 'browser' ? browser : undefined,
device: type !== 'device' ? device : undefined,
country: type !== 'country' ? country : undefined,
subdivision1: type !== 'subdivision1' ? subdivision1 : undefined,
subdivision2: type !== 'subdivision2' ? subdivision2 : undefined,
city: type !== 'city' ? city : undefined,
eventUrl: type !== 'url' && table === 'event' ? url : undefined,
query: type !== 'query' && table !== 'event' ? query : undefined,
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
subdivision1,
subdivision2,
city,
};
filters[type] = undefined;
const data = await getPageviewMetrics(websiteId, {
startDate,
endDate,
column,
filters,
type,
});
return ok(res, data);

View File

@ -12,7 +12,9 @@ export interface WebsiteStatsRequestQuery {
endAt: number;
url: string;
referrer: string;
pageTitle: string;
title: string;
query: string;
event: string;
os: string;
browser: string;
device: string;
@ -35,7 +37,9 @@ export default async (
endAt,
url,
referrer,
pageTitle,
title,
query,
event,
os,
browser,
device,
@ -63,7 +67,9 @@ export default async (
filters: {
url,
referrer,
pageTitle,
title,
query,
event,
os,
browser,
device,
@ -79,7 +85,9 @@ export default async (
filters: {
url,
referrer,
pageTitle,
title,
query,
event,
os,
browser,
device,

View File

@ -14,6 +14,7 @@ export async function getEventMetrics(
endDate: Date;
timezone: string;
unit: string;
column: string;
filters: {
url: string;
eventName: string;
@ -40,6 +41,7 @@ async function relationalQuery(
endDate: Date;
timezone: string;
unit: string;
column: string;
filters: {
url: string;
eventName: string;
@ -50,6 +52,7 @@ async function relationalQuery(
const website = await getWebsite({ id: websiteId });
const resetDate = website?.resetAt || website?.createdAt;
const params: any = [websiteId, resetDate, startDate, endDate];
const filterQuery = getFilterQuery(filters, params);
return rawQuery(
`select
@ -61,7 +64,7 @@ async function relationalQuery(
and created_at >= $2
and created_at between $3 and $4
and event_type = ${EVENT_TYPE.customEvent}
${getFilterQuery(filters, params)}
${filterQuery}
group by 1, 2
order by 2`,
params,

View File

@ -1,20 +1,17 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import cache from 'lib/cache';
import { Prisma } from '@prisma/client';
import { EVENT_TYPE } from 'lib/constants';
import { getWebsite } from 'queries';
import { EVENT_TYPE, FILTER_COLUMNS } from 'lib/constants';
import { loadWebsite } from 'lib/query';
export async function getPageviewMetrics(
...args: [
websiteId: string,
data: {
criteria: {
startDate: Date;
endDate: Date;
column: Prisma.WebsiteEventScalarFieldEnum | Prisma.SessionScalarFieldEnum;
column: string;
filters: object;
type: string;
},
]
) {
@ -26,25 +23,32 @@ export async function getPageviewMetrics(
async function relationalQuery(
websiteId: string,
data: {
criteria: {
startDate: Date;
endDate: Date;
column: Prisma.WebsiteEventScalarFieldEnum | Prisma.SessionScalarFieldEnum;
column: string;
filters: object;
type: string;
},
) {
const { startDate, endDate, column, filters = {}, type } = data;
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters, toUuid } = prisma;
const website = await getWebsite({ id: websiteId });
const website = await loadWebsite(websiteId);
const resetDate = website?.resetAt || website?.createdAt;
const params: any = [
websiteId,
resetDate,
startDate,
endDate,
type === 'event' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
];
let domainFilter = '';
if (column === 'referrer_domain') {
domainFilter = 'and website_event.referrer_domain != $6';
params.push(website.domain);
}
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery(
@ -55,6 +59,7 @@ async function relationalQuery(
and website_event.created_at >= $2
and website_event.created_at between $3 and $4
and event_type = $5
${domainFilter}
${filterQuery}
group by 1
order by 2 desc
@ -65,22 +70,30 @@ async function relationalQuery(
async function clickhouseQuery(
websiteId: string,
data: {
criteria: {
startDate: Date;
endDate: Date;
column: Prisma.WebsiteEventScalarFieldEnum | Prisma.SessionScalarFieldEnum;
column: string;
filters: object;
type: string;
},
) {
const { startDate, endDate, column, filters = {}, type } = data;
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, getDateFormat, parseFilters, getBetweenDates } = clickhouse;
const website = await cache.fetchWebsite(websiteId);
const website = await loadWebsite(websiteId);
const resetDate = website?.resetAt || website?.createdAt;
const params = {
websiteId,
eventType: type === 'event' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: undefined,
};
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = 'and referrer_domain != {domain:String}';
params.domain = website.domain;
}
const { filterQuery } = parseFilters(filters, params);
return rawQuery(
@ -90,6 +103,7 @@ async function clickhouseQuery(
and event_type = {eventType:UInt32}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
${excludeDomain}
${filterQuery}
group by x
order by y desc

View File

@ -8,7 +8,7 @@ import { getWebsite } from 'queries';
export async function getPageviewStats(
...args: [
websiteId: string,
data: {
criteria: {
startDate: Date;
endDate: Date;
timezone?: string;
@ -27,7 +27,7 @@ export async function getPageviewStats(
async function relationalQuery(
websiteId: string,
data: {
criteria: {
startDate: Date;
endDate: Date;
timezone?: string;
@ -45,7 +45,7 @@ async function relationalQuery(
count = '*',
filters = {},
sessionKey = 'session_id',
} = data;
} = criteria;
const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma;
const website = await getWebsite({ id: websiteId });
const resetDate = website?.resetAt || website?.createdAt;
@ -69,7 +69,7 @@ async function relationalQuery(
async function clickhouseQuery(
websiteId: string,
data: {
criteria: {
startDate: Date;
endDate: Date;
timezone?: string;
@ -79,7 +79,14 @@ async function clickhouseQuery(
sessionKey?: string;
},
) {
const { startDate, endDate, timezone = 'UTC', unit = 'day', count = '*', filters = {} } = data;
const {
startDate,
endDate,
timezone = 'UTC',
unit = 'day',
count = '*',
filters = {},
} = criteria;
const {
parseFilters,
getDateFormat,

View File

@ -2,13 +2,13 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import cache from 'lib/cache';
import { EVENT_TYPE } from 'lib/constants';
import { EVENT_TYPE, FILTER_COLUMNS } from 'lib/constants';
import { getWebsite } from 'queries';
export async function getSessionMetrics(
...args: [
websiteId: string,
data: { startDate: Date; endDate: Date; field: string; filters: object },
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
]
) {
return runQuery({
@ -19,17 +19,17 @@ export async function getSessionMetrics(
async function relationalQuery(
websiteId: string,
data: { startDate: Date; endDate: Date; field: string; filters: object },
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const website = await getWebsite({ id: websiteId });
const resetDate = website?.resetAt || website?.createdAt;
const { startDate, endDate, field, filters = {} } = data;
const { startDate, endDate, column, filters = {} } = criteria;
const { toUuid, parseFilters, rawQuery } = prisma;
const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery(
`select ${field} x, count(*) y
`select ${column} x, count(*) y
from session as x
where x.session_id in (
select website_event.session_id
@ -51,17 +51,17 @@ async function relationalQuery(
async function clickhouseQuery(
websiteId: string,
data: { startDate: Date; endDate: Date; field: string; filters: object },
data: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const { startDate, endDate, field, filters = {} } = data;
const { startDate, endDate, column, filters = {} } = data;
const { getDateFormat, parseFilters, getBetweenDates, rawQuery } = clickhouse;
const website = await cache.fetchWebsite(websiteId);
const resetDate = website?.resetAt || website?.createdAt;
const params = { websiteId };
const { filterQuery } = parseFilters(filters, params, field);
const { filterQuery } = parseFilters(filters, params);
return rawQuery(
`select ${field} x, count(distinct session_id) y
`select ${column} x, count(distinct session_id) y
from website_event as x
where website_id = {websiteId:UUID}
and event_type = ${EVENT_TYPE.pageView}

View File

@ -6,7 +6,10 @@ import { EVENT_TYPE } from 'lib/constants';
import { getWebsite } from 'queries';
export async function getWebsiteStats(
...args: [websiteId: string, data: { startDate: Date; endDate: Date; filters: object }]
...args: [
websiteId: string,
data: { startDate: Date; endDate: Date; type?: string; filters: object },
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -16,9 +19,9 @@ export async function getWebsiteStats(
async function relationalQuery(
websiteId: string,
data: { startDate: Date; endDate: Date; filters: object },
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = data;
const { startDate, endDate, filters = {} } = criteria;
const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;
const website = await getWebsite({ id: websiteId });
const resetDate = website?.resetAt || website?.createdAt;
@ -51,9 +54,9 @@ async function relationalQuery(
async function clickhouseQuery(
websiteId: string,
data: { startDate: Date; endDate: Date; filters: object },
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = data;
const { startDate, endDate, filters = {} } = criteria;
const { rawQuery, getDateFormat, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
const website = await cache.fetchWebsite(websiteId);
const resetDate = website?.resetAt || website?.createdAt;