Merge branch 'dev' of https://github.com/umami-software/umami into feat/um-366-event-data-migration

This commit is contained in:
Francis Cao 2023-07-11 12:05:05 -07:00
commit 3aa7565bf4
17 changed files with 639 additions and 699 deletions

View File

@ -1,13 +1,27 @@
import { Loading } from 'react-basics'; import { useState } from 'react';
import { Loading, cloneChildren } from 'react-basics';
import ErrorMessage from 'components/common/ErrorMessage'; import ErrorMessage from 'components/common/ErrorMessage';
import styles from './MetricsBar.module.css'; import styles from './MetricsBar.module.css';
import { formatLongNumber, formatNumber } from 'lib/format';
export function MetricsBar({ children, isLoading, isFetched, error }) {
const [format, setFormat] = useState(true);
const formatFunc = format
? n => (n >= 0 ? formatLongNumber(n) : `-${formatLongNumber(Math.abs(n))}`)
: formatNumber;
const handleSetFormat = () => {
setFormat(state => !state);
};
export function MetricsBar({ children, onClick, isLoading, isFetched, error }) {
return ( return (
<div className={styles.bar} onClick={onClick}> <div className={styles.bar} onClick={handleSetFormat}>
{isLoading && !isFetched && <Loading icon="dots" />} {isLoading && !isFetched && <Loading icon="dots" />}
{error && <ErrorMessage />} {error && <ErrorMessage />}
{children} {cloneChildren(children, child => {
return { format: child.props.format || formatFunc };
})}
</div> </div>
); );
} }

View File

@ -13,9 +13,9 @@ export function EventDataMetricsBar({ websiteId }) {
const { startDate, endDate, modified } = dateRange; const { startDate, endDate, modified } = dateRange;
const { data, error, isLoading, isFetched } = useQuery( const { data, error, isLoading, isFetched } = useQuery(
['event-data:fields', { websiteId, startDate, endDate, modified }], ['event-data:stats', { websiteId, startDate, endDate, modified }],
() => () =>
get(`/event-data/fields`, { get(`/event-data/stats`, {
websiteId, websiteId,
startAt: +startDate, startAt: +startDate,
endAt: +endDate, endAt: +endDate,
@ -27,11 +27,18 @@ export function EventDataMetricsBar({ websiteId }) {
<Column defaultSize={12} xl={8}> <Column defaultSize={12} xl={8}>
<MetricsBar isLoading={isLoading} isFetched={isFetched} error={error}> <MetricsBar isLoading={isLoading} isFetched={isFetched} error={error}>
{!error && isFetched && ( {!error && isFetched && (
<MetricCard <>
className={styles.card} <MetricCard
label={formatMessage(labels.fields)} className={styles.card}
value={data?.length} label={formatMessage(labels.fields)}
/> value={data?.fields}
/>
<MetricCard
className={styles.card}
label={formatMessage(labels.totalRecords)}
value={data?.records}
/>
</>
)} )}
</MetricsBar> </MetricsBar>
</Column> </Column>

View File

@ -3,7 +3,7 @@ import { GridTable, GridColumn } from 'react-basics';
import { useMessages, usePageQuery } from 'hooks'; import { useMessages, usePageQuery } from 'hooks';
import Empty from 'components/common/Empty'; import Empty from 'components/common/Empty';
export function EventDataTable({ data = [], showValue }) { export function EventDataTable({ data = [] }) {
const { formatMessage, labels } = useMessages(); const { formatMessage, labels } = useMessages();
const { resolveUrl } = usePageQuery(); const { resolveUrl } = usePageQuery();

View File

@ -3,6 +3,7 @@ import EventDataTable from 'components/pages/event-data/EventDataTable';
import EventDataValueTable from 'components/pages/event-data/EventDataValueTable'; import EventDataValueTable from 'components/pages/event-data/EventDataValueTable';
import { EventDataMetricsBar } from 'components/pages/event-data/EventDataMetricsBar'; import { EventDataMetricsBar } from 'components/pages/event-data/EventDataMetricsBar';
import { useDateRange, useApi, usePageQuery } from 'hooks'; import { useDateRange, useApi, usePageQuery } from 'hooks';
import styles from './WebsiteEventData.module.css';
function useFields(websiteId, field) { function useFields(websiteId, field) {
const [dateRange] = useDateRange(websiteId); const [dateRange] = useDateRange(websiteId);
@ -11,7 +12,7 @@ function useFields(websiteId, field) {
const { data, error, isLoading } = useQuery( const { data, error, isLoading } = useQuery(
['event-data:fields', { websiteId, startDate, endDate, field }], ['event-data:fields', { websiteId, startDate, endDate, field }],
() => () =>
get('/event-data', { get('/event-data/fields', {
websiteId, websiteId,
startAt: +startDate, startAt: +startDate,
endAt: +endDate, endAt: +endDate,
@ -30,7 +31,7 @@ export default function WebsiteEventData({ websiteId }) {
const { data } = useFields(websiteId, view); const { data } = useFields(websiteId, view);
return ( return (
<Flexbox direction="column" gap={20}> <Flexbox className={styles.container} direction="column" gap={20}>
<EventDataMetricsBar websiteId={websiteId} /> <EventDataMetricsBar websiteId={websiteId} />
{!view && <EventDataTable data={data} />} {!view && <EventDataTable data={data} />}
{view && <EventDataValueTable field={view} data={data} />} {view && <EventDataValueTable field={view} data={data} />}

View File

@ -1,4 +1,7 @@
.container { .container a {
display: flex; color: var(--font-color100);
flex-direction: column; }
.container a:hover {
color: var(--primary400);
} }

View File

@ -48,10 +48,7 @@ export function WebsiteHeader({ websiteId, showLinks = true, children }) {
{showLinks && ( {showLinks && (
<Flexbox alignItems="center"> <Flexbox alignItems="center">
{links.map(({ label, icon, path }) => { {links.map(({ label, icon, path }) => {
const query = path.indexOf('?'); const selected = path ? pathname.endsWith(path) : pathname === '/websites/[id]';
const selected = path
? asPath.endsWith(query >= 0 ? path.substring(0, query) : path)
: pathname === '/websites/[id]';
return ( return (
<Link key={label} href={`/websites/${websiteId}${path}`} shallow={true}> <Link key={label} href={`/websites/${websiteId}${path}`} shallow={true}>

View File

@ -14,7 +14,6 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
const { get, useQuery } = useApi(); const { get, useQuery } = useApi();
const [dateRange] = useDateRange(websiteId); const [dateRange] = useDateRange(websiteId);
const { startDate, endDate, modified } = dateRange; const { startDate, endDate, modified } = dateRange;
const [format, setFormat] = useState(true);
const { ref, isSticky } = useSticky({ enabled: sticky }); const { ref, isSticky } = useSticky({ enabled: sticky });
const { const {
query: { url, referrer, title, os, browser, device, country, region, city }, query: { url, referrer, title, os, browser, device, country, region, city },
@ -41,14 +40,6 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
}), }),
); );
const formatFunc = format
? n => (n >= 0 ? formatLongNumber(n) : `-${formatLongNumber(Math.abs(n))}`)
: formatNumber;
function handleSetFormat() {
setFormat(state => !state);
}
const { pageviews, uniques, bounces, totaltime } = data || {}; const { pageviews, uniques, bounces, totaltime } = data || {};
const num = Math.min(data && uniques.value, data && bounces.value); const num = Math.min(data && uniques.value, data && bounces.value);
const diffs = data && { const diffs = data && {
@ -67,12 +58,7 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
})} })}
> >
<Column defaultSize={12} xl={8}> <Column defaultSize={12} xl={8}>
<MetricsBar <MetricsBar isLoading={isLoading} isFetched={isFetched} error={error}>
isLoading={isLoading}
isFetched={isFetched}
error={error}
onClick={handleSetFormat}
>
{!error && isFetched && ( {!error && isFetched && (
<> <>
<MetricCard <MetricCard
@ -80,14 +66,12 @@ export function WebsiteMetricsBar({ websiteId, sticky }) {
label={formatMessage(labels.views)} label={formatMessage(labels.views)}
value={pageviews.value} value={pageviews.value}
change={pageviews.change} change={pageviews.change}
format={formatFunc}
/> />
<MetricCard <MetricCard
className={styles.card} className={styles.card}
label={formatMessage(labels.visitors)} label={formatMessage(labels.visitors)}
value={uniques.value} value={uniques.value}
change={uniques.change} change={uniques.change}
format={formatFunc}
/> />
<MetricCard <MetricCard
className={styles.card} className={styles.card}

View File

@ -11,10 +11,8 @@ ALTER TABLE "event_data" ADD COLUMN "job_id" UUID AFTER "created_at";
-- update event_data string -- update event_data string
alter table umami.event_data alter table umami.event_data
update string_value = number_value update string_value = number_value
where number_value is not null where data_type = 2
and string_value is null;
alter table umami.event_data alter table umami.event_data
update string_value = replaceOne(concat(CAST(toDateTime(date_value, 'UTC'), 'String'),'Z'), ' ', 'T') update string_value = replaceOne(concat(CAST(toDateTime(date_value, 'UTC'), 'String'),'Z'), ' ', 'T')
where date_value is not null where data_type = 4
and string_value is null;

View File

@ -94,11 +94,17 @@ export interface WebsiteEventMetric {
y: number; y: number;
} }
export interface WebsiteEventDataMetric { export interface WebsiteEventDataStats {
x: string; field: string;
t: string; type: number;
eventName?: string; total: number;
urlPath?: string; }
export interface WebsiteEventDataFields {
field: string;
type: number;
value?: string;
total: number;
} }
export interface WebsitePageviews { export interface WebsitePageviews {

View File

@ -79,7 +79,6 @@
"del": "^6.0.0", "del": "^6.0.0",
"detect-browser": "^5.2.0", "detect-browser": "^5.2.0",
"dotenv": "^10.0.0", "dotenv": "^10.0.0",
"formik": "^2.2.9",
"fs-extra": "^10.0.1", "fs-extra": "^10.0.1",
"immer": "^9.0.12", "immer": "^9.0.12",
"ipaddr.js": "^2.0.1", "ipaddr.js": "^2.0.1",

View File

@ -21,13 +21,13 @@ export default async (
await useAuth(req, res); await useAuth(req, res);
if (req.method === 'GET') { if (req.method === 'GET') {
const { websiteId, startAt, endAt } = req.query; const { websiteId, startAt, endAt, field } = req.query;
if (!(await canViewWebsite(req.auth, websiteId))) { if (!(await canViewWebsite(req.auth, websiteId))) {
return unauthorized(res); return unauthorized(res);
} }
const data = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt)); const data = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt), field);
return ok(res, data); return ok(res, data);
} }

View File

@ -3,7 +3,7 @@ import { useCors, useAuth } from 'lib/middleware';
import { NextApiRequestQueryBody } from 'lib/types'; import { NextApiRequestQueryBody } from 'lib/types';
import { NextApiResponse } from 'next'; import { NextApiResponse } from 'next';
import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { ok, methodNotAllowed, unauthorized } from 'next-basics';
import { getEventData } from 'queries'; import { getEventDataFields } from 'queries';
export interface EventDataRequestBody { export interface EventDataRequestBody {
websiteId: string; websiteId: string;
@ -22,13 +22,21 @@ export default async (
await useAuth(req, res); await useAuth(req, res);
if (req.method === 'GET') { if (req.method === 'GET') {
const { websiteId, startAt, endAt, field } = req.query; const { websiteId, startAt, endAt } = req.query;
if (!(await canViewWebsite(req.auth, websiteId))) { if (!(await canViewWebsite(req.auth, websiteId))) {
return unauthorized(res); return unauthorized(res);
} }
const data = await getEventData(websiteId, new Date(+startAt), new Date(+endAt), field); const results = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt));
const data = results.reduce(
(obj, row) => {
obj.records += row.total;
return obj;
},
{ fields: results.length, records: 0 },
);
return ok(res, data); return ok(res, data);
} }

View File

@ -1,94 +0,0 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataMetric } from 'lib/types';
import { loadWebsite } from 'lib/query';
import { DEFAULT_CREATED_AT } from 'lib/constants';
export async function getEventData(
...args: [websiteId: string, startDate: Date, endDate: Date, field?: string]
): Promise<WebsiteEventDataMetric[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { toUuid, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
if (field) {
return rawQuery(
`select event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = $1${toUuid()}
and event_key = $2
and created_at >= $3
and created_at between $4 and $5
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
[websiteId, field, resetDate, startDate, endDate] as any,
);
}
return rawQuery(
`select
event_key as field,
count(*) as total
from event_data
where website_id = $1${toUuid()}
and created_at >= $2
and created_at between $3 and $4
group by event_key
order by 2 desc, 1 asc
limit 100
`,
[websiteId, resetDate, startDate, endDate] as any,
);
}
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery, getDateFormat, getBetweenDates } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
if (field) {
return rawQuery(
`select
event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and event_key = {field:String}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field },
);
}
return rawQuery(
`select
event_key as field,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
group by event_key
order by 2 desc, 1 asc
limit 100
`,
{ websiteId },
);
}

View File

@ -1,51 +1,96 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma'; import { WebsiteEventDataFields } from 'lib/types';
import { WebsiteEventDataMetric } from 'lib/types';
import { loadWebsite } from 'lib/query'; import { loadWebsite } from 'lib/query';
import { DEFAULT_CREATED_AT } from 'lib/constants'; import { DEFAULT_CREATED_AT } from 'lib/constants';
export async function getEventDataFields( export async function getEventDataFields(
...args: [websiteId: string, startDate: Date, endDate: Date] ...args: [websiteId: string, startDate: Date, endDate: Date, field?: string]
): Promise<WebsiteEventDataMetric[]> { ): Promise<WebsiteEventDataFields[]> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
}); });
} }
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date) { async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { toUuid, rawQuery } = prisma; const { toUuid, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT); const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params: any = [websiteId, resetDate, startDate, endDate];
if (field) {
return rawQuery(
`select event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = $1${toUuid()}
and event_key = $2
and created_at >= $3
and created_at between $4 and $5
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
[websiteId, field, resetDate, startDate, endDate] as any,
);
}
return rawQuery( return rawQuery(
`select `select
distinct event_key as eventKey, data_type as eventDataType event_key as field,
from event_data data_type as type,
where website_id = $1${toUuid()} count(*) as total
and created_at >= $2 from event_data
and created_at between $3 and $4 where website_id = $1${toUuid()}
order by event_key asc`, and created_at >= $2
params, and created_at between $3 and $4
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
limit 100
`,
[websiteId, resetDate, startDate, endDate] as any,
); );
} }
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date) { async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery, getDateFormat, getBetweenDates } = clickhouse; const { rawQuery, getDateFormat, getBetweenDates } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT); const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = { websiteId };
if (field) {
return rawQuery(
`select
event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and event_key = {field:String}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field },
);
}
return rawQuery( return rawQuery(
`select `select
distinct event_key as eventKey, data_type as eventDataType event_key as field,
from event_data data_type as type,
where website_id = {websiteId:UUID} count(*) as total
and created_at >= ${getDateFormat(resetDate)} from event_data
and ${getBetweenDates('created_at', startDate, endDate)} where website_id = {websiteId:UUID}
order by event_key asc`, and created_at >= ${getDateFormat(resetDate)}
params, and ${getBetweenDates('created_at', startDate, endDate)}
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId },
); );
} }

View File

@ -91,7 +91,7 @@ async function clickhouseQuery(
count(*) AS count count(*) AS count
FROM ( FROM (
SELECT session_id, SELECT session_id,
windowFunnel({window:UInt32}) windowFunnel({window:UInt32}, 'strict_order')
( (
created_at created_at
${columnsQuery} ${columnsQuery}

View File

@ -6,7 +6,6 @@ export * from './admin/website';
export * from './analytics/event/getEventMetrics'; export * from './analytics/event/getEventMetrics';
export * from './analytics/event/getEventUsage'; export * from './analytics/event/getEventUsage';
export * from './analytics/event/getEvents'; export * from './analytics/event/getEvents';
export * from './analytics/eventData/getEventData';
export * from './analytics/eventData/getEventDataFields'; export * from './analytics/eventData/getEventDataFields';
export * from './analytics/eventData/getEventDataUsage'; export * from './analytics/eventData/getEventDataUsage';
export * from './analytics/event/saveEvent'; export * from './analytics/event/saveEvent';

1029
yarn.lock

File diff suppressed because it is too large Load Diff