m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2025-02-14 21:10:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactored realtime.

Browse Source
This commit is contained in:
Mike Cao 2024-06-19 21:47:27 -07:00
parent cda3ba345b
commit 5108b91f80
16 changed files with 205 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/app/(main)/websites/[websiteId]/realtime/RealtimeHeader.tsx
View File

@ -5,7 +5,7 @@ import styles from './RealtimeHeader.module.css';
export function RealtimeHeader({ data }: { data: RealtimeData }) {
const { formatMessage, labels } = useMessages();
const { pageviews, visitors, events, countries } = data || {};
const { totals }: any = data || {};
return (
<div className={styles.header}>
@ -13,22 +13,22 @@ export function RealtimeHeader({ data }: { data: RealtimeData }) {
<MetricCard
className={styles.card}
label={formatMessage(labels.views)}
value={pageviews?.length}
value={totals.views}
/>
<MetricCard
className={styles.card}
label={formatMessage(labels.visitors)}
value={visitors?.length}
value={totals.visitors}
/>
<MetricCard
className={styles.card}
label={formatMessage(labels.events)}
value={events?.length}
value={totals.events}
/>
<MetricCard
className={styles.card}
label={formatMessage(labels.countries)}
value={countries?.length}
value={totals.countries}
/>
</div>
</div>

24
src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx
View File

@ -54,20 +54,20 @@ export function RealtimeLog({ data }: { data: RealtimeData }) {
},
];
const getTime = ({ timestamp }) => format(timestamp, 'h:mm:ss');
const getTime = ({ timestamp }) => format(timestamp * 1000, 'h:mm:ss');
const getColor = ({ id, sessionId }) => stringToColor(sessionId || id);
const getIcon = ({ __type }) => icons[__type];
const getDetail = (log: {
__type: any;
eventName: any;
urlPath: any;
browser: any;
os: any;
country: any;
device: any;
__type: string;
eventName: string;
urlPath: string;
browser: string;
os: string;
country: string;
device: string;
}) => {
const { __type, eventName, urlPath: url, browser, os, country, device } = log;
@ -141,8 +141,12 @@ export function RealtimeLog({ data }: { data: RealtimeData }) {
return [];
}
const { pageviews, visitors, events } = data;
let logs = [...pageviews, ...visitors, ...events].sort(thenby.firstBy('createdAt', -1));
const { events, visitors } = data;
let logs = [
...events.map(e => ({ __type: e.eventName ? TYPE_EVENT : TYPE_PAGEVIEW, ...e })),
...visitors.map(v => ({ __type: TYPE_SESSION, ...v })),
].sort(thenby.firstBy('timestamp', -1));
if (search) {
logs = logs.filter(({ eventName, urlPath, browser, os, country, device }) => {

68
src/app/(main)/websites/[websiteId]/realtime/RealtimeUrls.tsx
View File

@ -1,4 +1,4 @@
import { Key, useContext, useMemo, useState } from 'react';
import { Key, useContext, useState } from 'react';
import { ButtonGroup, Button, Flexbox } from 'react-basics';
import thenby from 'thenby';
import { percentFilter } from 'lib/filters';
@ -11,7 +11,7 @@ import { WebsiteContext } from '../WebsiteProvider';
export function RealtimeUrls({ data }: { data: RealtimeData }) {
const website = useContext(WebsiteContext);
const { formatMessage, labels } = useMessages();
const { pageviews } = data || {};
const { referrers, urls } = data || {};
const [filter, setFilter] = useState<Key>(FILTER_REFERRERS);
const limit = 15;
@ -35,47 +35,29 @@ export function RealtimeUrls({ data }: { data: RealtimeData }) {
);
};
const [referrers = [], pages = []] = useMemo(() => {
if (pageviews) {
const referrers = percentFilter(
pageviews
.reduce((arr, { referrerDomain }) => {
if (referrerDomain) {
const row = arr.find(({ x }) => x === referrerDomain);
const domains = percentFilter(
Object.keys(referrers)
.map(key => {
return {
x: key,
y: referrers[key],
};
})
.sort(thenby.firstBy('y', -1))
.slice(0, limit),
);
if (!row) {
arr.push({ x: referrerDomain, y: 1 });
} else {
row.y += 1;
}
}
return arr;
}, [])
.sort(thenby.firstBy('y', -1))
.slice(0, limit),
);
const pages = percentFilter(
pageviews
.reduce((arr, { urlPath }) => {
const row = arr.find(({ x }) => x === urlPath);
if (!row) {
arr.push({ x: urlPath, y: 1 });
} else {
row.y += 1;
}
return arr;
}, [])
.sort(thenby.firstBy('y', -1))
.slice(0, limit),
);
return [referrers, pages];
}
return [];
}, [pageviews]);
const pages = percentFilter(
Object.keys(urls)
.map(key => {
return {
x: key,
y: urls[key],
};
})
.sort(thenby.firstBy('y', -1))
.slice(0, limit),
);
return (
<>
@ -89,7 +71,7 @@ export function RealtimeUrls({ data }: { data: RealtimeData }) {
title={formatMessage(labels.referrers)}
metric={formatMessage(labels.views)}
renderLabel={renderLink}
data={referrers}
data={domains}
/>
)}
{filter === FILTER_PAGES && (

12
src/app/(main)/websites/[websiteId]/realtime/WebsiteRealtimePage.tsx
View File

@ -1,4 +1,5 @@
'use client';
import { firstBy } from 'thenby';
import { Grid, GridRow } from 'components/layout/Grid';
import Page from 'components/layout/Page';
import RealtimeChart from 'components/metrics/RealtimeChart';
@ -10,6 +11,7 @@ import RealtimeUrls from './RealtimeUrls';
import RealtimeCountries from './RealtimeCountries';
import WebsiteHeader from '../WebsiteHeader';
import WebsiteProvider from '../WebsiteProvider';
import { percentFilter } from 'lib/filters';
export function WebsiteRealtimePage({ websiteId }) {
const { data, isLoading, error } = useRealtime(websiteId);
@ -18,6 +20,12 @@ export function WebsiteRealtimePage({ websiteId }) {
return <Page isLoading={isLoading} error={error} />;
}
const countries = percentFilter(
Object.keys(data.countries)
.map(key => ({ x: key, y: data.countries[key] }))
.sort(firstBy('y', -1)),
);
return (
<WebsiteProvider websiteId={websiteId}>
<WebsiteHeader websiteId={websiteId} />
@ -29,8 +37,8 @@ export function WebsiteRealtimePage({ websiteId }) {
<RealtimeLog data={data} />
</GridRow>
<GridRow columns="one-two">
<RealtimeCountries data={data?.countries} />
<WorldMap data={data?.countries} />
<RealtimeCountries data={countries} />
<WorldMap data={countries} />
</GridRow>
</Grid>
</WebsiteProvider>

76
src/components/hooks/queries/useRealtime.ts
View File

@ -1,87 +1,21 @@
import { useMemo, useRef } from 'react';
import { RealtimeData } from 'lib/types';
import { useApi } from './useApi';
import { REALTIME_INTERVAL, REALTIME_RANGE } from 'lib/constants';
import { startOfMinute, subMinutes } from 'date-fns';
import { percentFilter } from 'lib/filters';
import thenby from 'thenby';
function mergeData(state = [], data = [], time: number) {
const ids = state.map(({ id }) => id);
return state
.concat(data.filter(({ id }) => !ids.includes(id)))
.filter(({ timestamp }) => timestamp >= time);
}
import { REALTIME_INTERVAL } from 'lib/constants';
import { useTimezone } from 'components/hooks';
export function useRealtime(websiteId: string) {
const currentData = useRef({
pageviews: [],
sessions: [],
events: [],
countries: [],
visitors: [],
timestamp: 0,
});
const { get, useQuery } = useApi();
const { timezone } = useTimezone();
const { data, isLoading, error } = useQuery<RealtimeData>({
queryKey: ['realtime', websiteId],
queryFn: async () => {
const state = currentData.current;
const data = await get(`/realtime/${websiteId}`, { startAt: state?.timestamp || 0 });
const date = subMinutes(startOfMinute(new Date()), REALTIME_RANGE);
const time = date.getTime();
const { pageviews, sessions, events, timestamp } = data;
return {
pageviews: mergeData(state?.pageviews, pageviews, time),
sessions: mergeData(state?.sessions, sessions, time),
events: mergeData(state?.events, events, time),
timestamp,
};
return get(`/realtime/${websiteId}`, { timezone });
},
enabled: !!websiteId,
refetchInterval: REALTIME_INTERVAL,
});
const realtimeData: RealtimeData = useMemo(() => {
if (!data) {
return { pageviews: [], sessions: [], events: [], countries: [], visitors: [], timestamp: 0 };
}
data.countries = percentFilter(
data.sessions
.reduce((arr, data) => {
if (!arr.find(({ id }) => id === data.id)) {
return arr.concat(data);
}
return arr;
}, [])
.reduce((arr: { x: any; y: number }[], { country }: any) => {
if (country) {
const row = arr.find(({ x }) => x === country);
if (!row) {
arr.push({ x: country, y: 1 });
} else {
row.y += 1;
}
}
return arr;
}, [])
.sort(thenby.firstBy('y', -1)),
);
data.visitors = data.sessions.reduce((arr, val) => {
if (!arr.find(({ id }) => id === val.id)) {
return arr.concat(val);
}
return arr;
}, []);
return data;
}, [data]);
return { data: realtimeData, isLoading, error };
return { data, isLoading, error };
}
export default useRealtime;

2
src/components/hooks/useFields.ts
View File

@ -7,7 +7,6 @@ export function useFields() {
{ name: 'url', type: 'string', label: formatMessage(labels.url) },
{ name: 'title', type: 'string', label: formatMessage(labels.pageTitle) },
{ name: 'referrer', type: 'string', label: formatMessage(labels.referrer) },
{ name: 'host', type: 'string', label: formatMessage(labels.host) },
{ name: 'query', type: 'string', label: formatMessage(labels.query) },
{ name: 'browser', type: 'string', label: formatMessage(labels.browser) },
{ name: 'os', type: 'string', label: formatMessage(labels.os) },
@ -15,6 +14,7 @@ export function useFields() {
{ name: 'country', type: 'string', label: formatMessage(labels.country) },
{ name: 'region', type: 'string', label: formatMessage(labels.region) },
{ name: 'city', type: 'string', label: formatMessage(labels.city) },
{ name: 'host', type: 'string', label: formatMessage(labels.host) },
];
return { fields };

16
src/components/metrics/PageviewsChart.tsx
View File

@ -5,11 +5,11 @@ import { renderDateLabels } from 'lib/charts';
export interface PageviewsChartProps extends BarChartProps {
data: {
pageviews: any[];
sessions: any[];
views: any[];
visitors: any[];
compare?: {
pageviews: any[];
sessions: any[];
views: any[];
visitors: any[];
};
};
unit: string;
@ -30,14 +30,14 @@ export function PageviewsChart({ data, unit, isLoading, ...props }: PageviewsCha
datasets: [
{
label: formatMessage(labels.visitors),
data: data.sessions,
data: data.visitors,
borderWidth: 1,
...colors.chart.visitors,
order: 3,
},
{
label: formatMessage(labels.views),
data: data.pageviews,
data: data.views,
borderWidth: 1,
...colors.chart.views,
order: 4,
@ -47,7 +47,7 @@ export function PageviewsChart({ data, unit, isLoading, ...props }: PageviewsCha
{
type: 'line',
label: `${formatMessage(labels.views)} (${formatMessage(labels.previous)})`,
data: data.compare.pageviews,
data: data.compare.views,
borderWidth: 2,
backgroundColor: '#8601B0',
borderColor: '#8601B0',
@ -56,7 +56,7 @@ export function PageviewsChart({ data, unit, isLoading, ...props }: PageviewsCha
{
type: 'line',
label: `${formatMessage(labels.visitors)} (${formatMessage(labels.previous)})`,
data: data.compare.sessions,
data: data.compare.visitors,
borderWidth: 2,
backgroundColor: '#f15bb5',
borderColor: '#f15bb5',

27
src/components/metrics/RealtimeChart.tsx
View File

@ -1,29 +1,10 @@
import { useMemo, useRef } from 'react';
import { format, startOfMinute, subMinutes, isBefore } from 'date-fns';
import { startOfMinute, subMinutes, isBefore } from 'date-fns';
import PageviewsChart from './PageviewsChart';
import { getDateArray } from 'lib/date';
import { DEFAULT_ANIMATION_DURATION, REALTIME_RANGE } from 'lib/constants';
import { RealtimeData } from 'lib/types';
function mapData(data: any[]) {
let last = 0;
const arr = [];
data?.reduce((obj, { timestamp }) => {
const t = startOfMinute(new Date(timestamp));
if (t.getTime() > last) {
obj = { x: format(t, 'yyyy-LL-dd HH:mm:00'), y: 1 };
arr.push(obj);
last = t.getTime();
} else {
obj.y += 1;
}
return obj;
}, {});
return arr;
}
export interface RealtimeChartProps {
data: RealtimeData;
unit: string;
@ -37,12 +18,12 @@ export function RealtimeChart({ data, unit, ...props }: RealtimeChartProps) {
const chartData = useMemo(() => {
if (!data) {
return { pageviews: [], sessions: [] };
return { views: [], visitors: [] };
}
return {
pageviews: getDateArray(mapData(data.pageviews), startDate, endDate, unit),
sessions: getDateArray(mapData(data.visitors), startDate, endDate, unit),
views: getDateArray(data.series.views, startDate, endDate, unit),
visitors: getDateArray(data.series.visitors, startDate, endDate, unit),
};
}, [data, startDate, endDate, unit]);

2
src/lib/clickhouse.ts
View File

@ -9,7 +9,7 @@ import { maxDate } from './date';
import { filtersToArray } from './params';
export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00',
minute: '%Y-%m-%d %H:%i:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d',
month: '%Y-%m-01',

13
src/lib/constants.ts
View File

@ -23,7 +23,7 @@ export const DEFAULT_PAGE_SIZE = 10;
export const DEFAULT_DATE_COMPARE = 'prev';
export const REALTIME_RANGE = 30;
export const REALTIME_INTERVAL = 5000;
export const REALTIME_INTERVAL = 10000;
export const FILTER_COMBINED = 'filter-combined';
export const FILTER_RAW = 'filter-raw';
@ -33,7 +33,16 @@ export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'host'];
export const EVENT_COLUMNS = [
'url',
'entry',
'exit',
'referrer',
'title',
'query',
'event',
'host',
];
export const SESSION_COLUMNS = [
'browser',

19
src/lib/types.ts
View File

@ -199,12 +199,23 @@ export interface QueryOptions {
}
export interface RealtimeData {
pageviews: any[];
sessions: any[];
countries: { [key: string]: number };
events: any[];
pageviews: any[];
referrers: { [key: string]: number };
timestamp: number;
countries?: any[];
visitors?: any[];
series: {
views: any[];
visitors: any[];
};
totals: {
views: number;
visitors: number;
events: number;
countries: number;
};
urls: { [key: string]: number };
visitors: any[];
}
export interface SessionData {

14
src/pages/api/realtime/[websiteId].ts
View File

@ -10,13 +10,13 @@ import { REALTIME_RANGE } from 'lib/constants';
export interface RealtimeRequestQuery {
websiteId: string;
startAt: number;
timezone: string;
}
const schema = {
GET: yup.object().shape({
websiteId: yup.string().uuid().required(),
startAt: yup.number().integer().required(),
timezone: yup.string().required(),
}),
};
@ -28,19 +28,15 @@ export default async (
await useValidate(schema, req, res);
if (req.method === 'GET') {
const { websiteId, startAt } = req.query;
const { websiteId, timezone } = req.query;
if (!(await canViewWebsite(req.auth, websiteId))) {
return unauthorized(res);
}
let startTime = subMinutes(startOfMinute(new Date()), REALTIME_RANGE);
const startDate = subMinutes(startOfMinute(new Date()), REALTIME_RANGE);
if (+startAt > startTime.getTime()) {
startTime = new Date(+startAt);
}
const data = await getRealtimeData(websiteId, startTime);
const data = await getRealtimeData(websiteId, { startDate, timezone });
return ok(res, data);
}

17
src/queries/analytics/events/getEvents.ts
View File

@ -1,31 +1,34 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export function getEvents(...args: [websiteId: string, startDate: Date, eventType: number]) {
export function getEvents(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function relationalQuery(websiteId: string, startDate: Date, eventType: number) {
function relationalQuery(websiteId: string, filters: QueryFilters) {
const { startDate } = filters;
return prisma.client.websiteEvent.findMany({
where: {
websiteId,
eventType,
createdAt: {
gte: startDate,
},
},
orderBy: {
createdAt: 'asc',
createdAt: 'desc',
},
});
}
function clickhouseQuery(websiteId: string, startDate: Date, eventType: number) {
function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery } = clickhouse;
const { startDate } = filters;
return rawQuery(
`
@ -41,13 +44,11 @@ function clickhouseQuery(websiteId: string, startDate: Date, eventType: number)
from website_event
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime64}
and event_type = {eventType:UInt32}
order by created_at asc
order by created_at desc
`,
{
websiteId,
startDate,
eventType,
},
);
}

112
src/queries/analytics/getRealtimeData.ts
View File

@ -1,43 +1,91 @@
import { getSessions, getEvents } from 'queries/index';
import { EVENT_TYPE } from 'lib/constants';
import { getSessions, getEvents, getPageviewStats, getSessionStats } from 'queries/index';
export async function getRealtimeData(websiteId: string, startDate: Date) {
const [pageviews, sessions, events] = await Promise.all([
getEvents(websiteId, startDate, EVENT_TYPE.pageView),
getSessions(websiteId, startDate),
getEvents(websiteId, startDate, EVENT_TYPE.customEvent),
const MAX_SIZE = 50;
function increment(data: object, key: string) {
if (key) {
if (!data[key]) {
data[key] = 1;
} else {
data[key] += 1;
}
}
}
export async function getRealtimeData(
websiteId: string,
criteria: { startDate: Date; timezone: string },
) {
const { startDate, timezone } = criteria;
const filters = { startDate, endDate: new Date(), unit: 'minute', timezone };
const [events, sessions, pageviews, sessionviews] = await Promise.all([
getEvents(websiteId, { startDate }),
getSessions(websiteId, { startDate }),
getPageviewStats(websiteId, filters),
getSessionStats(websiteId, filters),
]);
const decorate = (type: string, data: any[]) => {
return data.map((values: { [key: string]: any }) => ({
...values,
__type: type,
timestamp: values.timestamp ? values.timestamp * 1000 : new Date(values.createdAt).getTime(),
}));
};
const uniques = new Set();
const set = new Set();
const uniques = (type: string, data: any[]) => {
return data.reduce((arr, values: { [key: string]: any }) => {
if (!set.has(values.id)) {
set.add(values.id);
const sessionStats = sessions.reduce(
(obj: { visitors: any; countries: any }, session: { id: any; country: any }) => {
const { countries, visitors } = obj;
const { id, country } = session;
return arr.concat({
...values,
__type: type,
timestamp: values.timestamp
? values.timestamp * 1000
: new Date(values.createdAt).getTime(),
});
if (!uniques.has(id)) {
uniques.add(id);
increment(countries, country);
if (visitors.length < MAX_SIZE) {
visitors.push(session);
}
}
return arr;
}, []);
};
return obj;
},
{
countries: {},
visitors: [],
},
);
const eventStats = events.reduce(
(
obj: { urls: any; referrers: any; events: any },
event: { urlPath: any; referrerDomain: any },
) => {
const { urls, referrers, events } = obj;
const { urlPath, referrerDomain } = event;
increment(urls, urlPath);
increment(referrers, referrerDomain);
if (events.length < MAX_SIZE) {
events.push(event);
}
return obj;
},
{
urls: {},
referrers: {},
events: [],
},
);
return {
pageviews: decorate('pageview', pageviews),
sessions: uniques('session', sessions),
events: decorate('event', events),
...sessionStats,
...eventStats,
series: {
views: pageviews,
visitors: sessionviews,
},
totals: {
views: events.filter(e => !e.eventName).length,
visitors: uniques.size,
events: events.filter(e => e.eventName).length,
countries: Object.keys(sessionStats.countries).length,
},
timestamp: Date.now(),
};
}

6
src/queries/analytics/reports/getGoals.ts
View File

@ -312,7 +312,7 @@ async function clickhouseQuery(
const where = getWhere(urls, events, eventData);
const urlResults = hasUrl
? await rawQuery<any>(
? await rawQuery(
`
select
${columns.url}
@ -332,7 +332,7 @@ async function clickhouseQuery(
: [];
const eventResults = hasEvent
? await rawQuery<any>(
? await rawQuery(
`
select
${columns.events}
@ -352,7 +352,7 @@ async function clickhouseQuery(
: [];
const eventDataResults = hasEventData
? await rawQuery<any>(
? await rawQuery(
`
select
${columns.eventData}

14
src/queries/analytics/sessions/getSessions.ts
View File

@ -1,15 +1,18 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';
import { QueryFilters } from 'lib/types';
export async function getSessions(...args: [websiteId: string, startAt: Date]) {
export async function getSessions(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, startDate: Date) {
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { startDate } = filters;
return prisma.client.session.findMany({
where: {
websiteId,
@ -18,13 +21,14 @@ async function relationalQuery(websiteId: string, startDate: Date) {
},
},
orderBy: {
createdAt: 'asc',
createdAt: 'desc',
},
});
}
async function clickhouseQuery(websiteId: string, startDate: Date) {
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery } = clickhouse;
const { startDate } = filters;
return rawQuery(
`
@ -46,7 +50,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date) {
from website_event
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime64}
order by created_at asc
order by created_at desc
`,
{
websiteId,