Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Brian Cao 2023-07-05 21:09:19 -07:00
commit 8e6620db53
13 changed files with 202 additions and 173 deletions

View File

@ -137,7 +137,7 @@ export const labels = defineMessages({
untitled: { id: 'labels.untitled', defaultMessage: 'Untitled' },
type: { id: 'labels.type', defaultMessage: 'Type' },
filters: { id: 'labels.filters', defaultMessage: 'Filters' },
groupBy: { id: 'labels.group-by', defaultMessage: 'Group by' },
breakdown: { id: 'labels.breakdown', defaultMessage: 'Breakdown' },
true: { id: 'labels.true', defaultMessage: 'True' },
false: { id: 'labels.false', defaultMessage: 'False' },
equals: { id: 'labels.equals', defaultMessage: 'Equals' },
@ -150,6 +150,13 @@ export const labels = defineMessages({
doesNotContain: { id: 'labels.does-not-contain', defaultMessage: 'Does not contain' },
before: { id: 'labels.before', defaultMessage: 'Before' },
after: { id: 'labels.after', defaultMessage: 'After' },
total: { id: 'labels.total', defaultMessage: 'Total' },
sum: { id: 'labels.sum', defaultMessage: 'Sum' },
average: { id: 'labels.average', defaultMessage: 'Average' },
min: { id: 'labels.min', defaultMessage: 'Min' },
max: { id: 'labels.max', defaultMessage: 'Max' },
unique: { id: 'labels.unique', defaultMessage: 'Unique' },
value: { id: 'labels.value', defaultMessage: 'Value' },
});
export const messages = defineMessages({

View File

@ -1,27 +1,30 @@
import { Form, FormRow, Menu, Item } from 'react-basics';
const options = {
number: [
{ label: 'SUM', value: 'sum' },
{ label: 'AVERAGE', value: 'average' },
{ label: 'MIN', value: 'min' },
{ label: 'MAX', value: 'max' },
],
date: [
{ label: 'MIN', value: 'min' },
{ label: 'MAX', value: 'max' },
],
string: [
{ label: 'COUNT', value: 'count' },
{ label: 'DISTINCT', value: 'distinct' },
],
};
import { useMessages } from 'hooks';
export default function FieldAggregateForm({ name, type, onSelect }) {
const { formatMessage, labels } = useMessages();
const options = {
number: [
{ label: formatMessage(labels.sum), value: 'sum' },
{ label: formatMessage(labels.average), value: 'average' },
{ label: formatMessage(labels.min), value: 'min' },
{ label: formatMessage(labels.max), value: 'max' },
],
date: [
{ label: formatMessage(labels.min), value: 'min' },
{ label: formatMessage(labels.max), value: 'max' },
],
string: [
{ label: formatMessage(labels.total), value: 'total' },
{ label: formatMessage(labels.unique), value: 'unique' },
],
};
const items = options[type];
const handleSelect = value => {
onSelect({ name, value });
onSelect({ name, type, value });
};
return (

View File

@ -17,7 +17,7 @@ export default function FieldFilterForm({ name, type, onSelect }) {
return (
<Form>
<FormRow label={name}>
<Menu onSelect={value => onSelect({ name, value: ['eq', value] })}>
<Menu onSelect={value => onSelect({ name, type, value: ['eq', value] })}>
{items.map(value => {
return <Item key={value}>{filters[value]}</Item>;
})}
@ -46,7 +46,7 @@ export default function FieldFilterForm({ name, type, onSelect }) {
</Flexbox>
<Button
variant="primary"
onClick={() => onSelect({ name, value: [filter, value] })}
onClick={() => onSelect({ name, type, value: [filter, value] })}
disabled={!filter || !value}
>
Add

View File

@ -13,5 +13,5 @@
}
.dropdown {
min-width: 60px;
min-width: 180px;
}

View File

@ -1,4 +1,4 @@
import { Icon, Text, TooltipPopup } from 'react-basics';
import { Icon, TooltipPopup } from 'react-basics';
import Icons from 'components/icons';
import Empty from 'components/common/Empty';
import { useMessages } from 'hooks';
@ -13,7 +13,7 @@ export function ParameterList({ items = [], children, onRemove }) {
{items.map((item, index) => {
return (
<div key={index} className={styles.item}>
<Text>{typeof children === 'function' ? children(item) : item}</Text>
{typeof children === 'function' ? children(item) : item}
<TooltipPopup
className={styles.icon}
label={formatMessage(labels.remove)}

View File

@ -13,6 +13,7 @@
border: 1px solid var(--base400);
border-radius: var(--border-radius);
box-shadow: 1px 1px 1px var(--base400);
gap: 10px;
}
.icon {

View File

@ -3,11 +3,11 @@ import { useApi, useMessages } from 'hooks';
import { Form, FormRow, FormButtons, SubmitButton, PopupTrigger, Icon, Popup } from 'react-basics';
import { ReportContext } from 'components/pages/reports/Report';
import Empty from 'components/common/Empty';
import { DATA_TYPES } from 'lib/constants';
import BaseParameters from '../BaseParameters';
import FieldAddForm from './FieldAddForm';
import ParameterList from '../ParameterList';
import { DATA_TYPES, REPORT_PARAMETERS } from 'lib/constants';
import Icons from 'components/icons';
import FieldAddForm from './FieldAddForm';
import BaseParameters from '../BaseParameters';
import ParameterList from '../ParameterList';
import styles from './EventDataParameters.module.css';
function useFields(websiteId, startDate, endDate) {
@ -33,15 +33,15 @@ export function EventDataParameters() {
const { parameters } = report || {};
const { websiteId, dateRange, fields, filters, groups } = parameters || {};
const { startDate, endDate } = dateRange || {};
const queryDisabled = !websiteId || !dateRange;
const queryEnabled = websiteId && dateRange && fields?.length;
const { data, error } = useFields(websiteId, startDate, endDate);
const parametersSelected = websiteId && startDate && endDate;
const hasData = data?.length !== 0;
const parameterGroups = [
{ label: formatMessage(labels.fields), type: 'fields' },
{ label: formatMessage(labels.filters), type: 'filters' },
{ label: formatMessage(labels.groupBy), type: 'groups' },
{ label: formatMessage(labels.fields), group: REPORT_PARAMETERS.fields },
{ label: formatMessage(labels.filters), group: REPORT_PARAMETERS.filters },
{ label: formatMessage(labels.breakdown), group: REPORT_PARAMETERS.groups },
];
const parameterData = {
@ -54,18 +54,21 @@ export function EventDataParameters() {
runReport(values);
};
const handleAdd = (type, value) => {
const data = parameterData[type];
updateReport({ parameters: { [type]: data.concat(value) } });
const handleAdd = (group, value) => {
const data = parameterData[group];
if (!data.find(({ name }) => name === value.name)) {
updateReport({ parameters: { [group]: data.concat(value) } });
}
};
const handleRemove = (type, index) => {
const data = [...parameterData[type]];
const handleRemove = (group, index) => {
const data = [...parameterData[group]];
data.splice(index, 1);
updateReport({ parameters: { [type]: data } });
updateReport({ parameters: { [group]: data } });
};
const AddButton = ({ type }) => {
const AddButton = ({ group }) => {
return (
<PopupTrigger>
<Icon>
@ -75,11 +78,11 @@ export function EventDataParameters() {
{(close, element) => {
return (
<FieldAddForm
type={type}
fields={data.map(({ eventKey, eventDataType }) => ({
name: eventKey,
type: DATA_TYPES[eventDataType],
}))}
group={group}
element={element}
onAdd={handleAdd}
onClose={close}
@ -97,30 +100,34 @@ export function EventDataParameters() {
{!hasData && <Empty message={formatMessage(messages.noEventData)} />}
{parametersSelected &&
hasData &&
parameterGroups.map(({ label, type }) => {
parameterGroups.map(({ label, group }) => {
return (
<FormRow key={label} label={label} action={<AddButton type={type} onAdd={handleAdd} />}>
<FormRow
key={label}
label={label}
action={<AddButton group={group} onAdd={handleAdd} />}
>
<ParameterList
items={parameterData[type]}
onRemove={index => handleRemove(type, index)}
items={parameterData[group]}
onRemove={index => handleRemove(group, index)}
>
{({ name, value }) => {
return (
<div className={styles.parameter}>
{type === 'fields' && (
{group === REPORT_PARAMETERS.fields && (
<>
<div className={styles.op}>{value}</div>
<div>{name}</div>
<div className={styles.op}>{value}</div>
</>
)}
{type === 'filters' && (
{group === REPORT_PARAMETERS.filters && (
<>
<div>{name}</div>
<div className={styles.op}>{value[0]}</div>
<div>{value[1]}</div>
</>
)}
{type === 'groups' && (
{group === REPORT_PARAMETERS.groups && (
<>
<div>{name}</div>
</>
@ -133,7 +140,7 @@ export function EventDataParameters() {
);
})}
<FormButtons>
<SubmitButton variant="primary" disabled={queryDisabled} loading={isRunning}>
<SubmitButton variant="primary" disabled={!queryEnabled} loading={isRunning}>
{formatMessage(labels.runQuery)}
</SubmitButton>
</FormButtons>

View File

@ -1,6 +1,10 @@
.parameter {
display: flex;
gap: 10px;
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
min-width: 0;
}
.op {

View File

@ -1,5 +1,5 @@
import { useContext } from 'react';
import DataTable from 'components/metrics/DataTable';
import { GridTable, GridColumn } from 'react-basics';
import { useMessages } from 'hooks';
import { ReportContext } from '../Report';
@ -8,12 +8,11 @@ export function EventDataTable() {
const { formatMessage, labels } = useMessages();
return (
<DataTable
data={report?.data}
title={formatMessage(labels.eventData)}
metric="#"
showPercentage={true}
/>
<GridTable data={report?.data || []}>
<GridColumn name="field" label={formatMessage(labels.field)} />
<GridColumn name="value" label={formatMessage(labels.value)} />
<GridColumn name="total" label={formatMessage(labels.total)} />
</GridTable>
);
}

View File

@ -1,16 +1,20 @@
import { useState } from 'react';
import { createPortal } from 'react-dom';
import { REPORT_PARAMETERS } from 'lib/constants';
import PopupForm from '../PopupForm';
import FieldSelectForm from '../FieldSelectForm';
import FieldAggregateForm from '../FieldAggregateForm';
import FieldFilterForm from '../FieldFilterForm';
import styles from './FieldAddForm.module.css';
export function FieldAddForm({ fields = [], type, element, onAdd, onClose }) {
export function FieldAddForm({ fields = [], group, element, onAdd, onClose }) {
const [selected, setSelected] = useState();
const handleSelect = value => {
if (type === 'groups') {
const { type } = value;
if (group === REPORT_PARAMETERS.groups || type === 'array' || type === 'boolean') {
value.value = group === REPORT_PARAMETERS.groups ? '' : 'total';
handleSave(value);
return;
}
@ -19,15 +23,19 @@ export function FieldAddForm({ fields = [], type, element, onAdd, onClose }) {
};
const handleSave = value => {
onAdd(type, value);
onAdd(group, value);
onClose();
};
return createPortal(
<PopupForm className={styles.popup} element={element} onClose={onClose}>
{!selected && <FieldSelectForm fields={fields} type={type} onSelect={handleSelect} />}
{selected && type === 'fields' && <FieldAggregateForm {...selected} onSelect={handleSave} />}
{selected && type === 'filters' && <FieldFilterForm {...selected} onSelect={handleSave} />}
{!selected && <FieldSelectForm fields={fields} onSelect={handleSelect} />}
{selected && group === REPORT_PARAMETERS.fields && (
<FieldAggregateForm {...selected} onSelect={handleSave} />
)}
{selected && group === REPORT_PARAMETERS.filters && (
<FieldFilterForm {...selected} onSelect={handleSave} />
)}
</PopupForm>,
document.body,
);

View File

@ -78,6 +78,12 @@ export const DATA_TYPES = {
[DATA_TYPE.array]: 'array',
};
export const REPORT_PARAMETERS = {
fields: 'fields',
filters: 'filters',
groups: 'groups',
} as const;
export const KAFKA_TOPIC = {
event: 'event',
eventData: 'event_data',

View File

@ -4,27 +4,39 @@ import { NextApiRequestQueryBody } from 'lib/types';
import { NextApiResponse } from 'next';
import { ok, methodNotAllowed, unauthorized } from 'next-basics';
import { getEventDataFields } from 'queries/analytics/eventData/getEventDataFields';
import { getEventData } from 'queries';
export interface EventDataRequestBody {
websiteId: string;
urls: string[];
window: number;
dateRange: {
startDate: string;
endDate: string;
};
}
export interface EventDataResponse {
urls: string[];
window: number;
startAt: number;
endAt: number;
fields: [
{
name: string;
type: string;
value: string;
},
];
filters: [
{
name: string;
type: string;
value: string;
},
];
groups: [
{
name: string;
type: string;
},
];
}
export default async (
req: NextApiRequestQueryBody<any, EventDataRequestBody>,
res: NextApiResponse<EventDataResponse>,
res: NextApiResponse<any>,
) => {
await useCors(req, res);
await useAuth(req, res);
@ -45,13 +57,19 @@ export default async (
const {
websiteId,
dateRange: { startDate, endDate },
...criteria
} = req.body;
if (!(await canViewWebsite(req.auth, websiteId))) {
return unauthorized(res);
}
const data = {};
const data = await getEventData(
websiteId,
new Date(startDate),
new Date(endDate),
criteria as any,
);
return ok(res, data);
}

View File

@ -1,26 +1,28 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { WebsiteEventDataMetric } from 'lib/types';
import { loadWebsite } from 'lib/query';
import { DEFAULT_CREATED_AT } from 'lib/constants';
export async function getEventData(
...args: [
websiteId: string,
data: {
startDate: Date;
endDate: Date;
eventName: string;
urlPath?: string;
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
export interface EventDataCriteria {
fields: [{ name: string; type: string; value: string }];
filters: [
{
name: string;
type: string;
value: [string, string];
},
]
];
groups: [
{
name: string;
type: string;
},
];
}
export async function getEventData(
...args: [websiteId: string, startDate: Date, endDate: Date, criteria: EventDataCriteria]
): Promise<WebsiteEventDataMetric[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -28,96 +30,70 @@ export async function getEventData(
});
}
async function relationalQuery(
websiteId: string,
data: {
startDate: Date;
endDate: Date;
timeSeries?: {
unit: string;
timezone: string;
};
eventName: string;
urlPath?: string;
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
},
) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params: any = [websiteId, resetDate, startDate, endDate, eventName || ''];
return rawQuery(
`select
count(*) x
${eventName ? `,event_name eventName` : ''}
${urlPath ? `,url_path urlPath` : ''}
${
timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
}
from event_data
${
eventName || urlPath
? 'join website_event on event_data.id = website_event.website_event_id'
: ''
}
where website_id = $1${toUuid()}
and created_at >= $2
and created_at between $3 and $4
${eventName ? `and eventName = $5` : ''}
${getEventDataFilterQuery(filters, params)}
${timeSeries ? 'group by t' : ''}`,
params,
);
async function relationalQuery() {
return null;
}
async function clickhouseQuery(
websiteId: string,
data: {
startDate: Date;
endDate: Date;
timeSeries?: {
unit: string;
timezone: string;
};
eventName?: string;
urlPath?: string;
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
},
startDate: Date,
endDate: Date,
criteria: EventDataCriteria,
) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { rawQuery, getDateFormat, getBetweenDates, getDateQuery, getEventDataFilterQuery } =
clickhouse;
const { fields } = criteria;
const { rawQuery, getDateFormat, getBetweenDates } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = { websiteId };
return rawQuery(
`select
count(*) x
${eventName ? `,event_name eventName` : ''}
${urlPath ? `,url_path urlPath` : ''}
${
timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
}
from event_data
where website_id = {websiteId:UUID}
${eventName ? `and eventName = ${eventName}` : ''}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
${getEventDataFilterQuery(filters, params)}
${timeSeries ? 'group by t' : ''}`,
params,
);
const uniqueFields = fields.reduce((obj, { name, type }) => {
if (!obj[name]) {
obj[name] = {
columns: ['event_key as field', `count(*) as total`, `${type}_value as value`],
groups: ['event_key', `${type}_value`],
};
}
return obj;
}, {});
const queries = Object.keys(uniqueFields).reduce((arr, key) => {
const field = uniqueFields[key];
const params = { websiteId, name: key };
return arr.concat(
rawQuery(
`select
${field.columns.join(',')}
from event_data
where website_id = {websiteId:UUID}
and event_key = {name:String}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
group by ${field.groups.join(',')}
limit 20
`,
params,
),
);
}, []);
const results = (await Promise.all(queries)).flatMap(n => n);
const columns = results.reduce((arr, row) => {
const keys = Object.keys(row);
for (const key of keys) {
if (!arr.includes(key)) {
arr.push(key);
}
}
return arr;
}, []);
return results.reduce((arr, row) => {
return arr.concat(
columns.reduce((obj, key) => {
obj[key] = row[key];
return obj;
}, {}),
);
}, []);
}