Merge remote-tracking branch 'origin/dev' into dev

# Conflicts:
#	queries/analytics/reports/getRetention.ts
This commit is contained in:
Mike Cao 2023-08-16 10:52:20 -07:00
commit d7bf793050
18 changed files with 153 additions and 104 deletions

View File

@ -72,7 +72,7 @@ export function TestConsole() {
} }
const [websiteId] = id || []; const [websiteId] = id || [];
const website = data.find(({ id }) => websiteId === id); const website = data?.data.find(({ id }) => websiteId === id);
return ( return (
<Page loading={isLoading} error={error}> <Page loading={isLoading} error={error}>

View File

@ -36,6 +36,7 @@ export function EventDataValueTable({ data = [], event }) {
<GridColumn name="dataType" label={formatMessage(labels.type)}> <GridColumn name="dataType" label={formatMessage(labels.type)}>
{row => DATA_TYPES[row.dataType]} {row => DATA_TYPES[row.dataType]}
</GridColumn> </GridColumn>
<GridColumn name="fieldValue" label={formatMessage(labels.value)} />
<GridColumn name="total" label={formatMessage(labels.totalRecords)} width="200px"> <GridColumn name="total" label={formatMessage(labels.totalRecords)} width="200px">
{({ total }) => total.toLocaleString()} {({ total }) => total.toLocaleString()}
</GridColumn> </GridColumn>

View File

@ -29,7 +29,6 @@ export function WebsitesList({ showTeam, showHeader = true, includeTeams, onlyTe
{ enabled: !!user }, { enabled: !!user },
); );
const { showToast } = useToasts(); const { showToast } = useToasts();
const hasData = data && data.length !== 0;
const handleSave = async () => { const handleSave = async () => {
await refetch(); await refetch();
@ -57,21 +56,14 @@ export function WebsitesList({ showTeam, showHeader = true, includeTeams, onlyTe
return ( return (
<Page loading={isLoading} error={error}> <Page loading={isLoading} error={error}>
{showHeader && <PageHeader title={formatMessage(labels.websites)}>{addButton}</PageHeader>} {showHeader && <PageHeader title={formatMessage(labels.websites)}>{addButton}</PageHeader>}
{hasData && ( <WebsitesTable
<WebsitesTable data={data}
data={data} showTeam={showTeam}
showTeam={showTeam} onFilterChange={handleFilterChange}
onFilterChange={handleFilterChange} onPageChange={handlePageChange}
onPageChange={handlePageChange} onPageSizeChange={handlePageSizeChange}
onPageSizeChange={handlePageSizeChange} filterValue={filter}
filterValue={filter} />
/>
)}
{!hasData && (
<EmptyPlaceholder message={formatMessage(messages.noDataAvailable)}>
{addButton}
</EmptyPlaceholder>
)}
</Page> </Page>
); );
} }

View File

@ -1,3 +1,4 @@
import EmptyPlaceholder from 'components/common/EmptyPlaceholder';
import Link from 'next/link'; import Link from 'next/link';
import { Button, Text, Icon, Icons } from 'react-basics'; import { Button, Text, Icon, Icons } from 'react-basics';
import SettingsTable from 'components/common/SettingsTable'; import SettingsTable from 'components/common/SettingsTable';
@ -13,10 +14,12 @@ export function WebsitesTable({
onPageSizeChange, onPageSizeChange,
showTeam, showTeam,
}) { }) {
const { formatMessage, labels } = useMessages(); const { formatMessage, labels, messages } = useMessages();
const { openExternal } = useConfig(); const { openExternal } = useConfig();
const { user } = useUser(); const { user } = useUser();
const showTable = data && (filterValue || data?.data.length !== 0);
const teamColumns = [ const teamColumns = [
{ name: 'teamName', label: formatMessage(labels.teamName) }, { name: 'teamName', label: formatMessage(labels.teamName) },
{ name: 'owner', label: formatMessage(labels.owner) }, { name: 'owner', label: formatMessage(labels.owner) },
@ -30,51 +33,56 @@ export function WebsitesTable({
]; ];
return ( return (
<SettingsTable <>
columns={columns} {showTable && (
data={data} <SettingsTable
showSearch={true} columns={columns}
showPaging={true} data={data}
onFilterChange={onFilterChange} showSearch={true}
onPageChange={onPageChange} showPaging={true}
onPageSizeChange={onPageSizeChange} onFilterChange={onFilterChange}
filterValue={filterValue} onPageChange={onPageChange}
> onPageSizeChange={onPageSizeChange}
{row => { filterValue={filterValue}
const { >
id, {row => {
teamWebsite, const {
user: { username, id: ownerId }, id,
} = row; teamWebsite,
if (showTeam) { user: { username, id: ownerId },
row.teamName = teamWebsite[0]?.team.name; } = row;
row.owner = username; if (showTeam) {
} row.teamName = teamWebsite[0]?.team.name;
row.owner = username;
}
return ( return (
<> <>
{(!showTeam || ownerId === user.id) && ( {(!showTeam || ownerId === user.id) && (
<Link href={`/settings/websites/${id}`}> <Link href={`/settings/websites/${id}`}>
<Button> <Button>
<Icon> <Icon>
<Icons.Edit /> <Icons.Edit />
</Icon> </Icon>
<Text>{formatMessage(labels.edit)}</Text> <Text>{formatMessage(labels.edit)}</Text>
</Button> </Button>
</Link> </Link>
)} )}
<Link href={`/websites/${id}`} target={openExternal ? '_blank' : null}> <Link href={`/websites/${id}`} target={openExternal ? '_blank' : null}>
<Button> <Button>
<Icon> <Icon>
<Icons.External /> <Icons.External />
</Icon> </Icon>
<Text>{formatMessage(labels.view)}</Text> <Text>{formatMessage(labels.view)}</Text>
</Button> </Button>
</Link> </Link>
</> </>
); );
}} }}
</SettingsTable> </SettingsTable>
)}
{!showTable && <EmptyPlaceholder message={formatMessage(messages.noDataAvailable)} />}
</>
); );
} }

View File

@ -5,18 +5,18 @@ import { EventDataMetricsBar } from 'components/pages/event-data/EventDataMetric
import { useDateRange, useApi, usePageQuery } from 'hooks'; import { useDateRange, useApi, usePageQuery } from 'hooks';
import styles from './WebsiteEventData.module.css'; import styles from './WebsiteEventData.module.css';
function useData(websiteId, eventName) { function useData(websiteId, event) {
const [dateRange] = useDateRange(websiteId); const [dateRange] = useDateRange(websiteId);
const { startDate, endDate } = dateRange; const { startDate, endDate } = dateRange;
const { get, useQuery } = useApi(); const { get, useQuery } = useApi();
const { data, error, isLoading } = useQuery( const { data, error, isLoading } = useQuery(
['event-data:events', { websiteId, startDate, endDate, eventName }], ['event-data:events', { websiteId, startDate, endDate, event }],
() => () =>
get('/event-data/events', { get('/event-data/events', {
websiteId, websiteId,
startAt: +startDate, startAt: +startDate,
endAt: +endDate, endAt: +endDate,
eventName, event,
}), }),
{ enabled: !!(websiteId && startDate && endDate) }, { enabled: !!(websiteId && startDate && endDate) },
); );

View File

@ -5,6 +5,7 @@ import { FILTER_COLUMNS, SESSION_COLUMNS, OPERATORS } from './constants';
import { loadWebsite } from './load'; import { loadWebsite } from './load';
import { maxDate } from './date'; import { maxDate } from './date';
import { QueryFilters, QueryOptions, SearchFilter } from './types'; import { QueryFilters, QueryOptions, SearchFilter } from './types';
import { Prisma } from '@prisma/client';
const MYSQL_DATE_FORMATS = { const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00', minute: '%Y-%m-%d %H:%i:00',
@ -34,6 +35,30 @@ function getAddMinutesQuery(field: string, minutes: number): string {
} }
} }
function getDayDiffQuery(field1: string, field2: string): string {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
return `${field1}::date - ${field2}::date`;
}
if (db === MYSQL) {
return `DATEDIFF(${field1}, ${field2})`;
}
}
function getCastColumnQuery(field: string, type: string): string {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
return `${field}::${type}`;
}
if (db === MYSQL) {
return `${field}`;
}
}
function getDateQuery(field: string, unit: string, timezone?: string): string { function getDateQuery(field: string, unit: string, timezone?: string): string {
const db = getDatabaseType(); const db = getDatabaseType();
@ -177,13 +202,28 @@ function getPageFilters(filters: SearchFilter<any>): [
]; ];
} }
function getSearchMode(): { mode?: Prisma.QueryMode } {
const db = getDatabaseType();
if (db === POSTGRESQL) {
return {
mode: 'insensitive',
};
}
return {};
}
export default { export default {
...prisma, ...prisma,
getAddMinutesQuery, getAddMinutesQuery,
getDayDiffQuery,
getCastColumnQuery,
getDateQuery, getDateQuery,
getTimestampIntervalQuery, getTimestampIntervalQuery,
getFilterQuery, getFilterQuery,
parseFilters, parseFilters,
getPageFilters, getPageFilters,
getSearchMode,
rawQuery, rawQuery,
}; };

View File

@ -126,13 +126,8 @@ export interface WebsiteEventMetric {
y: number; y: number;
} }
export interface WebsiteEventDataStats { export interface WebsiteEventData {
fieldName: string; eventName?: string;
dataType: number;
total: number;
}
export interface WebsiteEventDataFields {
fieldName: string; fieldName: string;
dataType: number; dataType: number;
fieldValue?: string; fieldValue?: string;

View File

@ -5,16 +5,17 @@ import { NextApiResponse } from 'next';
import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { ok, methodNotAllowed, unauthorized } from 'next-basics';
import { getEventDataEvents } from 'queries'; import { getEventDataEvents } from 'queries';
export interface EventDataFieldsRequestBody { export interface EventDataEventsRequestQuery {
websiteId: string; websiteId: string;
dateRange: { dateRange: {
startDate: string; startDate: string;
endDate: string; endDate: string;
}; };
event?: string;
} }
export default async ( export default async (
req: NextApiRequestQueryBody<any, EventDataFieldsRequestBody>, req: NextApiRequestQueryBody<EventDataEventsRequestQuery>,
res: NextApiResponse<any>, res: NextApiResponse<any>,
) => { ) => {
await useCors(req, res); await useCors(req, res);

View File

@ -5,7 +5,7 @@ import { NextApiResponse } from 'next';
import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { ok, methodNotAllowed, unauthorized } from 'next-basics';
import { getEventDataFields } from 'queries'; import { getEventDataFields } from 'queries';
export interface EventDataFieldsRequestBody { export interface EventDataFieldsRequestQuery {
websiteId: string; websiteId: string;
dateRange: { dateRange: {
startDate: string; startDate: string;
@ -15,7 +15,7 @@ export interface EventDataFieldsRequestBody {
} }
export default async ( export default async (
req: NextApiRequestQueryBody<any, EventDataFieldsRequestBody>, req: NextApiRequestQueryBody<EventDataFieldsRequestQuery>,
res: NextApiResponse<any>, res: NextApiResponse<any>,
) => { ) => {
await useCors(req, res); await useCors(req, res);

View File

@ -5,7 +5,7 @@ import { NextApiResponse } from 'next';
import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { ok, methodNotAllowed, unauthorized } from 'next-basics';
import { getEventDataFields } from 'queries'; import { getEventDataFields } from 'queries';
export interface EventDataRequestBody { export interface EventDataStatsRequestQuery {
websiteId: string; websiteId: string;
dateRange: { dateRange: {
startDate: string; startDate: string;
@ -15,7 +15,7 @@ export interface EventDataRequestBody {
} }
export default async ( export default async (
req: NextApiRequestQueryBody<any, EventDataRequestBody>, req: NextApiRequestQueryBody<EventDataStatsRequestQuery>,
res: NextApiResponse<any>, res: NextApiResponse<any>,
) => { ) => {
await useCors(req, res); await useCors(req, res);
@ -32,18 +32,18 @@ export default async (
const endDate = new Date(+endAt); const endDate = new Date(+endAt);
const results = await getEventDataFields(websiteId, { startDate, endDate }); const results = await getEventDataFields(websiteId, { startDate, endDate });
const events = new Set(); const fields = new Set();
const data = results.reduce( const data = results.reduce(
(obj, row) => { (obj, row) => {
events.add(row.fieldName); fields.add(row.fieldName);
obj.records += Number(row.total); obj.records += Number(row.total);
return obj; return obj;
}, },
{ fields: results.length, records: 0 }, { events: results.length, records: 0 },
); );
return ok(res, { ...data, events: events.size }); return ok(res, { ...data, fields: fields.size });
} }
return methodNotAllowed(res); return methodNotAllowed(res);

View File

@ -38,6 +38,8 @@ export async function getReports(
filterType = REPORT_FILTER_TYPES.all, filterType = REPORT_FILTER_TYPES.all,
} = ReportSearchFilter; } = ReportSearchFilter;
const mode = prisma.getSearchMode();
const where: Prisma.ReportWhereInput = { const where: Prisma.ReportWhereInput = {
...(userId && { userId: userId }), ...(userId && { userId: userId }),
...(websiteId && { websiteId: websiteId }), ...(websiteId && { websiteId: websiteId }),
@ -73,7 +75,7 @@ export async function getReports(
filterType === REPORT_FILTER_TYPES.name) && { filterType === REPORT_FILTER_TYPES.name) && {
name: { name: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}), }),
}, },
@ -82,7 +84,7 @@ export async function getReports(
filterType === REPORT_FILTER_TYPES.description) && { filterType === REPORT_FILTER_TYPES.description) && {
description: { description: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}), }),
}, },
@ -91,7 +93,7 @@ export async function getReports(
filterType === REPORT_FILTER_TYPES.type) && { filterType === REPORT_FILTER_TYPES.type) && {
type: { type: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}), }),
}, },
@ -101,7 +103,7 @@ export async function getReports(
user: { user: {
username: { username: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}, },
}), }),
@ -112,7 +114,7 @@ export async function getReports(
website: { website: {
name: { name: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}, },
}), }),
@ -123,7 +125,7 @@ export async function getReports(
website: { website: {
domain: { domain: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}, },
}), }),

View File

@ -86,6 +86,8 @@ export async function getTeams(
options?: { include?: Prisma.TeamInclude }, options?: { include?: Prisma.TeamInclude },
): Promise<FilterResult<Team[]>> { ): Promise<FilterResult<Team[]>> {
const { userId, filter, filterType = TEAM_FILTER_TYPES.all } = TeamSearchFilter; const { userId, filter, filterType = TEAM_FILTER_TYPES.all } = TeamSearchFilter;
const mode = prisma.getSearchMode();
const where: Prisma.TeamWhereInput = { const where: Prisma.TeamWhereInput = {
...(userId && { ...(userId && {
teamUser: { teamUser: {
@ -97,7 +99,7 @@ export async function getTeams(
OR: [ OR: [
{ {
...((filterType === TEAM_FILTER_TYPES.all || filterType === TEAM_FILTER_TYPES.name) && { ...((filterType === TEAM_FILTER_TYPES.all || filterType === TEAM_FILTER_TYPES.name) && {
name: { startsWith: filter, mode: 'insensitive' }, name: { startsWith: filter, ...mode },
}), }),
}, },
{ {
@ -109,7 +111,7 @@ export async function getTeams(
user: { user: {
username: { username: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}, },
}, },

View File

@ -41,6 +41,8 @@ export async function getUsers(
options?: { include?: Prisma.UserInclude }, options?: { include?: Prisma.UserInclude },
): Promise<FilterResult<User[]>> { ): Promise<FilterResult<User[]>> {
const { teamId, filter, filterType = USER_FILTER_TYPES.all } = UserSearchFilter; const { teamId, filter, filterType = USER_FILTER_TYPES.all } = UserSearchFilter;
const mode = prisma.getSearchMode();
const where: Prisma.UserWhereInput = { const where: Prisma.UserWhereInput = {
...(teamId && { ...(teamId && {
teamUser: { teamUser: {
@ -57,7 +59,7 @@ export async function getUsers(
filterType === USER_FILTER_TYPES.username) && { filterType === USER_FILTER_TYPES.username) && {
username: { username: {
startsWith: filter, startsWith: filter,
mode: 'insensitive', ...mode,
}, },
}), }),
}, },

View File

@ -30,6 +30,7 @@ export async function getWebsites(
filter, filter,
filterType = WEBSITE_FILTER_TYPES.all, filterType = WEBSITE_FILTER_TYPES.all,
} = WebsiteSearchFilter; } = WebsiteSearchFilter;
const mode = prisma.getSearchMode();
const where: Prisma.WebsiteWhereInput = { const where: Prisma.WebsiteWhereInput = {
...(teamId && { ...(teamId && {
@ -79,13 +80,13 @@ export async function getWebsites(
{ {
...((filterType === WEBSITE_FILTER_TYPES.all || ...((filterType === WEBSITE_FILTER_TYPES.all ||
filterType === WEBSITE_FILTER_TYPES.name) && { filterType === WEBSITE_FILTER_TYPES.name) && {
name: { startsWith: filter, mode: 'insensitive' }, name: { startsWith: filter, ...mode },
}), }),
}, },
{ {
...((filterType === WEBSITE_FILTER_TYPES.all || ...((filterType === WEBSITE_FILTER_TYPES.all ||
filterType === WEBSITE_FILTER_TYPES.domain) && { filterType === WEBSITE_FILTER_TYPES.domain) && {
domain: { startsWith: filter, mode: 'insensitive' }, domain: { startsWith: filter, ...mode },
}), }),
}, },
], ],

View File

@ -1,11 +1,11 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { QueryFilters, WebsiteEventDataFields } from 'lib/types'; import { QueryFilters, WebsiteEventData } from 'lib/types';
export async function getEventDataEvents( export async function getEventDataEvents(
...args: [websiteId: string, filters: QueryFilters] ...args: [websiteId: string, filters: QueryFilters]
): Promise<WebsiteEventDataFields[]> { ): Promise<WebsiteEventData[]> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),
@ -24,7 +24,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
website_event.event_name as "eventName", website_event.event_name as "eventName",
event_data.event_key as "fieldName", event_data.event_key as "fieldName",
event_data.data_type as "dataType", event_data.data_type as "dataType",
event_data.string_value as "value", event_data.string_value as "fieldValue",
count(*) as "total" count(*) as "total"
from event_data from event_data
inner join website_event inner join website_event
@ -71,7 +71,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
event_name as eventName, event_name as eventName,
event_key as fieldName, event_key as fieldName,
data_type as dataType, data_type as dataType,
string_value as value, string_value as fieldValue,
count(*) as total count(*) as total
from event_data from event_data
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}

View File

@ -1,11 +1,11 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { QueryFilters, WebsiteEventDataFields } from 'lib/types'; import { QueryFilters, WebsiteEventData } from 'lib/types';
export async function getEventDataFields( export async function getEventDataFields(
...args: [websiteId: string, filters: QueryFilters & { field?: string }] ...args: [websiteId: string, filters: QueryFilters & { field?: string }]
): Promise<WebsiteEventDataFields[]> { ): Promise<WebsiteEventData[]> {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),

View File

@ -41,7 +41,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
} }
async function clickhouseQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters; const { timezone = 'UTC', unit = 'day' } = filters;
const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, { const { filterQuery, params } = await parseFilters(websiteId, {
...filters, ...filters,

View File

@ -35,7 +35,7 @@ async function relationalQuery(
}[] }[]
> { > {
const { startDate, endDate, timezone = 'UTC' } = filters; const { startDate, endDate, timezone = 'UTC' } = filters;
const { getDateQuery, rawQuery } = prisma; const { getDateQuery, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma;
const unit = 'day'; const unit = 'day';
return rawQuery( return rawQuery(
@ -50,7 +50,10 @@ async function relationalQuery(
user_activities AS ( user_activities AS (
select distinct select distinct
w.session_id, w.session_id,
(${getDateQuery('created_at', unit, timezone)}::date - c.cohort_date::date) as day_number ${getDayDiffQuery(
getDateQuery('created_at', unit, timezone),
'c.cohort_date',
)} as day_number
from website_event w from website_event w
join cohort_items c join cohort_items c
on w.session_id = c.session_id on w.session_id = c.session_id
@ -79,7 +82,7 @@ async function relationalQuery(
c.day_number as day, c.day_number as day,
s.visitors, s.visitors,
c.visitors as "returnVisitors", c.visitors as "returnVisitors",
c.visitors::float * 100 / s.visitors as percentage ${getCastColumnQuery('c.visitors', 'float')} * 100 / s.visitors as percentage
from cohort_date c from cohort_date c
join cohort_size s join cohort_size s
on c.cohort_date = s.cohort_date on c.cohort_date = s.cohort_date
@ -90,7 +93,9 @@ async function relationalQuery(
startDate, startDate,
endDate, endDate,
}, },
); ).then(results => {
return results.map(i => ({ ...i, percentage: Number(i.percentage) || 0 }));
});
} }
async function clickhouseQuery( async function clickhouseQuery(