Use dataStartDate for resetDate.

This commit is contained in:
Mike Cao 2023-07-25 15:17:50 -07:00
parent ee8de858d1
commit 0f98fb5959
15 changed files with 172 additions and 85 deletions

View File

@ -1,25 +1,13 @@
import useApi from 'hooks/useApi';
import useDateRange from 'hooks/useDateRange';
import DateFilter from './DateFilter';
import styles from './WebsiteDateFilter.module.css';
export default function WebsiteDateFilter({ websiteId }) {
const { get } = useApi();
const [dateRange, setDateRange] = useDateRange(websiteId);
const { value, startDate, endDate } = dateRange;
const handleChange = async value => {
if (value === 'all' && websiteId) {
const data = await get(`/websites/${websiteId}`);
if (data) {
const start = new Date(data.createdAt).getTime();
const end = Date.now();
setDateRange(`range:${start}:${end}`);
}
} else if (value !== 'all') {
setDateRange(value);
}
setDateRange(value);
};
return (

View File

@ -94,7 +94,7 @@ export function RealtimePage({ websiteId }) {
<WebsiteHeader websiteId={websiteId} />
<RealtimeHeader websiteId={websiteId} data={currentData} />
<div className={styles.chart}>
<RealtimeChart data={realtimeData} unit="minute" records={REALTIME_RANGE} />
<RealtimeChart data={realtimeData} unit="minute" />
</div>
<GridRow>
<GridColumn xs={12} sm={12} md={12} lg={4} xl={4}>

View File

@ -45,15 +45,7 @@ export function WebsiteChart({ websiteId }) {
return { pageviews: [], sessions: [] };
}, [data, startDate, endDate, unit, modified]);
return (
<PageviewsChart
websiteId={websiteId}
data={chartData}
unit={unit}
records={getDateLength(startDate, endDate, unit)}
loading={isLoading}
/>
);
return <PageviewsChart websiteId={websiteId} data={chartData} unit={unit} loading={isLoading} />;
}
export default WebsiteChart;

View File

@ -26,7 +26,6 @@ import {
differenceInCalendarMonths,
differenceInCalendarYears,
format,
parseISO,
} from 'date-fns';
import { getDateLocale } from 'lib/lang';
@ -43,6 +42,14 @@ export function parseDateRange(value, locale = 'en-US') {
return value;
}
if (value === 'all') {
return {
startDate: new Date(0),
endDate: new Date(1),
value,
};
}
if (value?.startsWith?.('range')) {
const [, startAt, endAt] = value.split(':');
@ -148,17 +155,32 @@ export function parseDateRange(value, locale = 'en-US') {
}
}
export function getDateRangeValues(startDate, endDate) {
let unit = 'year';
if (differenceInHours(endDate, startDate) <= 48) {
unit = 'hour';
export function getAllowedUnits(unit) {
const units = ['minute', 'hour', 'day', 'month', 'year'];
return units.splice(units.indexOf(unit));
}
export function getMinimumUnit(startDate, endDate) {
if (differenceInMinutes(endDate, startDate) <= 60) {
return 'minute';
} else if (differenceInHours(endDate, startDate) <= 48) {
return 'hour';
} else if (differenceInCalendarDays(endDate, startDate) <= 90) {
unit = 'day';
return 'day';
} else if (differenceInCalendarMonths(endDate, startDate) <= 24) {
unit = 'month';
return 'month';
}
return { startDate: startOfDay(startDate), endDate: endOfDay(endDate), unit };
return 'year';
}
export function getDateRangeValues(startDate, endDate) {
return {
startDate: startOfDay(startDate),
endDate: endOfDay(endDate),
unit: getMinimumUnit(startDate, endDate),
};
}
export function getDateFromString(str) {

View File

@ -1,8 +1,9 @@
import cache from 'lib/cache';
import { getWebsite, getSession, getUser } from 'queries';
import { User, Website, Session } from '@prisma/client';
import { DEFAULT_RESET_DATE } from './constants';
export async function loadWebsite(websiteId: string): Promise<Website> {
export async function loadWebsite(websiteId: string): Promise<Website & { dataStartDate: Date }> {
let website;
if (cache.enabled) {
@ -11,6 +12,8 @@ export async function loadWebsite(websiteId: string): Promise<Website> {
website = await getWebsite({ id: websiteId });
}
website.dataStartDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
if (!website || website.deletedAt) {
return null;
}

View File

@ -3,11 +3,10 @@ import { NextApiResponse } from 'next';
import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';
import { NextApiRequestQueryBody, WebsitePageviews } from 'lib/types';
import { canViewWebsite } from 'lib/auth';
import { getAllowedUnits } from 'lib/date';
import { useAuth, useCors } from 'lib/middleware';
import { getPageviewStats } from 'queries';
const unitTypes = ['year', 'month', 'hour', 'day'];
export interface WebsitePageviewRequestQuery {
id: string;
startAt: number;
@ -57,7 +56,7 @@ export default async (
const startDate = new Date(+startAt);
const endDate = new Date(+endAt);
if (!moment.tz.zone(timezone) || !unitTypes.includes(unit)) {
if (!moment.tz.zone(timezone) || !getAllowedUnits(unit).includes(unit)) {
return badRequest(res);
}

View File

@ -27,7 +27,6 @@ async function relationalQuery(
) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { field, event } = filters;
if (event) {
@ -43,13 +42,13 @@ async function relationalQuery(
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId:uuid}}
and ed.event_key = {{field}}
and ed.created_at >= {{resetDate}}
and ed.created_at >= {{dataStartDate}}
and ed.created_at between {{startDate}} and {{endDate}}
and we.event_name = {{event}}
group by ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
{ ...filters, websiteId, resetDate, startDate, endDate },
{ ...filters, websiteId, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
return rawQuery(
@ -64,12 +63,12 @@ async function relationalQuery(
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.event_key = {{field}}
and ed.created_at >= {{resetDate}}
and ed.created_at >= {{dataStartDate}}
and ed.created_at between {{startDate}} and {{endDate}}
group by we.event_name, ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
{ websiteId, field, resetDate, startDate, endDate },
{ websiteId, field, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
@ -81,7 +80,6 @@ async function clickhouseQuery(
) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { event } = filters;
if (event) {
@ -95,14 +93,14 @@ async function clickhouseQuery(
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_name = {event:String}
group by event_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100
`,
{ ...filters, websiteId, resetDate, startDate, endDate },
{ ...filters, websiteId, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
@ -115,12 +113,12 @@ async function clickhouseQuery(
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, data_type, event_name
order by 1 asc, 2 asc
limit 100
`,
{ websiteId, resetDate, startDate, endDate },
{ websiteId, startDate, endDate, dataStartDate: website.dataStartDate },
);
}

View File

@ -17,7 +17,6 @@ export async function getEventDataFields(
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
if (field) {
return rawQuery(
@ -29,13 +28,13 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date
from event_data
where website_id = {{websiteId::uuid}}
and event_key = {{field}}
and created_at >= {{resetDate}}
and created_at >= {{dataStartDate}}
and created_at between {{startDate}} and {{endDate}}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field, resetDate, startDate, endDate },
{ websiteId, field, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
@ -47,20 +46,19 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date
count(*) as total
from event_data
where website_id = {{websiteId::uuid}}
and created_at >= {{resetDate}}
and created_at >= {{dataStartDate}}
and created_at between {{startDate}} and {{endDate}}
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId, resetDate, startDate, endDate },
{ websiteId, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
if (field) {
return rawQuery(
@ -72,13 +70,13 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date
from event_data
where website_id = {websiteId:UUID}
and event_key = {field:String}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field, resetDate, startDate, endDate },
{ websiteId, field, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
@ -90,12 +88,12 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId, resetDate, startDate, endDate },
{ websiteId, startDate, endDate, dataStartDate: website.dataStartDate },
);
}

View File

@ -47,7 +47,6 @@ async function relationalQuery(
) {
const { rawQuery, getDateQuery, getFilterQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const filterQuery = getFilterQuery(filters);
return rawQuery(
@ -58,14 +57,21 @@ async function relationalQuery(
count(*) y
from website_event
where website_id = {{websiteId::uuid}}
and created_at >= {{resetDate}}
and created_at >= {{dataStartDate}}
and created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1, 2
order by 2
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.customEvent },
{
...filters,
websiteId,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: EVENT_TYPE.customEvent,
},
);
}
@ -90,7 +96,6 @@ async function clickhouseQuery(
) {
const { rawQuery, getDateQuery, getFilterQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const filterQuery = getFilterQuery(filters);
return rawQuery(
@ -101,13 +106,20 @@ async function clickhouseQuery(
count(*) y
from website_event
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTIme}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by x, t
order by t
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.customEvent },
{
...filters,
websiteId,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: EVENT_TYPE.customEvent,
},
);
}

View File

@ -33,12 +33,11 @@ async function relationalQuery(
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params: any = {
websiteId,
resetDate,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
};
@ -59,7 +58,7 @@ async function relationalQuery(
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at >= {{resetDate}}
and website_event.created_at >= {{dataStartDate}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${excludeDomain}
@ -84,12 +83,11 @@ async function clickhouseQuery(
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const params = {
websiteId,
resetDate,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: undefined,
};
@ -108,7 +106,7 @@ async function clickhouseQuery(
select ${column} x, count(*) y
from website_event
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${excludeDomain}

View File

@ -47,7 +47,6 @@ async function relationalQuery(
} = criteria;
const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
@ -58,13 +57,20 @@ async function relationalQuery(
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at >= {{resetDate}}
and website_event.created_at >= {{dataStartDate}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.pageView },
{
...filters,
websiteId,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: EVENT_TYPE.pageView,
},
);
}
@ -90,7 +96,6 @@ async function clickhouseQuery(
} = criteria;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { filterQuery } = parseFilters(filters);
return rawQuery(
@ -104,7 +109,7 @@ async function clickhouseQuery(
count(${count !== '*' ? 'distinct session_id' : count}) as y
from website_event
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
@ -112,6 +117,13 @@ async function clickhouseQuery(
) as g
order by t
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.pageView },
{
...filters,
websiteId,
startDate,
endDate,
eventType: EVENT_TYPE.pageView,
dataStartDate: website.dataStartDate,
},
);
}

View File

@ -21,7 +21,6 @@ async function relationalQuery(
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { startDate, endDate, column, filters = {} } = criteria;
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession } = parseFilters(filters);
@ -36,14 +35,14 @@ async function relationalQuery(
on website_event.website_id = website.website_id
${joinSession}
where website.website_id = {{websiteId::uuid}}
and website_event.created_at >= {{resetDate}}
and website_event.created_at >= {{dataStartDate}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
)
group by 1
order by 2 desc
limit 100`,
{ ...filters, websiteId, resetDate, startDate, endDate },
{ ...filters, websiteId, startDate, endDate, dataStartDate: website.dataStartDate },
);
}
@ -54,7 +53,6 @@ async function clickhouseQuery(
const { startDate, endDate, column, filters = {} } = data;
const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { filterQuery } = parseFilters(filters);
return rawQuery(
@ -63,7 +61,7 @@ async function clickhouseQuery(
${column} x, count(distinct session_id) y
from website_event as x
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
@ -71,6 +69,13 @@ async function clickhouseQuery(
order by y desc
limit 100
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.pageView },
{
...filters,
websiteId,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: EVENT_TYPE.pageView,
},
);
}

View File

@ -0,0 +1,47 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { loadWebsite } from 'lib/query';
import { DEFAULT_RESET_DATE } from 'lib/constants';
export async function getWebsiteDateRange(...args: [websiteId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string) {
const { rawQuery } = prisma;
return rawQuery(
`
select
min(created_at) as min,
max(created_at) as max
from website_event
join website
on website_event.website_id = website.website_id
where website.website_id = {{websiteId::uuid}}
and website_event.created_at >= coalesce(website.reset_at, website.created_at)
`,
{ websiteId },
);
}
async function clickhouseQuery(websiteId: string) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
return rawQuery(
`
select
min(created_at) as min,
max(created_at) as max
from website_event
where website_id = {websiteId:UUID}
and created_at >= {dataStartDate:DateTime}
`,
{ websiteId, dataStartDate: website.dataStartDate },
);
}

View File

@ -23,7 +23,6 @@ async function relationalQuery(
const { startDate, endDate, filters = {} } = criteria;
const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
@ -45,13 +44,20 @@ async function relationalQuery(
${joinSession}
where event_type = {{eventType}}
and website.website_id = {{websiteId::uuid}}
and website_event.created_at >= {{resetDate}}
and website_event.created_at >= {{dataStartDate}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by 1, 2
) as t
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.pageView },
{
...filters,
websiteId,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: EVENT_TYPE.pageView,
},
);
}
@ -62,7 +68,6 @@ async function clickhouseQuery(
const { startDate, endDate, filters = {} } = criteria;
const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_RESET_DATE);
const { filterQuery } = parseFilters(filters);
return rawQuery(
@ -81,13 +86,20 @@ async function clickhouseQuery(
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
and created_at >= {resetDate:DateTime}
and created_at >= {dataStartDate:DateTime}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by session_id, time_series
) as t;
`,
{ ...filters, websiteId, resetDate, startDate, endDate, eventType: EVENT_TYPE.pageView },
{
...filters,
websiteId,
startDate,
endDate,
dataStartDate: website.dataStartDate,
eventType: EVENT_TYPE.pageView,
},
);
}

View File

@ -21,4 +21,5 @@ export * from './analytics/sessions/getSessions';
export * from './analytics/sessions/saveSessionData';
export * from './analytics/stats/getActiveVisitors';
export * from './analytics/stats/getRealtimeData';
export * from './analytics/stats/getWebsiteDateRange';
export * from './analytics/stats/getWebsiteStats';