diff --git a/lib/query.ts b/lib/query.ts index 8684215f..4ce18b09 100644 --- a/lib/query.ts +++ b/lib/query.ts @@ -1,9 +1,8 @@ import cache from 'lib/cache'; import { getWebsite, getSession, getUser } from 'queries'; import { User, Website, Session } from '@prisma/client'; -import { DEFAULT_RESET_DATE } from './constants'; -export async function loadWebsite(websiteId: string): Promise { +export async function loadWebsite(websiteId: string): Promise { let website; if (cache.enabled) { @@ -12,8 +11,6 @@ export async function loadWebsite(websiteId: string): Promise= {{dataStartDate}} and ed.created_at between {{startDate}} and {{endDate}} and we.event_name = {{event}} group by ed.event_key, ed.string_value order by 3 desc, 2 desc, 1 asc `, - { ...filters, websiteId, startDate, endDate, dataStartDate: website.dataStartDate }, + { ...filters, websiteId, startDate: max([startDate, website.resetAt]), endDate }, ); } return rawQuery( @@ -63,12 +63,11 @@ async function relationalQuery( on we.event_id = ed.website_event_id where ed.website_id = {{websiteId::uuid}} and ed.event_key = {{field}} - and ed.created_at >= {{dataStartDate}} and ed.created_at between {{startDate}} and {{endDate}} group by we.event_name, ed.event_key, ed.string_value order by 3 desc, 2 desc, 1 asc `, - { websiteId, field, startDate, endDate, dataStartDate: website.dataStartDate }, + { websiteId, field, startDate: max([startDate, website.resetAt]), endDate }, ); } @@ -93,14 +92,13 @@ async function clickhouseQuery( count(*) as total from event_data where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_name = {event:String} group by event_key, data_type, string_value, event_name order by 1 asc, 2 asc, 3 asc, 4 desc limit 100 `, - { ...filters, websiteId, startDate, endDate, dataStartDate: website.dataStartDate }, + { ...filters, websiteId, startDate: max([startDate, website.resetAt]), endDate }, ); } @@ -113,12 +111,11 @@ async function clickhouseQuery( count(*) as total from event_data where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} group by event_key, data_type, event_name order by 1 asc, 2 asc limit 100 `, - { websiteId, startDate, endDate, dataStartDate: website.dataStartDate }, + { websiteId, startDate: max([startDate, website.resetAt]), endDate }, ); } diff --git a/queries/analytics/eventData/getEventDataFields.ts b/queries/analytics/eventData/getEventDataFields.ts index ea58d336..65d2669d 100644 --- a/queries/analytics/eventData/getEventDataFields.ts +++ b/queries/analytics/eventData/getEventDataFields.ts @@ -3,7 +3,7 @@ import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { WebsiteEventDataFields } from 'lib/types'; import { loadWebsite } from 'lib/query'; -import { DEFAULT_RESET_DATE } from 'lib/constants'; +import { max } from 'date-fns'; export async function getEventDataFields( ...args: [websiteId: string, startDate: Date, endDate: Date, field?: string] @@ -28,13 +28,12 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date from event_data where website_id = {{websiteId::uuid}} and event_key = {{field}} - and created_at >= {{dataStartDate}} and created_at between {{startDate}} and {{endDate}} group by event_key, string_value order by 3 desc, 2 desc, 1 asc limit 100 `, - { websiteId, field, startDate, endDate, dataStartDate: website.dataStartDate }, + { websiteId, field, startDate: max([startDate, website.resetAt]), endDate }, ); } @@ -46,13 +45,12 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date count(*) as total from event_data where website_id = {{websiteId::uuid}} - and created_at >= {{dataStartDate}} and created_at between {{startDate}} and {{endDate}} group by event_key, data_type order by 3 desc, 2 asc, 1 asc limit 100 `, - { websiteId, startDate, endDate, dataStartDate: website.dataStartDate }, + { websiteId, startDate: max([startDate, website.resetAt]), endDate }, ); } @@ -70,13 +68,12 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date from event_data where website_id = {websiteId:UUID} and event_key = {field:String} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} group by event_key, string_value order by 3 desc, 2 desc, 1 asc limit 100 `, - { websiteId, field, startDate, endDate, dataStartDate: website.dataStartDate }, + { websiteId, field, startDate: max([startDate, website.resetAt]), endDate }, ); } @@ -88,12 +85,11 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date count(*) as total from event_data where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} group by event_key, data_type order by 3 desc, 2 asc, 1 asc limit 100 `, - { websiteId, startDate, endDate, dataStartDate: website.dataStartDate }, + { websiteId, startDate: max([startDate, website.resetAt]), endDate }, ); } diff --git a/queries/analytics/events/getEventMetrics.ts b/queries/analytics/events/getEventMetrics.ts index 60d207ee..0dcb68e7 100644 --- a/queries/analytics/events/getEventMetrics.ts +++ b/queries/analytics/events/getEventMetrics.ts @@ -4,6 +4,7 @@ import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { WebsiteEventMetric } from 'lib/types'; import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; +import { max } from 'date-fns'; export async function getEventMetrics( ...args: [ @@ -57,7 +58,6 @@ async function relationalQuery( count(*) y from website_event where website_id = {{websiteId::uuid}} - and created_at >= {{dataStartDate}} and created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} ${filterQuery} @@ -67,9 +67,8 @@ async function relationalQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: EVENT_TYPE.customEvent, }, ); @@ -106,7 +105,6 @@ async function clickhouseQuery( count(*) y from website_event where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} ${filterQuery} @@ -116,9 +114,8 @@ async function clickhouseQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: EVENT_TYPE.customEvent, }, ); diff --git a/queries/analytics/pageviews/getPageviewMetrics.ts b/queries/analytics/pageviews/getPageviewMetrics.ts index 90cf3aac..d792b82f 100644 --- a/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/queries/analytics/pageviews/getPageviewMetrics.ts @@ -1,8 +1,9 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; -import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; +import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; +import { max } from 'date-fns'; export async function getPageviewMetrics( ...args: [ @@ -35,9 +36,8 @@ async function relationalQuery( const website = await loadWebsite(websiteId); const params: any = { websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, }; @@ -58,7 +58,6 @@ async function relationalQuery( from website_event ${joinSession} where website_event.website_id = {{websiteId::uuid}} - and website_event.created_at >= {{dataStartDate}} and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} ${excludeDomain} @@ -85,9 +84,8 @@ async function clickhouseQuery( const website = await loadWebsite(websiteId); const params = { websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, domain: undefined, }; @@ -106,7 +104,6 @@ async function clickhouseQuery( select ${column} x, count(*) y from website_event where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} ${excludeDomain} diff --git a/queries/analytics/pageviews/getPageviewStats.ts b/queries/analytics/pageviews/getPageviewStats.ts index 8b20a0ca..f0aed17d 100644 --- a/queries/analytics/pageviews/getPageviewStats.ts +++ b/queries/analytics/pageviews/getPageviewStats.ts @@ -3,6 +3,7 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; +import { max } from 'date-fns'; export async function getPageviewStats( ...args: [ @@ -57,7 +58,6 @@ async function relationalQuery( from website_event ${joinSession} where website_event.website_id = {{websiteId::uuid}} - and website_event.created_at >= {{dataStartDate}} and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} ${filterQuery} @@ -66,9 +66,8 @@ async function relationalQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: EVENT_TYPE.pageView, }, ); @@ -109,7 +108,6 @@ async function clickhouseQuery( count(${count !== '*' ? 'distinct session_id' : count}) as y from website_event where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} ${filterQuery} @@ -120,10 +118,9 @@ async function clickhouseQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, eventType: EVENT_TYPE.pageView, - dataStartDate: website.dataStartDate, }, ); } diff --git a/queries/analytics/sessions/getSessionMetrics.ts b/queries/analytics/sessions/getSessionMetrics.ts index 389ac8c3..4a28afe1 100644 --- a/queries/analytics/sessions/getSessionMetrics.ts +++ b/queries/analytics/sessions/getSessionMetrics.ts @@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; +import { max } from 'date-fns'; export async function getSessionMetrics( ...args: [ @@ -35,14 +36,13 @@ async function relationalQuery( on website_event.website_id = website.website_id ${joinSession} where website.website_id = {{websiteId::uuid}} - and website_event.created_at >= {{dataStartDate}} and website_event.created_at between {{startDate}} and {{endDate}} ${filterQuery} ) group by 1 order by 2 desc limit 100`, - { ...filters, websiteId, startDate, endDate, dataStartDate: website.dataStartDate }, + { ...filters, websiteId, startDate: max([startDate, website.resetAt]), endDate }, ); } @@ -61,7 +61,6 @@ async function clickhouseQuery( ${column} x, count(distinct session_id) y from website_event as x where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} ${filterQuery} @@ -72,9 +71,8 @@ async function clickhouseQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: EVENT_TYPE.pageView, }, ); diff --git a/queries/analytics/stats/getWebsiteDateRange.ts b/queries/analytics/stats/getWebsiteDateRange.ts index f3da09c2..accb4021 100644 --- a/queries/analytics/stats/getWebsiteDateRange.ts +++ b/queries/analytics/stats/getWebsiteDateRange.ts @@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { loadWebsite } from 'lib/query'; import { DEFAULT_RESET_DATE } from 'lib/constants'; +import { max } from 'date-fns'; export async function getWebsiteDateRange(...args: [websiteId: string]) { return runQuery({ @@ -40,8 +41,8 @@ async function clickhouseQuery(websiteId: string) { max(created_at) as max from website_event where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} + and created_at >= {startDate:DateTime} `, - { websiteId, dataStartDate: website.dataStartDate }, + { websiteId, startDate: max([new Date('2000-01-01'), website.resetAt]) }, ); } diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index f9f88a87..fac9b8e0 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -3,6 +3,7 @@ import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; +import { max } from 'date-fns'; export async function getWebsiteStats( ...args: [ @@ -44,7 +45,6 @@ async function relationalQuery( ${joinSession} where event_type = {{eventType}} and website.website_id = {{websiteId::uuid}} - and website_event.created_at >= {{dataStartDate}} and website_event.created_at between {{startDate}} and {{endDate}} ${filterQuery} group by 1, 2 @@ -53,9 +53,8 @@ async function relationalQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: EVENT_TYPE.pageView, }, ); @@ -86,7 +85,6 @@ async function clickhouseQuery( max(created_at) max_time from website_event where website_id = {websiteId:UUID} - and created_at >= {dataStartDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} ${filterQuery} @@ -96,9 +94,8 @@ async function clickhouseQuery( { ...filters, websiteId, - startDate, + startDate: max([startDate, website.resetAt]), endDate, - dataStartDate: website.dataStartDate, eventType: EVENT_TYPE.pageView, }, );