m
/
umami
mirror of https://github.com/kremalicious/umami.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Database refactoring.

This commit is contained in:
Mike Cao 2022-08-27 21:38:35 -07:00
parent bb184dc2cc
commit 467c7f289f
37 changed files with 566 additions and 591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
lib/clickhouse.js
View File

@ -1,40 +1,51 @@
import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import debug from 'debug';
import { FILTER_IGNORED } from 'lib/constants';
import { CLICKHOUSE_DATE_FORMATS } from './constants';
import { CLICKHOUSE } from 'lib/db';
export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d',
month: '%Y-%m-01',
year: '%Y-01-01',
};
const log = debug('clickhouse');
function getClient() {
if (!process.env.ANALYTICS_URL) {
if (!process.env.CLICKHOUSE_URL) {
return null;
}
const url = new URL(process.env.ANALYTICS_URL);
const database = url.pathname.replace('/', '');
const {
hostname,
port,
pathname,
username = 'default',
password,
} = new URL(process.env.CLICKHOUSE_URL);
return new ClickHouse({
url: url.hostname,
port: Number(url.port),
basicAuth: url.password
? {
username: url.username || 'default',
password: url.password,
}
: null,
const client = new ClickHouse({
url: hostname,
port: Number(port),
format: 'json',
config: {
database,
database: pathname.replace('/', ''),
},
basicAuth: password ? { username, password } : null,
});
if (process.env.NODE_ENV !== 'production') {
global[CLICKHOUSE] = clickhouse;
}
log('Clickhouse initialized');
return client;
}
const clickhouse = global.clickhouse || getClient();
if (process.env.NODE_ENV !== 'production') {
global.clickhouse = clickhouse;
}
export { clickhouse };
function getDateStringQuery(data, unit) {
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
}
@ -176,7 +187,12 @@ async function findFirst(data) {
return data[0] ?? null;
}
// Initialization
const clickhouse = global[CLICKHOUSE] || getClient();
export default {
client: clickhouse,
log,
getDateStringQuery,
getDateQuery,
getDateFormat,

31
lib/constants.js
View File

@ -67,36 +67,6 @@ export const EVENT_COLORS = [
'#ffec16',
];
export const RELATIONAL = 'relational';
export const POSTGRESQL = 'postgresql';
export const MYSQL = 'mysql';
export const CLICKHOUSE = 'clickhouse';
export const KAFKA = 'kafka';
export const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d',
month: '%Y-%m-01',
year: '%Y-01-01',
};
export const POSTGRESQL_DATE_FORMATS = {
minute: 'YYYY-MM-DD HH24:MI:00',
hour: 'YYYY-MM-DD HH24:00:00',
day: 'YYYY-MM-DD',
month: 'YYYY-MM-01',
year: 'YYYY-01-01',
};
export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d',
month: '%Y-%m-01',
year: '%Y-01-01',
};
export const FILTER_IGNORED = Symbol.for('filter-ignored');
export const DOMAIN_REGEX =
@ -107,6 +77,7 @@ export const LAPTOP_SCREEN_WIDTH = 1024;
export const MOBILE_SCREEN_WIDTH = 479;
export const URL_LENGTH = 500;
export const EVENT_NAME_LENGTH = 50;
export const DESKTOP_OS = [
'Windows 3.11',

14
lib/db.js
View File

@ -1,5 +1,11 @@
import { POSTGRESQL, RELATIONAL, MYSQL, KAFKA, CLICKHOUSE } from 'lib/constants';
export const PRISMA = 'prisma';
export const POSTGRESQL = 'postgresql';
export const MYSQL = 'mysql';
export const CLICKHOUSE = 'clickhouse';
export const KAFKA = 'kafka';
export const KAFKA_PRODUCER = 'kafka-producer';
// Fixes issue with converting bigint values
BigInt.prototype.toJSON = function () {
return Number(this);
};
@ -14,11 +20,11 @@ export function getDatabaseType(url = process.env.DATABASE_URL) {
return type;
}
export async function runAnalyticsQuery(queries) {
const db = getDatabaseType(process.env.ANALYTICS_URL || process.env.DATABASE_URL);
export async function runQuery(queries) {
const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL);
if (db === POSTGRESQL || db === MYSQL) {
return queries[RELATIONAL]();
return queries[PRISMA]();
}
if (db === CLICKHOUSE) {

2
lib/format.js
View File

@ -74,7 +74,7 @@ export function stringToColor(str) {
let color = '#';
for (let i = 0; i < 3; i++) {
let value = (hash >> (i * 8)) & 0xff;
color += ('00' + value.toString(16)).substr(-2);
color += ('00' + value.toString(16)).substring(-2);
}
return color;
}

82
lib/kafka.js
View File

@ -1,57 +1,53 @@
import { Kafka, logLevel } from 'kafkajs';
import dateFormat from 'dateformat';
import debug from 'debug';
import { KAFKA, KAFKA_PRODUCER } from 'lib/db';
export function getClient() {
if (!process.env.KAFKA_URL) {
const log = debug('kafka');
function getClient() {
if (!process.env.KAFKA_URL || !process.env.KAFKA_BROKER) {
return null;
}
const url = new URL(process.env.KAFKA_URL);
const { username, password } = new URL(process.env.KAFKA_URL);
const brokers = process.env.KAFKA_BROKER.split(',');
if (url.username.length === 0 && url.password.length === 0) {
return new Kafka({
clientId: 'umami',
brokers: brokers,
connectionTimeout: 3000,
logLevel: logLevel.ERROR,
});
} else {
return new Kafka({
clientId: 'umami',
brokers: brokers,
connectionTimeout: 3000,
ssl: true,
sasl: {
mechanism: 'plain',
username: url.username,
password: url.password,
},
});
}
}
const kafka = global.kafka || getClient();
let kafkaProducer = null;
const ssl =
username && password
? {
ssl: true,
sasl: {
mechanism: 'plain',
username,
password,
},
}
: {};
(async () => {
if (kafka) {
kafkaProducer = global.kakfaProducer || (await getProducer());
}
const client = new Kafka({
clientId: 'umami',
brokers: brokers,
connectionTimeout: 3000,
logLevel: logLevel.ERROR,
...ssl,
});
if (process.env.NODE_ENV !== 'production') {
global.kafka = kafka;
if (kafka) {
global.kakfaProducer = kafkaProducer;
}
global[KAFKA] = client;
}
})();
export { kafka, kafkaProducer };
return client;
}
async function getProducer() {
const producer = kafka.producer();
await producer.connect();
if (process.env.NODE_ENV !== 'production') {
global[KAFKA_PRODUCER] = producer;
}
return producer;
}
@ -60,7 +56,7 @@ function getDateFormat(date) {
}
async function sendMessage(params, topic) {
await kafkaProducer.send({
await producer.send({
topic,
messages: [
{
@ -72,7 +68,19 @@ async function sendMessage(params, topic) {
});
}
// Initialization
let kafka;
let producer;
(async () => {
kafka = global[KAFKA] || getClient();
producer = global[KAFKA_PRODUCER] || (await getProducer());
})();
export default {
client: kafka,
producer: producer,
log,
getDateFormat,
sendMessage,
};

71
lib/relational.js → lib/prisma.js
View File

@ -1,19 +1,30 @@
import { PrismaClient } from '@prisma/client';
import chalk from 'chalk';
import {
FILTER_IGNORED,
MYSQL,
MYSQL_DATE_FORMATS,
POSTGRESQL,
POSTGRESQL_DATE_FORMATS,
} from 'lib/constants';
import { getDatabaseType } from 'lib/db';
import moment from 'moment-timezone';
import debug from 'debug';
import { PRISMA, MYSQL, POSTGRESQL } from 'lib/db';
import { FILTER_IGNORED } from 'lib/constants';
import { getDatabaseType } from 'lib/db';
const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d',
month: '%Y-%m-01',
year: '%Y-01-01',
};
const POSTGRESQL_DATE_FORMATS = {
minute: 'YYYY-MM-DD HH24:MI:00',
hour: 'YYYY-MM-DD HH24:00:00',
day: 'YYYY-MM-DD',
month: 'YYYY-MM-01',
year: 'YYYY-01-01',
};
const log = debug('prisma');
const options = {
const PRISMA_OPTIONS = {
log: [
{
emit: 'event',
@ -33,16 +44,16 @@ function getClient(options) {
prisma.$on('query', logQuery);
}
if (process.env.NODE_ENV !== 'production') {
global[PRISMA] = prisma;
}
log('Prisma initialized');
return prisma;
}
let prisma = global.prisma || getClient(options);
if (process.env.NODE_ENV !== 'production') {
global.prisma = prisma;
}
export function getDateQuery(field, unit, timezone) {
function getDateQuery(field, unit, timezone) {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
@ -63,7 +74,7 @@ export function getDateQuery(field, unit, timezone) {
}
}
export function getTimestampInterval(field) {
function getTimestampInterval(field) {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
@ -75,7 +86,7 @@ export function getTimestampInterval(field) {
}
}
export function getFilterQuery(table, column, filters = {}, params = []) {
function getFilterQuery(table, column, filters = {}, params = []) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
@ -135,7 +146,7 @@ export function getFilterQuery(table, column, filters = {}, params = []) {
return query.join('\n');
}
export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
filters;
@ -158,13 +169,7 @@ export function parseFilters(table, column, filters = {}, params = [], sessionKe
};
}
export async function runQuery(query) {
return query.catch(e => {
throw e;
});
}
export async function rawQuery(query, params = []) {
async function rawQuery(query, params = []) {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db !== POSTGRESQL && db !== MYSQL) {
@ -173,17 +178,23 @@ export async function rawQuery(query, params = []) {
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
return prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]);
}
export { prisma };
async function multiQuery(queries) {
return prisma.$transaction(queries);
}
// Initialization
const prisma = global[PRISMA] || getClient(PRISMA_OPTIONS);
export default {
prisma,
client: prisma,
log,
getDateQuery,
getTimestampInterval,
getFilterQuery,
parseFilters,
runQuery,
rawQuery,
multiQuery,
};

43
lib/redis.js
View File

@ -1,6 +1,10 @@
import { createClient } from 'redis';
import { startOfMonth } from 'date-fns';
import { getSessions, getAllWebsites } from '/queries';
import debug from 'debug';
const log = debug('db:redis');
const REDIS = Symbol.for('redis');
async function getClient() {
const redis = new createClient({
@ -10,30 +14,16 @@ async function getClient() {
await redis.connect();
if (process.env.LOG_QUERY) {
redis.on('error', err => console.log('Redis Client Error', err));
redis.on('error', err => log(err));
}
if (process.env.NODE_ENV !== 'production') {
global[REDIS] = redis;
}
return redis;
}
let redis = null;
(async () => {
redis = global.redis || (await getClient());
if (process.env.NODE_ENV !== 'production') {
global.redis = redis;
}
const value = await redis.get('initialized');
if (!value) {
await stageData();
}
})();
export default redis;
async function stageData() {
const sessions = await getSessions([], startOfMonth(new Date()).toUTCString());
const websites = await getAllWebsites();
@ -57,3 +47,18 @@ async function addRedis(ids) {
await redis.set(key, value);
}
}
// Initialization
let redis = null;
(async () => {
redis = global[REDIS] || (await getClient());
const value = await redis.get('initialized');
if (!value) {
await stageData();
}
})();
export default redis;

10
queries/admin/account/createAccount.js
View File

@ -1,9 +1,7 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function createAccount(data) {
return runQuery(
prisma.account.create({
data,
}),
);
return prisma.client.account.create({
data,
});
}

50
queries/admin/account/deleteAccount.js
View File

@ -1,28 +1,28 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function deleteAccount(user_id) {
return runQuery(
prisma.$transaction([
prisma.pageview.deleteMany({
where: { session: { website: { user_id } } },
}),
prisma.event_data.deleteMany({
where: { event: { session: { website: { user_id } } } },
}),
prisma.event.deleteMany({
where: { session: { website: { user_id } } },
}),
prisma.session.deleteMany({
where: { website: { user_id } },
}),
prisma.website.deleteMany({
where: { user_id },
}),
prisma.account.delete({
where: {
user_id,
},
}),
]),
);
const { client } = prisma;
return client.$transaction([
client.pageview.deleteMany({
where: { session: { website: { user_id } } },
}),
client.event_data.deleteMany({
where: { event: { session: { website: { user_id } } } },
}),
client.event.deleteMany({
where: { session: { website: { user_id } } },
}),
client.session.deleteMany({
where: { website: { user_id } },
}),
client.website.deleteMany({
where: { user_id },
}),
client.account.delete({
where: {
user_id,
},
}),
]);
}

14
queries/admin/account/getAccountById.js
View File

@ -1,11 +1,9 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getAccountById(user_id) {
return runQuery(
prisma.account.findUnique({
where: {
user_id,
},
}),
);
return prisma.client.account.findUnique({
where: {
user_id,
},
});
}

14
queries/admin/account/getAccountByUsername.js
View File

@ -1,11 +1,9 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getAccountByUsername(username) {
return runQuery(
prisma.account.findUnique({
where: {
username,
},
}),
);
return prisma.client.account.findUnique({
where: {
username,
},
});
}

32
queries/admin/account/getAccounts.js
View File

@ -1,21 +1,19 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getAccounts() {
return runQuery(
prisma.account.findMany({
orderBy: [
{ is_admin: 'desc' },
{
username: 'asc',
},
],
select: {
user_id: true,
username: true,
is_admin: true,
created_at: true,
updated_at: true,
return prisma.client.account.findMany({
orderBy: [
{ is_admin: 'desc' },
{
username: 'asc',
},
}),
);
],
select: {
user_id: true,
username: true,
is_admin: true,
created_at: true,
updated_at: true,
},
});
}

16
queries/admin/account/updateAccount.js
View File

@ -1,12 +1,10 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function updateAccount(user_id, data) {
return runQuery(
prisma.account.update({
where: {
user_id,
},
data,
}),
);
return prisma.client.account.update({
where: {
user_id,
},
data,
});
}

20
queries/admin/website/createWebsite.js
View File

@ -1,9 +1,9 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
import redis from 'lib/redis';
export async function createWebsite(user_id, data) {
return runQuery(
prisma.website.create({
return prisma.client.website
.create({
data: {
account: {
connect: {
@ -12,12 +12,12 @@ export async function createWebsite(user_id, data) {
},
...data,
},
}),
).then(async res => {
if (process.env.REDIS_URL) {
await redis.set(`website:${res.website_uuid}`, Number(res.website_id));
}
})
.then(async res => {
if (process.env.REDIS_URL) {
await redis.set(`website:${res.website_uuid}`, Number(res.website_id));
}
return res;
});
return res;
});
}

50
queries/admin/website/deleteWebsite.js
View File

@ -1,30 +1,28 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
import redis from 'lib/redis';
export async function deleteWebsite(website_id) {
return runQuery(
prisma
.$transaction([
prisma.pageview.deleteMany({
where: { session: { website: { website_id } } },
}),
prisma.event_data.deleteMany({
where: { event: { session: { website: { website_id } } } },
}),
prisma.event.deleteMany({
where: { session: { website: { website_id } } },
}),
prisma.session.deleteMany({
where: { website: { website_id } },
}),
prisma.website.delete({
where: { website_id },
}),
])
.then(async res => {
if (process.env.REDIS_URL) {
await redis.del(`website:${res.website_uuid}`);
}
}),
);
const { client, multiQuery } = prisma;
return multiQuery([
client.pageview.deleteMany({
where: { session: { website: { website_id } } },
}),
client.event_data.deleteMany({
where: { event: { session: { website: { website_id } } } },
}),
client.event.deleteMany({
where: { session: { website: { website_id } } },
}),
client.session.deleteMany({
where: { website: { website_id } },
}),
client.website.delete({
where: { website_id },
}),
]).then(async res => {
if (process.env.REDIS_URL) {
await redis.del(`website:${res.website_uuid}`);
}
});
}

35
queries/admin/website/getAllWebsites.js
View File

@ -1,24 +1,23 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getAllWebsites() {
let data = await runQuery(
prisma.website.findMany({
orderBy: [
{
user_id: 'asc',
},
{
name: 'asc',
},
],
include: {
account: {
select: {
username: true,
},
let data = await prisma.client.website.findMany({
orderBy: [
{
user_id: 'asc',
},
{
name: 'asc',
},
],
include: {
account: {
select: {
username: true,
},
},
}),
);
},
});
return data.map(i => ({ ...i, account: i.account.username }));
}

20
queries/admin/website/getUserWebsites.js
View File

@ -1,14 +1,12 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getUserWebsites(user_id) {
return runQuery(
prisma.website.findMany({
where: {
user_id,
},
orderBy: {
name: 'asc',
},
}),
);
return prisma.client.website.findMany({
where: {
user_id,
},
orderBy: {
name: 'asc',
},
});
}

14
queries/admin/website/getWebsiteById.js
View File

@ -1,11 +1,9 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getWebsiteById(website_id) {
return runQuery(
prisma.website.findUnique({
where: {
website_id,
},
}),
);
return prisma.client.website.findUnique({
where: {
website_id,
},
});
}

14
queries/admin/website/getWebsiteByShareId.js
View File

@ -1,11 +1,9 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getWebsiteByShareId(share_id) {
return runQuery(
prisma.website.findUnique({
where: {
share_id,
},
}),
);
return prisma.client.website.findUnique({
where: {
share_id,
},
});
}

14
queries/admin/website/getWebsiteByUuid.js
View File

@ -1,11 +1,9 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function getWebsiteByUuid(website_uuid) {
return runQuery(
prisma.website.findUnique({
where: {
website_uuid,
},
}),
);
return prisma.client.website.findUnique({
where: {
website_uuid,
},
});
}

24
queries/admin/website/resetWebsite.js
View File

@ -1,5 +1,25 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
import redis from 'lib/redis';
export async function resetWebsite(website_id) {
return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`);
const { client, multiQuery } = prisma;
return multiQuery([
client.pageview.deleteMany({
where: { session: { website: { website_id } } },
}),
client.event_data.deleteMany({
where: { event: { session: { website: { website_id } } } },
}),
client.event.deleteMany({
where: { session: { website: { website_id } } },
}),
client.session.deleteMany({
where: { website: { website_id } },
}),
]).then(async res => {
if (process.env.REDIS_URL) {
await redis.del(`website:${res.website_uuid}`);
}
});
}

16
queries/admin/website/updateWebsite.js
View File

@ -1,12 +1,10 @@
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
export async function updateWebsite(website_id, data) {
return runQuery(
prisma.website.update({
where: {
website_id,
},
data,
}),
);
return prisma.client.website.update({
where: {
website_id,
},
data,
});
}

29
queries/analytics/event/getEventMetrics.js
View File

@ -1,11 +1,10 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { getDateQuery, getFilterQuery, rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getEventMetrics(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@ -18,11 +17,11 @@ async function relationalQuery(
unit = 'day',
filters = {},
) {
const { rawQuery, getDateQuery, getFilterQuery } = prisma;
const params = [website_id, start_at, end_at];
return rawQuery(
`
select
`select
event_name x,
${getDateQuery('created_at', unit, timezone)} t,
count(*) y
@ -31,8 +30,7 @@ async function relationalQuery(
and created_at between $2 and $3
${getFilterQuery('event', filters, params)}
group by 1, 2
order by 2
`,
order by 2`,
params,
);
}
@ -45,21 +43,20 @@ async function clickhouseQuery(
unit = 'day',
filters = {},
) {
const { rawQuery, getDateQuery, getBetweenDates } = prisma;
const params = [website_id];
return clickhouse.rawQuery(
`
select
return rawQuery(
`select
event_name x,
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
${getDateQuery('created_at', unit, timezone)} t,
count(*) y
from event
where website_id= $1
and ${clickhouse.getBetweenDates('created_at', start_at, end_at)}
and ${getBetweenDates('created_at', start_at, end_at)}
${clickhouse.getFilterQuery('event', filters, params)}
group by x, t
order by t
`,
order by t`,
params,
);
}

41
queries/analytics/event/getEvents.js
View File

@ -1,36 +1,34 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export function getEvents(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function relationalQuery(websites, start_at) {
return runQuery(
prisma.event.findMany({
where: {
website: {
website_id: {
in: websites,
},
},
created_at: {
gte: start_at,
return prisma.client.event.findMany({
where: {
website: {
website_id: {
in: websites,
},
},
}),
);
created_at: {
gte: start_at,
},
},
});
}
function clickhouseQuery(websites, start_at) {
return clickhouse.rawQuery(
`
select
const { rawQuery, getDateFormat } = clickhouse;
return rawQuery(
`select
event_id,
website_id,
session_id,
@ -39,7 +37,6 @@ function clickhouseQuery(websites, start_at) {
event_name
from event
where website_id in (${websites.join[',']}
and created_at >= ${clickhouse.getDateFormat(start_at)})
`,
and created_at >= ${getDateFormat(start_at)})`,
);
}

43
queries/analytics/event/saveEvent.js
View File

@ -1,12 +1,12 @@
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import kafka from 'lib/kafka';
import { prisma, runQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
import { URL_LENGTH, EVENT_NAME_LENGTH } from 'lib/constants';
export async function saveEvent(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
@ -16,8 +16,8 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_
const data = {
website_id,
session_id,
url: url?.substr(0, URL_LENGTH),
event_name: event_name?.substr(0, 50),
url: url?.substring(0, URL_LENGTH),
event_name: event_name?.substring(0, EVENT_NAME_LENGTH),
};
if (event_data) {
@ -28,39 +28,38 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_
};
}
return runQuery(
prisma.event.create({
data,
}),
);
return prisma.client.event.create({
data,
});
}
async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, event_name }) {
const { rawQuery, getDateFormat } = clickhouse;
const params = [
website_id,
event_uuid,
session_uuid,
url?.substr(0, URL_LENGTH),
event_name?.substr(0, 50),
url?.substring(0, URL_LENGTH),
event_name?.substring(0, EVENT_NAME_LENGTH),
];
return clickhouse.rawQuery(
`
insert into umami.event (created_at, website_id, session_uuid, url, event_name)
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
return rawQuery(
`insert into umami.event (created_at, website_id, session_uuid, url, event_name)
values (${getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_name }) {
const { getDateFormat, sendMessage } = kafka;
const params = {
event_uuid: event_uuid,
website_id: website_id,
session_uuid: session_uuid,
created_at: kafka.getDateFormat(new Date()),
url: url?.substr(0, URL_LENGTH),
event_name: event_name?.substr(0, 50),
created_at: getDateFormat(new Date()),
url: url?.substring(0, URL_LENGTH),
event_name: event_name?.substring(0, EVENT_NAME_LENGTH),
};
await kafka.sendMessage(params, 'event');
await sendMessage(params, 'event');
}

29
queries/analytics/pageview/getPageviewMetrics.js
View File

@ -1,16 +1,16 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { parseFilters, rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewMetrics(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {
const { rawQuery, parseFilters } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
);
return rawQuery(
`
select ${column} x, count(*) y
`select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id=$1
@ -30,15 +29,15 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
${joinSession && sessionQuery}
${eventQuery}
group by 1
order by 2 desc
`,
order by 2 desc`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) {
const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters(
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
column,
filters,
@ -46,19 +45,17 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt
'session_uuid',
);
return clickhouse.rawQuery(
`
select ${column} x, count(*) y
return rawQuery(
`select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id= $1
and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)}
and ${getBetweenDates(table + '.created_at', start_at, end_at)}
${pageviewQuery}
${joinSession && sessionQuery}
${eventQuery}
group by x
order by y desc
`,
order by y desc`,
params,
);
}

15
queries/analytics/pageview/getPageviewParams.js
View File

@ -1,14 +1,15 @@
import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/relational';
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewParams(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {
const { parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
@ -18,8 +19,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
);
return rawQuery(
`
select url x,
`select url x,
count(*) y
from ${table}
${joinSession}
@ -30,8 +30,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
${joinSession && sessionQuery}
${eventQuery}
group by 1
order by 2 desc
`,
order by 2 desc`,
params,
);
}

31
queries/analytics/pageview/getPageviewStats.js
View File

@ -1,11 +1,10 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { getDateQuery, parseFilters, rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewStats(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@ -20,6 +19,7 @@ async function relationalQuery(
filters = {},
sessionKey = 'session_id',
) {
const { getDateQuery, parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@ -29,8 +29,7 @@ async function relationalQuery(
);
return rawQuery(
`
select ${getDateQuery('pageview.created_at', unit, timezone)} t,
`select ${getDateQuery('pageview.created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
@ -38,8 +37,7 @@ async function relationalQuery(
and pageview.created_at between $2 and $3
${pageviewQuery}
${sessionQuery}
group by 1
`,
group by 1`,
params,
);
}
@ -54,6 +52,7 @@ async function clickhouseQuery(
filters = {},
sessionKey = 'session_uuid',
) {
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@ -63,24 +62,22 @@ async function clickhouseQuery(
sessionKey,
);
return clickhouse.rawQuery(
`
select
${clickhouse.getDateStringQuery('g.t', unit)} as t,
return rawQuery(
`select
${getDateStringQuery('g.t', unit)} as t,
g.y as y
from
(select
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
${getDateQuery('created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
where pageview.website_id= $1
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
and ${getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by t) g
order by t
`,
order by t`,
params,
);
}

37
queries/analytics/pageview/getPageviews.js
View File

@ -1,36 +1,32 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { prisma, runQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviews(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websites, start_at) {
return runQuery(
prisma.pageview.findMany({
where: {
website: {
website_id: {
in: websites,
},
},
created_at: {
gte: start_at,
return prisma.client.pageview.findMany({
where: {
website: {
website_id: {
in: websites,
},
},
}),
);
created_at: {
gte: start_at,
},
},
});
}
async function clickhouseQuery(websites, start_at) {
return clickhouse.rawQuery(
`
select
`select
view_id,
website_id,
session_id,
@ -38,7 +34,6 @@ async function clickhouseQuery(websites, start_at) {
url
from pageview
where website_id in (${websites.join[',']}
and created_at >= ${clickhouse.getDateFormat(start_at)})
`,
and created_at >= ${clickhouse.getDateFormat(start_at)})`,
);
}

44
queries/analytics/pageview/savePageView.js
View File

@ -1,54 +1,52 @@
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db';
import kafka from 'lib/kafka';
import { prisma, runQuery } from 'lib/relational';
import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
import { URL_LENGTH } from 'lib/constants';
export async function savePageView(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
async function relationalQuery(website_id, { session_id, url, referrer }) {
return runQuery(
prisma.pageview.create({
data: {
website_id,
session_id,
url: url?.substr(0, URL_LENGTH),
referrer: referrer?.substr(0, URL_LENGTH),
},
}),
);
return prisma.client.pageview.create({
data: {
website_id,
session_id,
url: url?.substring(0, URL_LENGTH),
referrer: referrer?.substring(0, URL_LENGTH),
},
});
}
async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {
const params = [
website_id,
session_uuid,
url?.substr(0, URL_LENGTH),
referrer?.substr(0, URL_LENGTH),
url?.substring(0, URL_LENGTH),
referrer?.substring(0, URL_LENGTH),
];
return clickhouse.rawQuery(
`
insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
`insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { session_uuid, url, referrer }) {
const { getDateFormat, sendMessage } = kafka;
const params = {
website_id: website_id,
session_uuid: session_uuid,
created_at: kafka.getDateFormat(new Date()),
url: url?.substr(0, URL_LENGTH),
referrer: referrer?.substr(0, URL_LENGTH),
created_at: getDateFormat(new Date()),
url: url?.substring(0, URL_LENGTH),
referrer: referrer?.substring(0, URL_LENGTH),
};
await kafka.sendMessage(params, 'pageview');
await sendMessage(params, 'pageview');
}

37
queries/analytics/session/createSession.js
View File

@ -1,21 +1,20 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';
import { runAnalyticsQuery } from 'lib/db';
import kafka from 'lib/kafka';
import redis from 'lib/redis';
import { prisma, runQuery } from 'lib/relational';
import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
export async function createSession(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
async function relationalQuery(website_id, data) {
return runQuery(
prisma.session.create({
return prisma.client.session
.create({
data: {
website_id,
...data,
@ -23,14 +22,14 @@ async function relationalQuery(website_id, data) {
select: {
session_id: true,
},
}),
).then(async res => {
if (process.env.REDIS_URL) {
await redis.set(`session:${res.session_uuid}`, '');
}
})
.then(async res => {
if (process.env.REDIS_URL) {
await redis.set(`session:${res.session_uuid}`, '');
}
return res;
});
return res;
});
}
async function clickhouseQuery(
@ -48,10 +47,11 @@ async function clickhouseQuery(
language,
country ? country : null,
];
const { rawQuery, getDateFormat } = clickhouse;
await clickhouse.rawQuery(
await rawQuery(
`insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
values (${getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
params,
);
}
@ -60,10 +60,11 @@ async function kafkaQuery(
website_id,
{ session_uuid, hostname, browser, os, screen, language, country, device },
) {
const { getDateFormat, sendMessage } = kafka;
const params = {
session_uuid: session_uuid,
website_id: website_id,
created_at: kafka.getDateFormat(new Date()),
created_at: getDateFormat(new Date()),
hostname: hostname,
browser: browser,
os: os,
@ -73,7 +74,7 @@ async function kafkaQuery(
country: country ? country : null,
};
await kafka.sendMessage(params, 'session');
await sendMessage(params, 'session');
await redis.set(`session:${session_uuid}`, '');
}

38
queries/analytics/session/getSessionByUuid.js
View File

@ -1,32 +1,28 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { prisma, runQuery } from 'lib/relational';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getSessionByUuid(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(session_uuid) {
return runQuery(
prisma.session.findUnique({
where: {
session_uuid,
},
}),
);
return prisma.client.session.findUnique({
where: {
session_uuid,
},
});
}
async function clickhouseQuery(session_uuid) {
const { rawQuery, findFirst } = clickhouse;
const params = [session_uuid];
return clickhouse
.rawQuery(
`
select
return rawQuery(
`select
session_uuid,
website_id,
created_at,
@ -35,12 +31,10 @@ async function clickhouseQuery(session_uuid) {
os,
device,
screen,
"language",
language,
country
from session
where session_uuid = $1
`,
params,
)
.then(result => clickhouse.findFirst(result));
where session_uuid = $1`,
params,
).then(result => findFirst(result));
}

29
queries/analytics/session/getSessionMetrics.js
View File

@ -1,16 +1,16 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db';
import { parseFilters, rawQuery } from 'lib/relational';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getSessionMetrics(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, field, filters = {}) {
const { parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}
);
return rawQuery(
`
select ${field} x, count(*) y
`select ${field} x, count(*) y
from session as x
where x.session_id in (
select pageview.session_id
@ -33,15 +32,15 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}
${sessionQuery}
)
group by 1
order by 2 desc
`,
order by 2 desc`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) {
const { parseFilters, getBetweenDates, rawQuery } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
null,
filters,
@ -49,22 +48,20 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}
'session_uuid',
);
return clickhouse.rawQuery(
`
select ${field} x, count(*) y
return rawQuery(
`select ${field} x, count(*) y
from session as x
where x.session_uuid in (
select pageview.session_uuid
from pageview
${joinSession}
where pageview.website_id=$1
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
and ${getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
)
group by x
order by y desc
`,
order by y desc`,
params,
);
}

23
queries/analytics/session/getSessions.js
View File

@ -1,18 +1,17 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/relational';
import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';
export async function getSessions(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websites, start_at) {
return runQuery(
prisma.session.findMany({
prisma.client.session.findMany({
where: {
...(websites && websites.length > 0
? {
@ -32,9 +31,10 @@ async function relationalQuery(websites, start_at) {
}
async function clickhouseQuery(websites, start_at) {
return clickhouse.rawQuery(
`
select
const { rawQuery, getDateFormat } = clickhouse;
return rawQuery(
`select
session_uuid,
website_id,
created_at,
@ -43,11 +43,10 @@ async function clickhouseQuery(websites, start_at) {
os,
device,
screen,
"language",
language,
country
from session
where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'}
and created_at >= ${clickhouse.getDateFormat(start_at)}
`,
and created_at >= ${getDateFormat(start_at)}`,
);
}

26
queries/analytics/stats/getActiveVisitors.js
View File

@ -1,12 +1,11 @@
import { subMinutes } from 'date-fns';
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getActiveVisitors(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@ -15,27 +14,24 @@ async function relationalQuery(website_id) {
const date = subMinutes(new Date(), 5);
const params = [website_id, date];
return rawQuery(
`
select count(distinct session_id) x
return prisma.rawQuery(
`select count(distinct session_id) x
from pageview
where website_id = $1
and created_at >= $2
`,
and created_at >= $2`,
params,
);
}
async function clickhouseQuery(website_id) {
const { rawQuery, getDateFormat } = clickhouse;
const params = [website_id];
return clickhouse.rawQuery(
`
select count(distinct session_uuid) x
return rawQuery(
`select count(distinct session_uuid) x
from pageview
where website_id = $1
and created_at >= ${clickhouse.getDateFormat(subMinutes(new Date(), 5))}
`,
and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`,
params,
);
}

83
queries/analytics/stats/getWebsiteStats.js
View File

@ -1,16 +1,16 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { getDateQuery, getTimestampInterval, parseFilters, rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getWebsiteStats(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, filters = {}) {
const { getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@ -20,32 +20,31 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) {
);
return rawQuery(
`
select sum(t.c) as "pageviews",
`select sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(t.time) as "totaltime"
from (
select pageview.session_id,
${getDateQuery('pageview.created_at', 'hour')},
count(*) c,
${getTimestampInterval('pageview.created_at')} as "time"
from pageview
${joinSession}
where pageview.website_id=$1
and pageview.created_at between $2 and $3
${pageviewQuery}
${sessionQuery}
group by 1, 2
) t
`,
select pageview.session_id,
${getDateQuery('pageview.created_at', 'hour')},
count(*) c,
${getTimestampInterval('pageview.created_at')} as "time"
from pageview
${joinSession}
where pageview.website_id=$1
and pageview.created_at between $2 and $3
${pageviewQuery}
${sessionQuery}
group by 1, 2
) t`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
null,
filters,
@ -53,28 +52,26 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
'session_uuid',
);
return clickhouse.rawQuery(
`
select
sum(t.c) as "pageviews",
count(distinct t.session_uuid) as "uniques",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
from (
select pageview.session_uuid,
${clickhouse.getDateQuery('pageview.created_at', 'day')} time_series,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from pageview
${joinSession}
where pageview.website_id = $1
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by pageview.session_uuid, time_series
) t;
`,
return rawQuery(
`select
sum(t.c) as "pageviews",
count(distinct t.session_uuid) as "uniques",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
from (
select pageview.session_uuid,
${getDateQuery('pageview.created_at', 'day')} time_series,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from pageview
${joinSession}
where pageview.website_id = $1
and ${getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by pageview.session_uuid, time_series
) t;`,
params,
);
}

6
scripts/copy-db-files.js
View File

@ -3,10 +3,8 @@ const fse = require('fs-extra');
const path = require('path');
const del = require('del');
function getDatabaseType() {
const type =
process.env.DATABASE_TYPE ||
(process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]);
function getDatabaseType(url = process.env.DATABASE_URL) {
const type = process.env.DATABASE_TYPE || (url && url.split(':')[0]);
if (type === 'postgres') {
return 'postgresql';