m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2024-11-15 09:45:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

split db files

Browse Source
This commit is contained in:
Brian Cao 2022-08-25 22:04:32 -07:00
parent 46b4b98d40
commit bb50753704
36 changed files with 496 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
db/clickhouse/migrations/01_init/migration.sql
View File

@ -19,7 +19,7 @@ CREATE TABLE pageview_queue (
referrer String
)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'localhost:9092,localhost:9093,localhost:9094', -- input broker list
SETTINGS kafka_broker_list = 'dev-01.umami.dev:9092,dev-01.umami.dev:9093,dev-01.umami.dev:9094', -- input broker list
kafka_topic_list = 'pageview',
kafka_group_name = 'pageview_consumer_group',
kafka_format = 'JSONEachRow',
@ -65,7 +65,7 @@ CREATE TABLE session_queue (
country LowCardinality(String)
)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'localhost:9092,localhost:9093,localhost:9094', -- input broker list
SETTINGS kafka_broker_list = 'dev-01.umami.dev:9092,dev-01.umami.dev:9093,dev-01.umami.dev:9094', -- input broker list
kafka_topic_list = 'session',
kafka_group_name = 'session_consumer_group',
kafka_format = 'JSONEachRow',
@ -110,7 +110,7 @@ CREATE TABLE event_queue (
event_data String
)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'localhost:9092,localhost:9093,localhost:9094', -- input broker list
SETTINGS kafka_broker_list = 'dev-01.umami.dev:9092,dev-01.umami.dev:9093,dev-01.umami.dev:9094', -- input broker list
kafka_topic_list = 'event',
kafka_group_name = 'event_consumer_group',
kafka_format = 'JSONEachRow',

173
lib/clickhouse.js Normal file
View File

@ -0,0 +1,173 @@
import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import { FILTER_IGNORED } from 'lib/constants';
import { CLICKHOUSE_DATE_FORMATS } from './constants';
function getClient() {
if (!process.env.ANALYTICS_URL) {
return null;
}
const url = new URL(process.env.ANALYTICS_URL);
const database = url.pathname.replace('/', '');
return new ClickHouse({
url: url.hostname,
port: Number(url.port),
basicAuth: url.password
? {
username: url.username || 'default',
password: url.password,
}
: null,
format: 'json',
config: {
database,
},
});
}
const clickhouse = global.clickhouse || getClient();
if (process.env.NODE_ENV !== 'production') {
global.clickhouse = clickhouse;
}
export { clickhouse };
export function getDateStringQuery(data, unit) {
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
}
export function getDateQuery(field, unit, timezone) {
if (timezone) {
return `date_trunc('${unit}', ${field}, '${timezone}')`;
}
return `date_trunc('${unit}', ${field})`;
}
export function getDateFormat(date) {
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
}
export function getBetweenDates(field, start_at, end_at) {
return `${field} between ${getDateFormat(start_at)}
and ${getDateFormat(end_at)}`;
}
export function getFilterQuery(table, column, filters = {}, params = []) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter === undefined || filter === FILTER_IGNORED) {
return arr;
}
switch (key) {
case 'url':
if (table === 'pageview' || table === 'event') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'os':
case 'browser':
case 'device':
case 'country':
if (table === 'session') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'event_name':
if (table === 'event') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'referrer':
if (table === 'pageview' || table === 'event') {
arr.push(`and ${table}.referrer like $${params.length + 1}`);
params.push(`%${decodeURIComponent(filter)}%`);
}
break;
case 'domain':
if (table === 'pageview') {
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
arr.push(`and ${table}.referrer not like '/%'`);
params.push(`%://${filter}/%`);
}
break;
case 'query':
if (table === 'pageview') {
arr.push(`and ${table}.url like '%?%'`);
}
}
return arr;
}, []);
return query.join('\n');
}
export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
filters;
const pageviewFilters = { domain, url, referrer, query };
const sessionFilters = { os, browser, device, country };
const eventFilters = { url: event_url, event_name };
return {
pageviewFilters,
sessionFilters,
eventFilters,
event: { event_name },
joinSession:
os || browser || device || country
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
: '',
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
eventQuery: getFilterQuery('event', column, eventFilters, params),
};
}
export function replaceQuery(string, params = []) {
let formattedString = string;
params.forEach((a, i) => {
let replace = a;
if (typeof a === 'string' || a instanceof String) {
replace = `'${replace}'`;
}
formattedString = formattedString.replace(`$${i + 1}`, replace);
});
return formattedString;
}
export async function rawQuery(query, params = [], debug = false) {
let formattedQuery = replaceQuery(query, params);
if (debug || process.env.LOG_QUERY) {
console.log(formattedQuery);
}
return clickhouse.query(formattedQuery).toPromise();
}
export async function findUnique(data) {
if (data.length > 1) {
throw `${data.length} records found when expecting 1.`;
}
return data[0] ?? null;
}

286
lib/db.js
View File

@ -1,305 +1,33 @@
import { PrismaClient } from '@prisma/client';
import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import chalk from 'chalk';
import { getKafkaService } from './db/kafka';
import {
MYSQL,
MYSQL_DATE_FORMATS,
POSTGRESQL,
POSTGRESQL_DATE_FORMATS,
CLICKHOUSE,
RELATIONAL,
FILTER_IGNORED,
KAFKA,
} from 'lib/constants';
import moment from 'moment-timezone';
import { CLICKHOUSE_DATE_FORMATS } from './constants';
import { POSTGRESQL, RELATIONAL, MYSQL, KAFKA } from 'lib/constants';
import { CLICKHOUSE } from 'lib/constants';
BigInt.prototype.toJSON = function () {
return Number(this);
};
const options = {
log: [
{
emit: 'event',
level: 'query',
},
],
};
function logQuery(e) {
console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
}
function getPrismaClient(options) {
const prisma = new PrismaClient(options);
if (process.env.LOG_QUERY) {
prisma.$on('query', logQuery);
}
return prisma;
}
function getClickhouseClient() {
if (!process.env.ANALYTICS_URL) {
return null;
}
const url = new URL(process.env.ANALYTICS_URL);
const database = url.pathname.replace('/', '');
return new ClickHouse({
url: url.hostname,
port: Number(url.port),
basicAuth: url.password
? {
username: url.username || 'default',
password: url.password,
}
: null,
format: 'json',
config: {
database,
},
});
}
const prisma = global.prisma || getPrismaClient(options);
const clickhouse = global.clickhouse || getClickhouseClient();
if (process.env.NODE_ENV !== 'production') {
global.prisma = prisma;
global.clickhouse = clickhouse;
}
export { prisma, clickhouse };
export function getDatabase() {
const type =
process.env.DATABASE_TYPE ||
(process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]);
if (type === 'postgres') {
return POSTGRESQL;
}
return type;
}
export function getAnalyticsDatabase() {
const type = process.env.ANALYTICS_URL && process.env.ANALYTICS_URL.split(':')[0];
export function getDatabase(database, databaseType, fallback) {
const type = databaseType || (database && database.split(':')[0]);
if (type === 'postgres') {
return POSTGRESQL;
}
if (!type) {
return getDatabase();
return getDatabase(fallback);
}
return type;
}
export function getDateStringQueryClickhouse(data, unit) {
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
}
export function getDateQuery(field, unit, timezone) {
const db = getDatabase();
if (db === POSTGRESQL) {
if (timezone) {
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
if (db === MYSQL) {
if (timezone) {
const tz = moment.tz(timezone).format('Z');
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
}
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
}
}
export function getDateQueryClickhouse(field, unit, timezone) {
if (timezone) {
return `date_trunc('${unit}', ${field}, '${timezone}')`;
}
return `date_trunc('${unit}', ${field})`;
}
export function getDateFormatClickhouse(date) {
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
}
export function getBetweenDatesClickhouse(field, start_at, end_at) {
return `${field} between ${getDateFormatClickhouse(start_at)}
and ${getDateFormatClickhouse(end_at)}`;
}
export function getTimestampInterval(field) {
const db = getDatabase();
if (db === POSTGRESQL) {
return `floor(extract(epoch from max(${field}) - min(${field})))`;
}
if (db === MYSQL) {
return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`;
}
}
export function getFilterQuery(table, column, filters = {}, params = []) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter === undefined || filter === FILTER_IGNORED) {
return arr;
}
switch (key) {
case 'url':
if (table === 'pageview' || table === 'event') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'os':
case 'browser':
case 'device':
case 'country':
if (table === 'session') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'event_name':
if (table === 'event') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'referrer':
if (table === 'pageview' || table === 'event') {
arr.push(`and ${table}.referrer like $${params.length + 1}`);
params.push(`%${decodeURIComponent(filter)}%`);
}
break;
case 'domain':
if (table === 'pageview') {
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
arr.push(`and ${table}.referrer not like '/%'`);
params.push(`%://${filter}/%`);
}
break;
case 'query':
if (table === 'pageview') {
arr.push(`and ${table}.url like '%?%'`);
}
}
return arr;
}, []);
return query.join('\n');
}
export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
filters;
const pageviewFilters = { domain, url, referrer, query };
const sessionFilters = { os, browser, device, country };
const eventFilters = { url: event_url, event_name };
return {
pageviewFilters,
sessionFilters,
eventFilters,
event: { event_name },
joinSession:
os || browser || device || country
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
: '',
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
eventQuery: getFilterQuery('event', column, eventFilters, params),
};
}
export function replaceQueryClickhouse(string, params = []) {
let formattedString = string;
params.forEach((a, i) => {
let replace = a;
if (typeof a === 'string' || a instanceof String) {
replace = `'${replace}'`;
}
formattedString = formattedString.replace(`$${i + 1}`, replace);
});
return formattedString;
}
export async function runQuery(query) {
return query.catch(e => {
throw e;
});
}
export async function rawQuery(query, params = []) {
const db = getDatabase();
if (db !== POSTGRESQL && db !== MYSQL) {
return Promise.reject(new Error('Unknown database.'));
}
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
}
export async function rawQueryClickhouse(query, params = [], debug = false) {
let formattedQuery = replaceQueryClickhouse(query, params);
if (debug || process.env.LOG_QUERY) {
console.log(formattedQuery);
}
return clickhouse.query(formattedQuery).toPromise();
}
export async function findUnique(data) {
if (data.length > 1) {
throw `${data.length} records found when expecting 1.`;
}
return data[0] ?? null;
}
export async function runAnalyticsQuery(queries) {
const db = getAnalyticsDatabase();
const db = getDatabase(process.env.ANALYTICS_URL, null, process.env.DATABASE_URL);
if (db === POSTGRESQL || db === MYSQL) {
return queries[RELATIONAL]();
}
if (db === CLICKHOUSE) {
const kafka = getKafkaService();
const kafka = getDatabase(process.env.KAFKA_URL);
if (kafka === KAFKA && queries[KAFKA]) {
return queries[KAFKA]();
}

24
lib/db/kafka.js → lib/kafka.js
View File

@ -1,7 +1,7 @@
import { Kafka, logLevel } from 'kafkajs';
import dateFormat from 'dateformat';
export function getKafkaClient() {
export function getClient() {
if (!process.env.KAFKA_URL) {
return null;
}
@ -30,11 +30,11 @@ export function getKafkaClient() {
});
}
}
const kafka = global.kafka || getKafkaClient();
const kafka = global.kafka || getClient();
let kafkaProducer = null;
(async () => {
kafkaProducer = global.kakfaProducer || (await getKafkaProducer());
kafkaProducer = global.kakfaProducer || (await getProducer());
if (process.env.NODE_ENV !== 'production') {
global.kafka = kafka;
@ -44,14 +44,18 @@ let kafkaProducer = null;
export { kafka, kafkaProducer };
export async function getKafkaProducer() {
export async function getProducer() {
const producer = kafka.producer();
await producer.connect();
return producer;
}
export async function sendKafkaMessage(params, topic) {
export function getDateFormat(date) {
return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
}
export async function sendMessage(params, topic) {
await kafkaProducer.send({
topic,
messages: [
@ -63,13 +67,3 @@ export async function sendKafkaMessage(params, topic) {
acks: 0,
});
}
export function getDateFormatKafka(date) {
return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
}
export function getKafkaService() {
const type = process.env.KAFKA_URL && process.env.KAFKA_URL.split(':')[0];
return type;
}

163
lib/relational.js Normal file
View File

@ -0,0 +1,163 @@
import { PrismaClient } from '@prisma/client';
import chalk from 'chalk';
import {
FILTER_IGNORED,
MYSQL,
MYSQL_DATE_FORMATS,
POSTGRESQL,
POSTGRESQL_DATE_FORMATS,
} from 'lib/constants';
import { getDatabase } from './db';
import moment from 'moment-timezone';
const options = {
log: [
{
emit: 'event',
level: 'query',
},
],
};
function logQuery(e) {
console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
}
function getPrismaClient(options) {
const prisma = new PrismaClient(options);
if (process.env.LOG_QUERY) {
prisma.$on('query', logQuery);
}
return prisma;
}
const prisma = global.prisma || getPrismaClient(options);
if (process.env.NODE_ENV !== 'production') {
global.prisma = prisma;
}
export { prisma };
export function getDateQuery(field, unit, timezone) {
const db = getDatabase();
if (db === POSTGRESQL) {
if (timezone) {
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
if (db === MYSQL) {
if (timezone) {
const tz = moment.tz(timezone).format('Z');
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
}
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
}
}
export function getFilterQuery(table, column, filters = {}, params = []) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
if (filter === undefined || filter === FILTER_IGNORED) {
return arr;
}
switch (key) {
case 'url':
if (table === 'pageview' || table === 'event') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'os':
case 'browser':
case 'device':
case 'country':
if (table === 'session') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'event_name':
if (table === 'event') {
arr.push(`and ${table}.${key}=$${params.length + 1}`);
params.push(decodeURIComponent(filter));
}
break;
case 'referrer':
if (table === 'pageview' || table === 'event') {
arr.push(`and ${table}.referrer like $${params.length + 1}`);
params.push(`%${decodeURIComponent(filter)}%`);
}
break;
case 'domain':
if (table === 'pageview') {
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
arr.push(`and ${table}.referrer not like '/%'`);
params.push(`%://${filter}/%`);
}
break;
case 'query':
if (table === 'pageview') {
arr.push(`and ${table}.url like '%?%'`);
}
}
return arr;
}, []);
return query.join('\n');
}
export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
filters;
const pageviewFilters = { domain, url, referrer, query };
const sessionFilters = { os, browser, device, country };
const eventFilters = { url: event_url, event_name };
return {
pageviewFilters,
sessionFilters,
eventFilters,
event: { event_name },
joinSession:
os || browser || device || country
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
: '',
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
eventQuery: getFilterQuery('event', column, eventFilters, params),
};
}
export async function runQuery(query) {
return query.catch(e => {
throw e;
});
}
export async function rawQuery(query, params = []) {
const db = getDatabase(process.env.DATABASE_URL, process.env.DATABASE_TYPE);
if (db !== POSTGRESQL && db !== MYSQL) {
return Promise.reject(new Error('Unknown database.'));
}
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
}

2
lib/session.js
View File

@ -37,6 +37,8 @@ export async function getSession(req) {
let session = await getSessionByUuid(session_uuid);
console.log('session here!: ', session);
if (!session) {
try {
session = await createSession(website_id, {

39
pages/api/collect.js
View File

@ -4,7 +4,7 @@ import ipaddr from 'ipaddr.js';
import { savePageView, saveEvent } from 'queries';
import { useCors, useSession } from 'lib/middleware';
import { getJsonBody, getIpAddress } from 'lib/request';
import { ok, send, badRequest, forbidden } from 'lib/response';
import { unauthorized, send, badRequest, forbidden } from 'lib/response';
import { createToken } from 'lib/crypto';
import { removeTrailingSlash } from 'lib/url';
import { uuid } from 'lib/crypto';
@ -13,7 +13,7 @@ export default async (req, res) => {
await useCors(req, res);
if (isbot(req.headers['user-agent'])) {
return ok(res);
return unauthorized(res);
}
const ignoreIps = process.env.IGNORE_IP;
@ -93,3 +93,38 @@ export default async (req, res) => {
return send(res, token);
};
// async function relational(req, res) {
// await useSession(req, res);
// const {
// session: { website_id, session_id, session_uuid },
// } = req;
// const { type, payload } = getJsonBody(req);
// let { url, referrer, event_name, event_data } = payload;
// if (process.env.REMOVE_TRAILING_SLASH) {
// url = removeTrailingSlash(url);
// }
// const event_uuid = uuid();
// if (type === 'pageview') {
// await savePageView(website_id, { session_id, session_uuid, url, referrer });
// } else if (type === 'event') {
// await saveEvent(website_id, {
// event_uuid,
// session_id,
// session_uuid,
// url,
// event_name,
// event_data,
// });
// } else {
// return badRequest(res);
// }
// }
// function clickhouse() {}

2
queries/admin/account/createAccount.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function createAccount(data) {
return runQuery(

2
queries/admin/account/deleteAccount.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function deleteAccount(user_id) {
return runQuery(

2
queries/admin/account/getAccountById.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getAccountById(user_id) {
return runQuery(

2
queries/admin/account/getAccountByUsername.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getAccountByUsername(username) {
return runQuery(

2
queries/admin/account/getAccounts.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getAccounts() {
return runQuery(

2
queries/admin/account/updateAccount.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function updateAccount(user_id, data) {
return runQuery(

2
queries/admin/website/createWebsite.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function createWebsite(user_id, data) {
return runQuery(

2
queries/admin/website/deleteWebsite.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function deleteWebsite(website_id) {
return runQuery(

2
queries/admin/website/getAllWebsites.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getAllWebsites() {
let data = await runQuery(

2
queries/admin/website/getUserWebsites.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getUserWebsites(user_id) {
return runQuery(

2
queries/admin/website/getWebsiteById.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getWebsiteById(website_id) {
return runQuery(

2
queries/admin/website/getWebsiteByShareId.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getWebsiteByShareId(share_id) {
return runQuery(

2
queries/admin/website/getWebsiteByUuid.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getWebsiteByUuid(website_uuid) {
return runQuery(

2
queries/admin/website/resetWebsite.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function resetWebsite(website_id) {
return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`);

2
queries/admin/website/updateWebsite.js
View File

@ -1,4 +1,4 @@
import { prisma, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function updateWebsite(website_id, data) {
return runQuery(

20
queries/analytics/event/getEventMetrics.js
View File

@ -1,13 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
getBetweenDatesClickhouse,
getDateQuery,
getDateQueryClickhouse,
getFilterQuery,
rawQuery,
rawQueryClickhouse,
runAnalyticsQuery,
} from 'lib/db';
import clickhouse from 'lib/clickhouse';
import { getDateQuery, getFilterQuery, rawQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
export async function getEventMetrics(...args) {
return runAnalyticsQuery({
@ -53,16 +47,16 @@ async function clickhouseQuery(
) {
const params = [website_id];
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
event_name x,
${getDateQueryClickhouse('created_at', unit, timezone)} t,
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
count(*) y
from event
where website_id= $1
and ${getBetweenDatesClickhouse('created_at', start_at, end_at)}
${getFilterQuery('event', filters, params)}
and ${clickhouse.getBetweenDates('created_at', start_at, end_at)}
${clickhouse.getFilterQuery('event', filters, params)}
group by x, t
order by t
`,

14
queries/analytics/event/getEvents.js
View File

@ -1,11 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
rawQueryClickhouse,
getDateFormatClickhouse,
prisma,
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db/db';
export function getEvents(...args) {
return runAnalyticsQuery({
@ -32,7 +28,7 @@ function relationalQuery(websites, start_at) {
}
function clickhouseQuery(websites, start_at) {
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
event_id,
@ -43,7 +39,7 @@ function clickhouseQuery(websites, start_at) {
event_name
from event
where website_id in (${websites.join[',']}
and created_at >= ${getDateFormatClickhouse(start_at)})
and created_at >= ${clickhouse.getDateFormat(start_at)})
`,
);
}

22
queries/analytics/event/saveEvent.js
View File

@ -1,12 +1,8 @@
import { CLICKHOUSE, RELATIONAL, KAFKA, URL_LENGTH } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
rawQueryClickhouse,
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { sendKafkaMessage, getDateFormatKafka } from 'lib/db/kafka';
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
import clickhouse from 'lib/clickhouse';
import kafka from 'lib/db/kafka';
import { prisma, runQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
export async function saveEvent(...args) {
return runAnalyticsQuery({
@ -48,10 +44,10 @@ async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, even
event_name?.substr(0, 50),
];
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
insert into umami.event (created_at, website_id, session_uuid, url, event_name)
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
@ -61,10 +57,10 @@ async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_nam
event_uuid: event_uuid,
website_id: website_id,
session_uuid: session_uuid,
created_at: getDateFormatKafka(new Date()),
created_at: kafka.getDateFormat(new Date()),
url: url?.substr(0, URL_LENGTH),
event_name: event_name?.substr(0, 50),
};
await sendKafkaMessage(params, 'event');
await kafka.sendKafkaMessage(params, 'event');
}

16
queries/analytics/pageview/getPageviewMetrics.js
View File

@ -1,11 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
rawQueryClickhouse,
runAnalyticsQuery,
parseFilters,
rawQuery,
getBetweenDatesClickhouse,
} from 'lib/db';
import clickhouse from 'lib/clickhouse';
import { parseFilters, rawQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
export async function getPageviewMetrics(...args) {
return runAnalyticsQuery({
@ -42,7 +38,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) {
const params = [website_id];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters(
table,
column,
filters,
@ -50,13 +46,13 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt
'session_uuid',
);
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id= $1
and ${getBetweenDatesClickhouse(table + '.created_at', start_at, end_at)}
and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)}
${pageviewQuery}
${joinSession && sessionQuery}
${eventQuery}

2
queries/analytics/pageview/getPageviewParams.js
View File

@ -1,4 +1,4 @@
import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/db';
import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/db/relational';
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
export async function getPageviewParams(...args) {

23
queries/analytics/pageview/getPageviewStats.js
View File

@ -1,14 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
getBetweenDatesClickhouse,
getDateQuery,
getDateQueryClickhouse,
getDateStringQueryClickhouse,
parseFilters,
rawQuery,
rawQueryClickhouse,
runAnalyticsQuery,
} from 'lib/db';
import { getDateQuery, parseFilters, rawQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
import clickhouse from 'lib/clickhouse';
export async function getPageviewStats(...args) {
return runAnalyticsQuery({
@ -62,7 +55,7 @@ async function clickhouseQuery(
sessionKey = 'session_uuid',
) {
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
'pageview',
null,
filters,
@ -70,19 +63,19 @@ async function clickhouseQuery(
sessionKey,
);
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
${getDateStringQueryClickhouse('g.t', unit)} as t,
${clickhouse.getDateStringQuery('g.t', unit)} as t,
g.y as y
from
(select
${getDateQueryClickhouse('created_at', unit, timezone)} t,
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
where pageview.website_id= $1
and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by t) g

14
queries/analytics/pageview/getPageviews.js
View File

@ -1,11 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
rawQueryClickhouse,
getDateFormatClickhouse,
prisma,
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
import clickhouse from 'lib/clickhouse';
export async function getPageviews(...args) {
return runAnalyticsQuery({
@ -32,7 +28,7 @@ async function relationalQuery(websites, start_at) {
}
async function clickhouseQuery(websites, start_at) {
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
view_id,
@ -42,7 +38,7 @@ async function clickhouseQuery(websites, start_at) {
url
from pageview
where website_id in (${websites.join[',']}
and created_at >= ${getDateFormatClickhouse(start_at)})
and created_at >= ${clickhouse.getDateFormat(start_at)})
`,
);
}

22
queries/analytics/pageview/savePageView.js
View File

@ -1,12 +1,8 @@
import { CLICKHOUSE, RELATIONAL, KAFKA, URL_LENGTH } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
rawQueryClickhouse,
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { sendKafkaMessage, getDateFormatKafka } from 'lib/db/kafka';
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db/db';
import kafka from 'lib/db/kafka';
import { prisma, runQuery } from 'lib/db/relational';
export async function savePageView(...args) {
return runAnalyticsQuery({
@ -37,10 +33,10 @@ async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {
referrer?.substr(0, URL_LENGTH),
];
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
@ -49,10 +45,10 @@ async function kafkaQuery(website_id, { session_uuid, url, referrer }) {
const params = {
website_id: website_id,
session_uuid: session_uuid,
created_at: getDateFormatKafka(new Date()),
created_at: kafka.getDateFormat(new Date()),
url: url?.substr(0, URL_LENGTH),
referrer: referrer?.substr(0, URL_LENGTH),
};
await sendKafkaMessage(params, 'pageview');
await kafka.sendKafkaMessage(params, 'pageview');
}

27
queries/analytics/session/createSession.js
View File

@ -1,13 +1,8 @@
import { CLICKHOUSE, RELATIONAL, KAFKA } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
rawQueryClickhouse,
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { sendKafkaMessage, getDateFormatKafka } from 'lib/db/kafka';
import { getSessionByUuid } from 'queries';
import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';
import { prisma, runQuery } from 'lib/db/relational';
import clickhouse from 'lib/clickhouse';
import kafka from 'lib/db/kafka';
import { runAnalyticsQuery } from 'lib/db/db';
export async function createSession(...args) {
return runAnalyticsQuery({
@ -47,13 +42,11 @@ async function clickhouseQuery(
country ? country : null,
];
await rawQueryClickhouse(
await clickhouse.rawQuery(
`insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
params,
);
return getSessionByUuid(session_uuid);
}
async function kafkaQuery(
@ -63,7 +56,7 @@ async function kafkaQuery(
const params = {
session_uuid: session_uuid,
website_id: website_id,
created_at: getDateFormatKafka(new Date()),
created_at: kafka.getDateFormat(new Date()),
hostname: hostname,
browser: browser,
os: os,
@ -73,7 +66,5 @@ async function kafkaQuery(
country: country ? country : null,
};
await sendKafkaMessage(params, 'session');
return getSessionByUuid(session_uuid);
await kafka.sendKafkaMessage(params, 'session');
}

8
queries/analytics/session/getSessionByUuid.js
View File

@ -1,5 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { rawQueryClickhouse, findUnique, prisma, runAnalyticsQuery, runQuery } from 'lib/db';
import { prisma, runQuery } from 'lib/db/relational';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db/db';
export async function getSessionByUuid(...args) {
return runAnalyticsQuery({
@ -21,7 +23,7 @@ async function relationalQuery(session_uuid) {
async function clickhouseQuery(session_uuid) {
const params = [session_uuid];
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
session_uuid,
@ -38,5 +40,5 @@ async function clickhouseQuery(session_uuid) {
where session_uuid = $1
`,
params,
).then(data => findUnique(data));
);
}

16
queries/analytics/session/getSessionMetrics.js
View File

@ -1,11 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
getBetweenDatesClickhouse,
parseFilters,
rawQuery,
rawQueryClickhouse,
runAnalyticsQuery,
} from 'lib/db';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db/db';
import { parseFilters, rawQuery } from 'lib/db/relational';
export async function getSessionMetrics(...args) {
return runAnalyticsQuery({
@ -45,7 +41,7 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}
async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) {
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
'pageview',
null,
filters,
@ -53,7 +49,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}
'session_uuid',
);
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select ${field} x, count(*) y
from session as x
@ -62,7 +58,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}
from pageview
${joinSession}
where pageview.website_id=$1
and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
)

14
queries/analytics/session/getSessions.js
View File

@ -1,11 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
rawQueryClickhouse,
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db/db';
import { prisma, runQuery } from 'lib/db/relational';
export async function getSessions(...args) {
return runAnalyticsQuery({
@ -32,7 +28,7 @@ async function relationalQuery(websites, start_at) {
}
async function clickhouseQuery(websites, start_at) {
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
session_id,
@ -48,7 +44,7 @@ async function clickhouseQuery(websites, start_at) {
country
from session
where website_id in (${websites.join[',']}
and created_at >= ${getDateFormatClickhouse(start_at)})
and created_at >= ${clickhouse.getDateFormat(start_at)})
`,
);
}

8
queries/analytics/stats/getActiveVisitors.js
View File

@ -1,6 +1,8 @@
import { subMinutes } from 'date-fns';
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { getDateFormatClickhouse, rawQuery, rawQueryClickhouse, runAnalyticsQuery } from 'lib/db';
import { rawQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
import clickhouse from 'lib/clickhouse';
export async function getActiveVisitors(...args) {
return runAnalyticsQuery({
@ -27,12 +29,12 @@ async function relationalQuery(website_id) {
async function clickhouseQuery(website_id) {
const params = [website_id];
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select count(distinct session_uuid) x
from pageview
where website_id = $1
and created_at >= ${getDateFormatClickhouse(subMinutes(new Date(), 5))}
and created_at >= ${clickhouse.getDateFormat(subMinutes(new Date(), 5))}
`,
params,
);

21
queries/analytics/stats/getWebsiteStats.js
View File

@ -1,14 +1,7 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import {
getDateQuery,
getBetweenDatesClickhouse,
getDateQueryClickhouse,
getTimestampInterval,
parseFilters,
rawQuery,
rawQueryClickhouse,
runAnalyticsQuery,
} from 'lib/db';
import { getDateQuery, getTimestampInterval, parseFilters, rawQuery } from 'lib/db/relational';
import { runAnalyticsQuery } from 'lib/db/db';
import clickhouse from 'lib/clickhouse';
export async function getWebsiteStats(...args) {
return runAnalyticsQuery({
@ -52,7 +45,7 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) {
async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
'pageview',
null,
filters,
@ -60,7 +53,7 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
'session_uuid',
);
return rawQueryClickhouse(
return clickhouse.rawQuery(
`
select
sum(t.c) as "pageviews",
@ -69,14 +62,14 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
from (
select pageview.session_uuid,
${getDateQueryClickhouse('pageview.created_at', 'day')} time_series,
${clickhouse.getDateQuery('pageview.created_at', 'day')} time_series,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from pageview
${joinSession}
where pageview.website_id = $1
and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by pageview.session_uuid, time_series