mirror of
https://github.com/kremalicious/umami.git
synced 2025-02-14 21:10:34 +01:00
Merge branch 'brian/uc-33-kafka-optimizations' into dev
This commit is contained in:
commit
8933150a9c
190
lib/clickhouse.js
Normal file
190
lib/clickhouse.js
Normal file
@ -0,0 +1,190 @@
|
||||
import { ClickHouse } from 'clickhouse';
|
||||
import dateFormat from 'dateformat';
|
||||
import { FILTER_IGNORED } from 'lib/constants';
|
||||
import { CLICKHOUSE_DATE_FORMATS } from './constants';
|
||||
|
||||
function getClient() {
|
||||
if (!process.env.ANALYTICS_URL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = new URL(process.env.ANALYTICS_URL);
|
||||
const database = url.pathname.replace('/', '');
|
||||
|
||||
return new ClickHouse({
|
||||
url: url.hostname,
|
||||
port: Number(url.port),
|
||||
basicAuth: url.password
|
||||
? {
|
||||
username: url.username || 'default',
|
||||
password: url.password,
|
||||
}
|
||||
: null,
|
||||
format: 'json',
|
||||
config: {
|
||||
database,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const clickhouse = global.clickhouse || getClient();
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global.clickhouse = clickhouse;
|
||||
}
|
||||
|
||||
export { clickhouse };
|
||||
|
||||
function getDateStringQuery(data, unit) {
|
||||
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
function getDateQuery(field, unit, timezone) {
|
||||
if (timezone) {
|
||||
return `date_trunc('${unit}', ${field}, '${timezone}')`;
|
||||
}
|
||||
return `date_trunc('${unit}', ${field})`;
|
||||
}
|
||||
|
||||
function getDateFormat(date) {
|
||||
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
|
||||
}
|
||||
|
||||
function getBetweenDates(field, start_at, end_at) {
|
||||
return `${field} between ${getDateFormat(start_at)}
|
||||
and ${getDateFormat(end_at)}`;
|
||||
}
|
||||
|
||||
function getFilterQuery(table, column, filters = {}, params = []) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined || filter === FILTER_IGNORED) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
switch (key) {
|
||||
case 'url':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'os':
|
||||
case 'browser':
|
||||
case 'device':
|
||||
case 'country':
|
||||
if (table === 'session') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'event_name':
|
||||
if (table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'referrer':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.referrer like $${params.length + 1}`);
|
||||
params.push(`%${decodeURIComponent(filter)}%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'domain':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
|
||||
arr.push(`and ${table}.referrer not like '/%'`);
|
||||
params.push(`%://${filter}/%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'query':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.url like '%?%'`);
|
||||
}
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
|
||||
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
|
||||
filters;
|
||||
|
||||
const pageviewFilters = { domain, url, referrer, query };
|
||||
const sessionFilters = { os, browser, device, country };
|
||||
const eventFilters = { url: event_url, event_name };
|
||||
|
||||
return {
|
||||
pageviewFilters,
|
||||
sessionFilters,
|
||||
eventFilters,
|
||||
event: { event_name },
|
||||
joinSession:
|
||||
os || browser || device || country
|
||||
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
|
||||
: '',
|
||||
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
|
||||
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
|
||||
eventQuery: getFilterQuery('event', column, eventFilters, params),
|
||||
};
|
||||
}
|
||||
|
||||
function replaceQuery(string, params = []) {
|
||||
let formattedString = string;
|
||||
|
||||
params.forEach((a, i) => {
|
||||
let replace = a;
|
||||
|
||||
if (typeof a === 'string' || a instanceof String) {
|
||||
replace = `'${replace}'`;
|
||||
}
|
||||
|
||||
formattedString = formattedString.replace(`$${i + 1}`, replace);
|
||||
});
|
||||
|
||||
return formattedString;
|
||||
}
|
||||
|
||||
async function rawQuery(query, params = [], debug = false) {
|
||||
let formattedQuery = replaceQuery(query, params);
|
||||
|
||||
if (debug || process.env.LOG_QUERY) {
|
||||
console.log(formattedQuery);
|
||||
}
|
||||
|
||||
return clickhouse.query(formattedQuery).toPromise();
|
||||
}
|
||||
|
||||
async function findUnique(data) {
|
||||
if (data.length > 1) {
|
||||
throw `${data.length} records found when expecting 1.`;
|
||||
}
|
||||
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
async function findFirst(data) {
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
export default {
|
||||
getDateStringQuery,
|
||||
getDateQuery,
|
||||
getDateFormat,
|
||||
getBetweenDates,
|
||||
getFilterQuery,
|
||||
parseFilters,
|
||||
replaceQuery,
|
||||
rawQuery,
|
||||
findUnique,
|
||||
findFirst,
|
||||
};
|
286
lib/db.js
286
lib/db.js
@ -1,305 +1,33 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { ClickHouse } from 'clickhouse';
|
||||
import dateFormat from 'dateformat';
|
||||
import chalk from 'chalk';
|
||||
import { getKafkaService } from './kafka';
|
||||
import {
|
||||
MYSQL,
|
||||
MYSQL_DATE_FORMATS,
|
||||
POSTGRESQL,
|
||||
POSTGRESQL_DATE_FORMATS,
|
||||
CLICKHOUSE,
|
||||
RELATIONAL,
|
||||
FILTER_IGNORED,
|
||||
KAFKA,
|
||||
} from 'lib/constants';
|
||||
import moment from 'moment-timezone';
|
||||
import { CLICKHOUSE_DATE_FORMATS } from './constants';
|
||||
import { POSTGRESQL, RELATIONAL, MYSQL, KAFKA } from 'lib/constants';
|
||||
import { CLICKHOUSE } from 'lib/constants';
|
||||
|
||||
BigInt.prototype.toJSON = function () {
|
||||
return Number(this);
|
||||
};
|
||||
|
||||
const options = {
|
||||
log: [
|
||||
{
|
||||
emit: 'event',
|
||||
level: 'query',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function logQuery(e) {
|
||||
console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
|
||||
}
|
||||
|
||||
function getPrismaClient(options) {
|
||||
const prisma = new PrismaClient(options);
|
||||
|
||||
if (process.env.LOG_QUERY) {
|
||||
prisma.$on('query', logQuery);
|
||||
}
|
||||
|
||||
return prisma;
|
||||
}
|
||||
|
||||
function getClickhouseClient() {
|
||||
if (!process.env.ANALYTICS_URL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = new URL(process.env.ANALYTICS_URL);
|
||||
const database = url.pathname.replace('/', '');
|
||||
|
||||
return new ClickHouse({
|
||||
url: url.hostname,
|
||||
port: Number(url.port),
|
||||
basicAuth: url.password
|
||||
? {
|
||||
username: url.username || 'default',
|
||||
password: url.password,
|
||||
}
|
||||
: null,
|
||||
format: 'json',
|
||||
config: {
|
||||
database,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const prisma = global.prisma || getPrismaClient(options);
|
||||
const clickhouse = global.clickhouse || getClickhouseClient();
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global.prisma = prisma;
|
||||
global.clickhouse = clickhouse;
|
||||
}
|
||||
|
||||
export { prisma, clickhouse };
|
||||
|
||||
export function getDatabase() {
|
||||
const type =
|
||||
process.env.DATABASE_TYPE ||
|
||||
(process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]);
|
||||
|
||||
if (type === 'postgres') {
|
||||
return POSTGRESQL;
|
||||
}
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
export function getAnalyticsDatabase() {
|
||||
const type = process.env.ANALYTICS_URL && process.env.ANALYTICS_URL.split(':')[0];
|
||||
export function getDatabase(database, databaseType, fallback) {
|
||||
const type = databaseType || (database && database.split(':')[0]);
|
||||
|
||||
if (type === 'postgres') {
|
||||
return POSTGRESQL;
|
||||
}
|
||||
|
||||
if (!type) {
|
||||
return getDatabase();
|
||||
return getDatabase(fallback);
|
||||
}
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
export function getDateStringQueryClickhouse(data, unit) {
|
||||
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
export function getDateQuery(field, unit, timezone) {
|
||||
const db = getDatabase();
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
if (timezone) {
|
||||
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
if (timezone) {
|
||||
const tz = moment.tz(timezone).format('Z');
|
||||
|
||||
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
}
|
||||
|
||||
export function getDateQueryClickhouse(field, unit, timezone) {
|
||||
if (timezone) {
|
||||
return `date_trunc('${unit}', ${field}, '${timezone}')`;
|
||||
}
|
||||
return `date_trunc('${unit}', ${field})`;
|
||||
}
|
||||
|
||||
export function getDateFormatClickhouse(date) {
|
||||
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
|
||||
}
|
||||
|
||||
export function getBetweenDatesClickhouse(field, start_at, end_at) {
|
||||
return `${field} between ${getDateFormatClickhouse(start_at)}
|
||||
and ${getDateFormatClickhouse(end_at)}`;
|
||||
}
|
||||
|
||||
export function getTimestampInterval(field) {
|
||||
const db = getDatabase();
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
return `floor(extract(epoch from max(${field}) - min(${field})))`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`;
|
||||
}
|
||||
}
|
||||
|
||||
export function getFilterQuery(table, column, filters = {}, params = []) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined || filter === FILTER_IGNORED) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
switch (key) {
|
||||
case 'url':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'os':
|
||||
case 'browser':
|
||||
case 'device':
|
||||
case 'country':
|
||||
if (table === 'session') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'event_name':
|
||||
if (table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'referrer':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.referrer like $${params.length + 1}`);
|
||||
params.push(`%${decodeURIComponent(filter)}%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'domain':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
|
||||
arr.push(`and ${table}.referrer not like '/%'`);
|
||||
params.push(`%://${filter}/%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'query':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.url like '%?%'`);
|
||||
}
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
|
||||
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
|
||||
filters;
|
||||
|
||||
const pageviewFilters = { domain, url, referrer, query };
|
||||
const sessionFilters = { os, browser, device, country };
|
||||
const eventFilters = { url: event_url, event_name };
|
||||
|
||||
return {
|
||||
pageviewFilters,
|
||||
sessionFilters,
|
||||
eventFilters,
|
||||
event: { event_name },
|
||||
joinSession:
|
||||
os || browser || device || country
|
||||
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
|
||||
: '',
|
||||
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
|
||||
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
|
||||
eventQuery: getFilterQuery('event', column, eventFilters, params),
|
||||
};
|
||||
}
|
||||
|
||||
export function replaceQueryClickhouse(string, params = []) {
|
||||
let formattedString = string;
|
||||
|
||||
params.forEach((a, i) => {
|
||||
let replace = a;
|
||||
|
||||
if (typeof a === 'string' || a instanceof String) {
|
||||
replace = `'${replace}'`;
|
||||
}
|
||||
|
||||
formattedString = formattedString.replace(`$${i + 1}`, replace);
|
||||
});
|
||||
|
||||
return formattedString;
|
||||
}
|
||||
|
||||
export async function runQuery(query) {
|
||||
return query.catch(e => {
|
||||
throw e;
|
||||
});
|
||||
}
|
||||
|
||||
export async function rawQuery(query, params = []) {
|
||||
const db = getDatabase();
|
||||
|
||||
if (db !== POSTGRESQL && db !== MYSQL) {
|
||||
return Promise.reject(new Error('Unknown database.'));
|
||||
}
|
||||
|
||||
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
|
||||
|
||||
return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
|
||||
}
|
||||
|
||||
export async function rawQueryClickhouse(query, params = [], debug = false) {
|
||||
let formattedQuery = replaceQueryClickhouse(query, params);
|
||||
|
||||
if (debug || process.env.LOG_QUERY) {
|
||||
console.log(formattedQuery);
|
||||
}
|
||||
|
||||
return clickhouse.query(formattedQuery).toPromise();
|
||||
}
|
||||
|
||||
export async function findUnique(data) {
|
||||
if (data.length > 1) {
|
||||
throw `${data.length} records found when expecting 1.`;
|
||||
}
|
||||
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
export async function runAnalyticsQuery(queries) {
|
||||
const db = getAnalyticsDatabase();
|
||||
const db = getDatabase(process.env.ANALYTICS_URL, null, process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL || db === MYSQL) {
|
||||
return queries[RELATIONAL]();
|
||||
}
|
||||
|
||||
if (db === CLICKHOUSE) {
|
||||
const kafka = getKafkaService();
|
||||
const kafka = getDatabase(process.env.KAFKA_URL);
|
||||
if (kafka === KAFKA && queries[KAFKA]) {
|
||||
return queries[KAFKA]();
|
||||
}
|
||||
|
46
lib/kafka.js
46
lib/kafka.js
@ -1,7 +1,7 @@
|
||||
import { Kafka } from 'kafkajs';
|
||||
import { Kafka, logLevel } from 'kafkajs';
|
||||
import dateFormat from 'dateformat';
|
||||
|
||||
export function getKafkaClient() {
|
||||
export function getClient() {
|
||||
if (!process.env.KAFKA_URL) {
|
||||
return null;
|
||||
}
|
||||
@ -14,6 +14,7 @@ export function getKafkaClient() {
|
||||
clientId: 'umami',
|
||||
brokers: brokers,
|
||||
connectionTimeout: 3000,
|
||||
logLevel: logLevel.ERROR,
|
||||
});
|
||||
} else {
|
||||
return new Kafka({
|
||||
@ -29,20 +30,33 @@ export function getKafkaClient() {
|
||||
});
|
||||
}
|
||||
}
|
||||
const kafka = global.kafka || getClient();
|
||||
let kafkaProducer = null;
|
||||
|
||||
const kafka = global.kafka || getKafkaClient();
|
||||
(async () => {
|
||||
kafkaProducer = global.kakfaProducer || (await getProducer());
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global.kafka = kafka;
|
||||
}
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global.kafka = kafka;
|
||||
global.kakfaProducer = kafkaProducer;
|
||||
}
|
||||
})();
|
||||
|
||||
export { kafka };
|
||||
export { kafka, kafkaProducer };
|
||||
|
||||
export async function kafkaProducer(params, topic) {
|
||||
async function getProducer() {
|
||||
const producer = kafka.producer();
|
||||
await producer.connect();
|
||||
|
||||
await producer.send({
|
||||
return producer;
|
||||
}
|
||||
|
||||
function getDateFormat(date) {
|
||||
return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
|
||||
}
|
||||
|
||||
async function sendMessage(params, topic) {
|
||||
await kafkaProducer.send({
|
||||
topic,
|
||||
messages: [
|
||||
{
|
||||
@ -50,15 +64,11 @@ export async function kafkaProducer(params, topic) {
|
||||
value: JSON.stringify(params),
|
||||
},
|
||||
],
|
||||
acks: 0,
|
||||
});
|
||||
}
|
||||
|
||||
export function getDateFormatKafka(date) {
|
||||
return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
|
||||
}
|
||||
|
||||
export function getKafkaService() {
|
||||
const type = process.env.KAFKA_URL && process.env.KAFKA_URL.split(':')[0];
|
||||
|
||||
return type;
|
||||
}
|
||||
export default {
|
||||
getDateFormat,
|
||||
sendMessage,
|
||||
};
|
||||
|
59
lib/redis.js
Normal file
59
lib/redis.js
Normal file
@ -0,0 +1,59 @@
|
||||
import { createClient } from 'redis';
|
||||
import { startOfMonth } from 'date-fns';
|
||||
import { getSessions, getAllWebsites } from '/queries';
|
||||
|
||||
async function getClient() {
|
||||
const redis = new createClient({
|
||||
url: process.env.REDIS_URL,
|
||||
});
|
||||
|
||||
await redis.connect();
|
||||
|
||||
if (process.env.LOG_QUERY) {
|
||||
redis.on('error', err => console.log('Redis Client Error', err));
|
||||
}
|
||||
|
||||
return redis;
|
||||
}
|
||||
|
||||
let redis = null;
|
||||
|
||||
(async () => {
|
||||
redis = global.redis || (await getClient());
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global.redis = redis;
|
||||
}
|
||||
|
||||
const value = await redis.get('initialized');
|
||||
|
||||
if (!value) {
|
||||
await stageData();
|
||||
}
|
||||
})();
|
||||
|
||||
export default redis;
|
||||
|
||||
async function stageData() {
|
||||
const sessions = await getSessions([], startOfMonth(new Date()).toUTCString());
|
||||
const websites = await getAllWebsites();
|
||||
|
||||
const sessionUuids = sessions.map(a => {
|
||||
return { key: `session:${a.session_uuid}`, value: '' };
|
||||
});
|
||||
const websiteIds = websites.map(a => {
|
||||
return { key: `website:${a.website_uuid}`, value: Number(a.website_id) };
|
||||
});
|
||||
|
||||
await addRedis(sessionUuids);
|
||||
await addRedis(websiteIds);
|
||||
|
||||
await redis.set('initialized', 'initialized');
|
||||
}
|
||||
|
||||
async function addRedis(ids) {
|
||||
for (let i = 0; i < ids.length; i++) {
|
||||
const { key, value } = ids[i];
|
||||
await redis.set(key, value);
|
||||
}
|
||||
}
|
163
lib/relational.js
Normal file
163
lib/relational.js
Normal file
@ -0,0 +1,163 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import chalk from 'chalk';
|
||||
import {
|
||||
FILTER_IGNORED,
|
||||
MYSQL,
|
||||
MYSQL_DATE_FORMATS,
|
||||
POSTGRESQL,
|
||||
POSTGRESQL_DATE_FORMATS,
|
||||
} from 'lib/constants';
|
||||
import { getDatabase } from './db';
|
||||
import moment from 'moment-timezone';
|
||||
|
||||
const options = {
|
||||
log: [
|
||||
{
|
||||
emit: 'event',
|
||||
level: 'query',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function logQuery(e) {
|
||||
console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
|
||||
}
|
||||
|
||||
function getClient(options) {
|
||||
const prisma = new PrismaClient(options);
|
||||
|
||||
if (process.env.LOG_QUERY) {
|
||||
prisma.$on('query', logQuery);
|
||||
}
|
||||
|
||||
return prisma;
|
||||
}
|
||||
const prisma = global.prisma || getClient(options);
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global.prisma = prisma;
|
||||
}
|
||||
|
||||
export { prisma };
|
||||
|
||||
export function getDateQuery(field, unit, timezone) {
|
||||
const db = getDatabase();
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
if (timezone) {
|
||||
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
if (timezone) {
|
||||
const tz = moment.tz(timezone).format('Z');
|
||||
|
||||
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
}
|
||||
|
||||
export function getFilterQuery(table, column, filters = {}, params = []) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined || filter === FILTER_IGNORED) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
switch (key) {
|
||||
case 'url':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'os':
|
||||
case 'browser':
|
||||
case 'device':
|
||||
case 'country':
|
||||
if (table === 'session') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'event_name':
|
||||
if (table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'referrer':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.referrer like $${params.length + 1}`);
|
||||
params.push(`%${decodeURIComponent(filter)}%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'domain':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
|
||||
arr.push(`and ${table}.referrer not like '/%'`);
|
||||
params.push(`%://${filter}/%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'query':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.url like '%?%'`);
|
||||
}
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
|
||||
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
|
||||
filters;
|
||||
|
||||
const pageviewFilters = { domain, url, referrer, query };
|
||||
const sessionFilters = { os, browser, device, country };
|
||||
const eventFilters = { url: event_url, event_name };
|
||||
|
||||
return {
|
||||
pageviewFilters,
|
||||
sessionFilters,
|
||||
eventFilters,
|
||||
event: { event_name },
|
||||
joinSession:
|
||||
os || browser || device || country
|
||||
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
|
||||
: '',
|
||||
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
|
||||
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
|
||||
eventQuery: getFilterQuery('event', column, eventFilters, params),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runQuery(query) {
|
||||
return query.catch(e => {
|
||||
throw e;
|
||||
});
|
||||
}
|
||||
|
||||
export async function rawQuery(query, params = []) {
|
||||
const db = getDatabase(process.env.DATABASE_URL, process.env.DATABASE_TYPE);
|
||||
|
||||
if (db !== POSTGRESQL && db !== MYSQL) {
|
||||
return Promise.reject(new Error('Unknown database.'));
|
||||
}
|
||||
|
||||
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
|
||||
|
||||
return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
|
||||
}
|
@ -1,6 +1,8 @@
|
||||
import { getWebsiteByUuid, getSessionByUuid, createSession } from 'queries';
|
||||
import { getJsonBody, getClientInfo } from 'lib/request';
|
||||
import { uuid, isValidUuid, parseToken } from 'lib/crypto';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import { RELATIONAL, CLICKHOUSE } from 'lib/constants';
|
||||
|
||||
export async function getSession(req) {
|
||||
const { payload } = getJsonBody(req);
|
||||
@ -9,7 +11,6 @@ export async function getSession(req) {
|
||||
throw new Error('Invalid request');
|
||||
}
|
||||
|
||||
const { website: website_uuid, hostname, screen, language } = payload;
|
||||
const cache = req.headers['x-umami-cache'];
|
||||
|
||||
if (cache) {
|
||||
@ -20,21 +21,29 @@ export async function getSession(req) {
|
||||
}
|
||||
}
|
||||
|
||||
const { website: website_uuid, hostname, screen, language } = payload;
|
||||
|
||||
if (!isValidUuid(website_uuid)) {
|
||||
throw new Error(`Invalid website: ${website_uuid}`);
|
||||
}
|
||||
|
||||
const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);
|
||||
|
||||
const website = await getWebsiteByUuid(website_uuid);
|
||||
|
||||
if (!website) {
|
||||
throw new Error(`Website not found: ${website_uuid}`);
|
||||
}
|
||||
|
||||
const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);
|
||||
|
||||
const { website_id } = website;
|
||||
const session_uuid = uuid(website_id, hostname, ip, userAgent);
|
||||
|
||||
// logic placeholder for redis
|
||||
runAnalyticsQuery({
|
||||
[RELATIONAL]: () => {},
|
||||
[CLICKHOUSE]: () => {},
|
||||
});
|
||||
|
||||
let session = await getSessionByUuid(session_uuid);
|
||||
|
||||
if (!session) {
|
||||
|
@ -97,6 +97,7 @@
|
||||
"react-tooltip": "^4.2.21",
|
||||
"react-use-measure": "^2.0.4",
|
||||
"react-window": "^1.8.6",
|
||||
"redis": "^4.3.0",
|
||||
"request-ip": "^3.3.0",
|
||||
"semver": "^7.3.6",
|
||||
"thenby": "^1.3.4",
|
||||
|
@ -4,7 +4,7 @@ import ipaddr from 'ipaddr.js';
|
||||
import { savePageView, saveEvent } from 'queries';
|
||||
import { useCors, useSession } from 'lib/middleware';
|
||||
import { getJsonBody, getIpAddress } from 'lib/request';
|
||||
import { ok, send, badRequest, forbidden } from 'lib/response';
|
||||
import { unauthorized, send, badRequest, forbidden } from 'lib/response';
|
||||
import { createToken } from 'lib/crypto';
|
||||
import { removeTrailingSlash } from 'lib/url';
|
||||
import { uuid } from 'lib/crypto';
|
||||
@ -12,8 +12,8 @@ import { uuid } from 'lib/crypto';
|
||||
export default async (req, res) => {
|
||||
await useCors(req, res);
|
||||
|
||||
if (isbot(req.headers['user-agent'])) {
|
||||
return ok(res);
|
||||
if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) {
|
||||
return unauthorized(res);
|
||||
}
|
||||
|
||||
const ignoreIps = process.env.IGNORE_IP;
|
||||
|
16
pages/api/heartbeat.js
Normal file
16
pages/api/heartbeat.js
Normal file
@ -0,0 +1,16 @@
|
||||
import { redis } from 'lib/redis';
|
||||
import { ok } from 'lib/response';
|
||||
|
||||
export default async (req, res) => {
|
||||
console.log('------------------------------------------------------------------------------');
|
||||
// const redis = new createClient({
|
||||
// url: process.env.REDIS_URL,
|
||||
// });
|
||||
|
||||
//console.log(redis.get);
|
||||
|
||||
const value = await redis.get('session:77c0b9de-677a-5268-8543-6018f0776a81');
|
||||
console.log('complete');
|
||||
|
||||
return ok(res, value);
|
||||
};
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function createAccount(data) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function deleteAccount(user_id) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getAccountById(user_id) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getAccountByUsername(username) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getAccounts() {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function updateAccount(user_id, data) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,5 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
import redis from 'lib/redis';
|
||||
|
||||
export async function createWebsite(user_id, data) {
|
||||
return runQuery(
|
||||
@ -12,5 +13,11 @@ export async function createWebsite(user_id, data) {
|
||||
...data,
|
||||
},
|
||||
}),
|
||||
);
|
||||
).then(async res => {
|
||||
if (process.env.REDIS_URL) {
|
||||
await redis.set(`website:${res.website_uuid}`, Number(res.website_id));
|
||||
}
|
||||
|
||||
return res;
|
||||
});
|
||||
}
|
||||
|
@ -1,23 +1,30 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
import redis from 'lib/redis';
|
||||
|
||||
export async function deleteWebsite(website_id) {
|
||||
return runQuery(
|
||||
prisma.$transaction([
|
||||
prisma.pageview.deleteMany({
|
||||
where: { session: { website: { website_id } } },
|
||||
prisma
|
||||
.$transaction([
|
||||
prisma.pageview.deleteMany({
|
||||
where: { session: { website: { website_id } } },
|
||||
}),
|
||||
prisma.event_data.deleteMany({
|
||||
where: { event: { session: { website: { website_id } } } },
|
||||
}),
|
||||
prisma.event.deleteMany({
|
||||
where: { session: { website: { website_id } } },
|
||||
}),
|
||||
prisma.session.deleteMany({
|
||||
where: { website: { website_id } },
|
||||
}),
|
||||
prisma.website.delete({
|
||||
where: { website_id },
|
||||
}),
|
||||
])
|
||||
.then(async res => {
|
||||
if (process.env.REDIS_URL) {
|
||||
await redis.del(`website:${res.website_uuid}`);
|
||||
}
|
||||
}),
|
||||
prisma.event_data.deleteMany({
|
||||
where: { event: { session: { website: { website_id } } } },
|
||||
}),
|
||||
prisma.event.deleteMany({
|
||||
where: { session: { website: { website_id } } },
|
||||
}),
|
||||
prisma.session.deleteMany({
|
||||
where: { website: { website_id } },
|
||||
}),
|
||||
prisma.website.delete({
|
||||
where: { website_id },
|
||||
}),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getAllWebsites() {
|
||||
let data = await runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getUserWebsites(user_id) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getWebsiteById(website_id) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getWebsiteByShareId(share_id) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getWebsiteByUuid(website_uuid) {
|
||||
return runQuery(
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function resetWebsite(website_id) {
|
||||
return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`);
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { prisma, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function updateWebsite(website_id, data) {
|
||||
return runQuery(
|
||||
|
@ -1,13 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
getBetweenDatesClickhouse,
|
||||
getDateQuery,
|
||||
getDateQueryClickhouse,
|
||||
getFilterQuery,
|
||||
rawQuery,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
} from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { getDateQuery, getFilterQuery, rawQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
|
||||
export async function getEventMetrics(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -53,16 +47,16 @@ async function clickhouseQuery(
|
||||
) {
|
||||
const params = [website_id];
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select
|
||||
event_name x,
|
||||
${getDateQueryClickhouse('created_at', unit, timezone)} t,
|
||||
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
|
||||
count(*) y
|
||||
from event
|
||||
where website_id= $1
|
||||
and ${getBetweenDatesClickhouse('created_at', start_at, end_at)}
|
||||
${getFilterQuery('event', filters, params)}
|
||||
and ${clickhouse.getBetweenDates('created_at', start_at, end_at)}
|
||||
${clickhouse.getFilterQuery('event', filters, params)}
|
||||
group by x, t
|
||||
order by t
|
||||
`,
|
||||
|
@ -1,11 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
rawQueryClickhouse,
|
||||
getDateFormatClickhouse,
|
||||
prisma,
|
||||
runAnalyticsQuery,
|
||||
runQuery,
|
||||
} from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
|
||||
export function getEvents(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -32,7 +28,7 @@ function relationalQuery(websites, start_at) {
|
||||
}
|
||||
|
||||
function clickhouseQuery(websites, start_at) {
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select
|
||||
event_id,
|
||||
@ -43,7 +39,7 @@ function clickhouseQuery(websites, start_at) {
|
||||
event_name
|
||||
from event
|
||||
where website_id in (${websites.join[',']}
|
||||
and created_at >= ${getDateFormatClickhouse(start_at)})
|
||||
and created_at >= ${clickhouse.getDateFormat(start_at)})
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
@ -1,12 +1,8 @@
|
||||
import { CLICKHOUSE, RELATIONAL, KAFKA, URL_LENGTH } from 'lib/constants';
|
||||
import {
|
||||
getDateFormatClickhouse,
|
||||
prisma,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
runQuery,
|
||||
} from 'lib/db';
|
||||
import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';
|
||||
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import kafka from 'lib/kafka';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
|
||||
export async function saveEvent(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -48,10 +44,10 @@ async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, even
|
||||
event_name?.substr(0, 50),
|
||||
];
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
insert into umami.event (created_at, website_id, session_uuid, url, event_name)
|
||||
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,
|
||||
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
|
||||
params,
|
||||
);
|
||||
}
|
||||
@ -61,10 +57,10 @@ async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_nam
|
||||
event_uuid: event_uuid,
|
||||
website_id: website_id,
|
||||
session_uuid: session_uuid,
|
||||
created_at: getDateFormatKafka(new Date()),
|
||||
created_at: kafka.getDateFormat(new Date()),
|
||||
url: url?.substr(0, URL_LENGTH),
|
||||
event_name: event_name?.substr(0, 50),
|
||||
};
|
||||
|
||||
await kafkaProducer(params, 'event');
|
||||
await kafka.sendKafkaMessage(params, 'event');
|
||||
}
|
||||
|
@ -1,11 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
parseFilters,
|
||||
rawQuery,
|
||||
getBetweenDatesClickhouse,
|
||||
} from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { parseFilters, rawQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
|
||||
export async function getPageviewMetrics(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -42,7 +38,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
|
||||
|
||||
async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) {
|
||||
const params = [website_id];
|
||||
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
|
||||
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters(
|
||||
table,
|
||||
column,
|
||||
filters,
|
||||
@ -50,13 +46,13 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt
|
||||
'session_uuid',
|
||||
);
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select ${column} x, count(*) y
|
||||
from ${table}
|
||||
${joinSession}
|
||||
where ${table}.website_id= $1
|
||||
and ${getBetweenDatesClickhouse(table + '.created_at', start_at, end_at)}
|
||||
and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)}
|
||||
${pageviewQuery}
|
||||
${joinSession && sessionQuery}
|
||||
${eventQuery}
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/db';
|
||||
import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/relational';
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
|
||||
export async function getPageviewParams(...args) {
|
||||
|
@ -1,14 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
getBetweenDatesClickhouse,
|
||||
getDateQuery,
|
||||
getDateQueryClickhouse,
|
||||
getDateStringQueryClickhouse,
|
||||
parseFilters,
|
||||
rawQuery,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
} from 'lib/db';
|
||||
import { getDateQuery, parseFilters, rawQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
|
||||
export async function getPageviewStats(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -70,19 +63,19 @@ async function clickhouseQuery(
|
||||
sessionKey,
|
||||
);
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select
|
||||
${getDateStringQueryClickhouse('g.t', unit)} as t,
|
||||
${clickhouse.getDateStringQuery('g.t', unit)} as t,
|
||||
g.y as y
|
||||
from
|
||||
(select
|
||||
${getDateQueryClickhouse('created_at', unit, timezone)} t,
|
||||
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
|
||||
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
|
||||
from pageview
|
||||
${joinSession}
|
||||
where pageview.website_id= $1
|
||||
and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}
|
||||
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
|
||||
${pageviewQuery}
|
||||
${sessionQuery}
|
||||
group by t) g
|
||||
|
@ -1,11 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
rawQueryClickhouse,
|
||||
getDateFormatClickhouse,
|
||||
prisma,
|
||||
runAnalyticsQuery,
|
||||
runQuery,
|
||||
} from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
|
||||
export async function getPageviews(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -32,7 +28,7 @@ async function relationalQuery(websites, start_at) {
|
||||
}
|
||||
|
||||
async function clickhouseQuery(websites, start_at) {
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select
|
||||
view_id,
|
||||
@ -42,7 +38,7 @@ async function clickhouseQuery(websites, start_at) {
|
||||
url
|
||||
from pageview
|
||||
where website_id in (${websites.join[',']}
|
||||
and created_at >= ${getDateFormatClickhouse(start_at)})
|
||||
and created_at >= ${clickhouse.getDateFormat(start_at)})
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
@ -1,12 +1,8 @@
|
||||
import { CLICKHOUSE, RELATIONAL, KAFKA, URL_LENGTH } from 'lib/constants';
|
||||
import {
|
||||
getDateFormatClickhouse,
|
||||
prisma,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
runQuery,
|
||||
} from 'lib/db';
|
||||
import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';
|
||||
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import kafka from 'lib/kafka';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function savePageView(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -37,10 +33,10 @@ async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {
|
||||
referrer?.substr(0, URL_LENGTH),
|
||||
];
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
|
||||
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,
|
||||
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
|
||||
params,
|
||||
);
|
||||
}
|
||||
@ -49,10 +45,10 @@ async function kafkaQuery(website_id, { session_uuid, url, referrer }) {
|
||||
const params = {
|
||||
website_id: website_id,
|
||||
session_uuid: session_uuid,
|
||||
created_at: getDateFormatKafka(new Date()),
|
||||
created_at: kafka.getDateFormat(new Date()),
|
||||
url: url?.substr(0, URL_LENGTH),
|
||||
referrer: referrer?.substr(0, URL_LENGTH),
|
||||
};
|
||||
|
||||
await kafkaProducer(params, 'pageview');
|
||||
await kafka.sendKafkaMessage(params, 'pageview');
|
||||
}
|
||||
|
@ -1,13 +1,9 @@
|
||||
import { CLICKHOUSE, RELATIONAL, KAFKA } from 'lib/constants';
|
||||
import {
|
||||
getDateFormatClickhouse,
|
||||
prisma,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
runQuery,
|
||||
} from 'lib/db';
|
||||
import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';
|
||||
import { getSessionByUuid } from 'queries';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import kafka from 'lib/kafka';
|
||||
import redis from 'lib/redis';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function createSession(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -28,7 +24,13 @@ async function relationalQuery(website_id, data) {
|
||||
session_id: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
).then(async res => {
|
||||
if (process.env.REDIS_URL) {
|
||||
await redis.set(`session:${res.session_uuid}`, '');
|
||||
}
|
||||
|
||||
return res;
|
||||
});
|
||||
}
|
||||
|
||||
async function clickhouseQuery(
|
||||
@ -47,13 +49,11 @@ async function clickhouseQuery(
|
||||
country ? country : null,
|
||||
];
|
||||
|
||||
await rawQueryClickhouse(
|
||||
await clickhouse.rawQuery(
|
||||
`insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)
|
||||
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
|
||||
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
|
||||
params,
|
||||
);
|
||||
|
||||
return getSessionByUuid(session_uuid);
|
||||
}
|
||||
|
||||
async function kafkaQuery(
|
||||
@ -63,7 +63,7 @@ async function kafkaQuery(
|
||||
const params = {
|
||||
session_uuid: session_uuid,
|
||||
website_id: website_id,
|
||||
created_at: getDateFormatKafka(new Date()),
|
||||
created_at: kafka.getDateFormat(new Date()),
|
||||
hostname: hostname,
|
||||
browser: browser,
|
||||
os: os,
|
||||
@ -73,7 +73,7 @@ async function kafkaQuery(
|
||||
country: country ? country : null,
|
||||
};
|
||||
|
||||
await kafkaProducer(params, 'session');
|
||||
await kafka.sendKafkaMessage(params, 'session');
|
||||
|
||||
return getSessionByUuid(session_uuid);
|
||||
await redis.set(`session:${session_uuid}`, '');
|
||||
}
|
||||
|
@ -1,5 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import { rawQueryClickhouse, findUnique, prisma, runAnalyticsQuery, runQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
|
||||
export async function getSessionByUuid(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -21,8 +23,9 @@ async function relationalQuery(session_uuid) {
|
||||
async function clickhouseQuery(session_uuid) {
|
||||
const params = [session_uuid];
|
||||
|
||||
return rawQueryClickhouse(
|
||||
`
|
||||
return clickhouse
|
||||
.rawQuery(
|
||||
`
|
||||
select
|
||||
session_uuid,
|
||||
website_id,
|
||||
@ -37,6 +40,7 @@ async function clickhouseQuery(session_uuid) {
|
||||
from session
|
||||
where session_uuid = $1
|
||||
`,
|
||||
params,
|
||||
).then(data => findUnique(data));
|
||||
params,
|
||||
)
|
||||
.then(result => clickhouse.findFirst(result));
|
||||
}
|
||||
|
@ -1,11 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
getBetweenDatesClickhouse,
|
||||
parseFilters,
|
||||
rawQuery,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
} from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import { parseFilters, rawQuery } from 'lib/relational';
|
||||
|
||||
export async function getSessionMetrics(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -45,7 +41,7 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}
|
||||
|
||||
async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) {
|
||||
const params = [website_id];
|
||||
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
|
||||
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
|
||||
'pageview',
|
||||
null,
|
||||
filters,
|
||||
@ -53,7 +49,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}
|
||||
'session_uuid',
|
||||
);
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select ${field} x, count(*) y
|
||||
from session as x
|
||||
@ -62,7 +58,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}
|
||||
from pageview
|
||||
${joinSession}
|
||||
where pageview.website_id=$1
|
||||
and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}
|
||||
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
|
||||
${pageviewQuery}
|
||||
${sessionQuery}
|
||||
)
|
||||
|
@ -1,11 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
getDateFormatClickhouse,
|
||||
prisma,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
runQuery,
|
||||
} from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import { prisma, runQuery } from 'lib/relational';
|
||||
|
||||
export async function getSessions(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -18,11 +14,15 @@ async function relationalQuery(websites, start_at) {
|
||||
return runQuery(
|
||||
prisma.session.findMany({
|
||||
where: {
|
||||
website: {
|
||||
website_id: {
|
||||
in: websites,
|
||||
},
|
||||
},
|
||||
...(websites && websites.length > 0
|
||||
? {
|
||||
website: {
|
||||
website_id: {
|
||||
in: websites,
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
created_at: {
|
||||
gte: start_at,
|
||||
},
|
||||
@ -32,10 +32,9 @@ async function relationalQuery(websites, start_at) {
|
||||
}
|
||||
|
||||
async function clickhouseQuery(websites, start_at) {
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select
|
||||
session_id,
|
||||
session_uuid,
|
||||
website_id,
|
||||
created_at,
|
||||
@ -47,8 +46,8 @@ async function clickhouseQuery(websites, start_at) {
|
||||
"language",
|
||||
country
|
||||
from session
|
||||
where website_id in (${websites.join[',']}
|
||||
and created_at >= ${getDateFormatClickhouse(start_at)})
|
||||
where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'}
|
||||
and created_at >= ${clickhouse.getDateFormat(start_at)}
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
@ -1,6 +1,8 @@
|
||||
import { subMinutes } from 'date-fns';
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import { getDateFormatClickhouse, rawQuery, rawQueryClickhouse, runAnalyticsQuery } from 'lib/db';
|
||||
import { rawQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
|
||||
export async function getActiveVisitors(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -27,12 +29,12 @@ async function relationalQuery(website_id) {
|
||||
async function clickhouseQuery(website_id) {
|
||||
const params = [website_id];
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select count(distinct session_uuid) x
|
||||
from pageview
|
||||
where website_id = $1
|
||||
and created_at >= ${getDateFormatClickhouse(subMinutes(new Date(), 5))}
|
||||
and created_at >= ${clickhouse.getDateFormat(subMinutes(new Date(), 5))}
|
||||
`,
|
||||
params,
|
||||
);
|
||||
|
@ -1,14 +1,7 @@
|
||||
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
|
||||
import {
|
||||
getDateQuery,
|
||||
getBetweenDatesClickhouse,
|
||||
getDateQueryClickhouse,
|
||||
getTimestampInterval,
|
||||
parseFilters,
|
||||
rawQuery,
|
||||
rawQueryClickhouse,
|
||||
runAnalyticsQuery,
|
||||
} from 'lib/db';
|
||||
import { getDateQuery, getTimestampInterval, parseFilters, rawQuery } from 'lib/relational';
|
||||
import { runAnalyticsQuery } from 'lib/db';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
|
||||
export async function getWebsiteStats(...args) {
|
||||
return runAnalyticsQuery({
|
||||
@ -52,7 +45,7 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) {
|
||||
|
||||
async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
|
||||
const params = [website_id];
|
||||
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
|
||||
const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
|
||||
'pageview',
|
||||
null,
|
||||
filters,
|
||||
@ -60,7 +53,7 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
|
||||
'session_uuid',
|
||||
);
|
||||
|
||||
return rawQueryClickhouse(
|
||||
return clickhouse.rawQuery(
|
||||
`
|
||||
select
|
||||
sum(t.c) as "pageviews",
|
||||
@ -69,14 +62,14 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
|
||||
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
|
||||
from (
|
||||
select pageview.session_uuid,
|
||||
${getDateQueryClickhouse('pageview.created_at', 'day')} time_series,
|
||||
${clickhouse.getDateQuery('pageview.created_at', 'day')} time_series,
|
||||
count(*) c,
|
||||
min(created_at) min_time,
|
||||
max(created_at) max_time
|
||||
from pageview
|
||||
${joinSession}
|
||||
where pageview.website_id = $1
|
||||
and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}
|
||||
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
|
||||
${pageviewQuery}
|
||||
${sessionQuery}
|
||||
group by pageview.session_uuid, time_series
|
||||
|
58
yarn.lock
58
yarn.lock
@ -1512,6 +1512,40 @@
|
||||
"@react-spring/shared" "~9.5.0"
|
||||
"@react-spring/types" "~9.5.0"
|
||||
|
||||
"@redis/bloom@1.0.2":
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf"
|
||||
integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==
|
||||
|
||||
"@redis/client@1.3.0":
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77"
|
||||
integrity sha512-XCFV60nloXAefDsPnYMjHGtvbtHR8fV5Om8cQ0JYqTNbWcQo/4AryzJ2luRj4blveWazRK/j40gES8M7Cp6cfQ==
|
||||
dependencies:
|
||||
cluster-key-slot "1.1.0"
|
||||
generic-pool "3.8.2"
|
||||
yallist "4.0.0"
|
||||
|
||||
"@redis/graph@1.0.1":
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99"
|
||||
integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==
|
||||
|
||||
"@redis/json@1.0.3":
|
||||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.3.tgz#a13fde1d22ebff0ae2805cd8e1e70522b08ea866"
|
||||
integrity sha512-4X0Qv0BzD9Zlb0edkUoau5c1bInWSICqXAGrpwEltkncUwcxJIGEcVryZhLgb0p/3PkKaLIWkjhHRtLe9yiA7Q==
|
||||
|
||||
"@redis/search@1.1.0":
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df"
|
||||
integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==
|
||||
|
||||
"@redis/time-series@1.0.3":
|
||||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4"
|
||||
integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==
|
||||
|
||||
"@rollup/plugin-buble@^0.21.3":
|
||||
version "0.21.3"
|
||||
resolved "https://registry.npmjs.org/@rollup/plugin-buble/-/plugin-buble-0.21.3.tgz"
|
||||
@ -2458,6 +2492,11 @@ clone-regexp@^2.1.0:
|
||||
dependencies:
|
||||
is-regexp "^2.0.0"
|
||||
|
||||
cluster-key-slot@1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
|
||||
integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
|
||||
|
||||
color-convert@^1.9.0, color-convert@^1.9.3:
|
||||
version "1.9.3"
|
||||
resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz"
|
||||
@ -3573,6 +3612,11 @@ functions-have-names@^1.2.2:
|
||||
resolved "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz"
|
||||
integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==
|
||||
|
||||
generic-pool@3.8.2:
|
||||
version "3.8.2"
|
||||
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
|
||||
integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
|
||||
|
||||
gensync@^1.0.0-beta.2:
|
||||
version "1.0.0-beta.2"
|
||||
resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz"
|
||||
@ -5564,6 +5608,18 @@ redent@^3.0.0:
|
||||
indent-string "^4.0.0"
|
||||
strip-indent "^3.0.0"
|
||||
|
||||
redis@^4.3.0:
|
||||
version "4.3.0"
|
||||
resolved "https://registry.yarnpkg.com/redis/-/redis-4.3.0.tgz#c01de4cf13821406addb7fcb70afe203266c4c39"
|
||||
integrity sha512-RXRUor0iU1vizu4viHoUyLpe1ZO/RngZp0V9DyXBHTI+7tC7rEz6Wzn4Sv9v0tTJeqGAzdJ+q5YVbNKKQ5hX9A==
|
||||
dependencies:
|
||||
"@redis/bloom" "1.0.2"
|
||||
"@redis/client" "1.3.0"
|
||||
"@redis/graph" "1.0.1"
|
||||
"@redis/json" "1.0.3"
|
||||
"@redis/search" "1.1.0"
|
||||
"@redis/time-series" "1.0.3"
|
||||
|
||||
redux@^4.0.0, redux@^4.0.4:
|
||||
version "4.2.0"
|
||||
resolved "https://registry.npmjs.org/redux/-/redux-4.2.0.tgz"
|
||||
@ -6711,7 +6767,7 @@ write-json-file@^4.3.0:
|
||||
sort-keys "^4.0.0"
|
||||
write-file-atomic "^3.0.0"
|
||||
|
||||
yallist@^4.0.0:
|
||||
yallist@4.0.0, yallist@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz"
|
||||
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
|
||||
|
Loading…
Reference in New Issue
Block a user