mirror of
https://github.com/kremalicious/umami.git
synced 2024-11-15 17:55:08 +01:00
Refactored redis usage. Added lib/cache.
This commit is contained in:
parent
3485b6268b
commit
f118bc95c1
@ -1,6 +1,6 @@
|
|||||||
import React from 'react';
|
import React from 'react';
|
||||||
import { useSpring, animated } from 'react-spring';
|
import { useSpring, animated } from 'react-spring';
|
||||||
import { formatNumber } from '../../lib/format';
|
import { formatNumber } from 'lib/format';
|
||||||
import styles from './MetricCard.module.css';
|
import styles from './MetricCard.module.css';
|
||||||
|
|
||||||
const MetricCard = ({
|
const MetricCard = ({
|
||||||
|
@ -5,7 +5,7 @@ import Button from 'components/common/Button';
|
|||||||
import useLocale from 'hooks/useLocale';
|
import useLocale from 'hooks/useLocale';
|
||||||
import { DEFAULT_LOCALE } from 'lib/constants';
|
import { DEFAULT_LOCALE } from 'lib/constants';
|
||||||
import styles from './TimezoneSetting.module.css';
|
import styles from './TimezoneSetting.module.css';
|
||||||
import { languages } from '../../lib/lang';
|
import { languages } from 'lib/lang';
|
||||||
|
|
||||||
export default function LanguageSetting() {
|
export default function LanguageSetting() {
|
||||||
const { locale, saveLocale } = useLocale();
|
const { locale, saveLocale } = useLocale();
|
||||||
|
84
lib/cache.js
84
lib/cache.js
@ -0,0 +1,84 @@
|
|||||||
|
import { getWebsite, getUser, getSession } from '../queries';
|
||||||
|
import redis, { DELETED } from 'lib/redis';
|
||||||
|
|
||||||
|
async function fetchObject(key, query) {
|
||||||
|
const obj = await redis.get(key);
|
||||||
|
|
||||||
|
if (!obj) {
|
||||||
|
return query().then(async data => {
|
||||||
|
if (data) {
|
||||||
|
await redis.set(key, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function storeObject(key, data) {
|
||||||
|
return redis.set(key, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteObject(key) {
|
||||||
|
return redis.set(key, DELETED);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchWebsite(id) {
|
||||||
|
return fetchObject(`website:${id}`, () => getWebsite({ id }));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function storeWebsite(data) {
|
||||||
|
const { id } = data;
|
||||||
|
const key = `website:${id}`;
|
||||||
|
|
||||||
|
return storeObject(key, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteWebsite(id) {
|
||||||
|
return deleteObject(`website:${id}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchUser(id) {
|
||||||
|
return fetchObject(`user:${id}`, () => getUser({ id }));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function storeUser(data) {
|
||||||
|
const { id } = data;
|
||||||
|
const key = `user:${id}`;
|
||||||
|
|
||||||
|
return storeObject(key, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteUser(id) {
|
||||||
|
return deleteObject(`user:${id}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchSession(id) {
|
||||||
|
return fetchObject(`session:${id}`, () => getSession({ id }));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function storeSession(data) {
|
||||||
|
const { id } = data;
|
||||||
|
const key = `session:${id}`;
|
||||||
|
|
||||||
|
return storeObject(key, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteSession(id) {
|
||||||
|
return deleteObject(`session:${id}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default {
|
||||||
|
fetchWebsite,
|
||||||
|
storeWebsite,
|
||||||
|
deleteWebsite,
|
||||||
|
fetchUser,
|
||||||
|
storeUser,
|
||||||
|
deleteUser,
|
||||||
|
fetchSession,
|
||||||
|
storeSession,
|
||||||
|
deleteSession,
|
||||||
|
enabled: redis.enabled,
|
||||||
|
};
|
@ -1,23 +1,19 @@
|
|||||||
import { createMiddleware, unauthorized, badRequest, serverError } from 'next-basics';
|
import { createMiddleware, unauthorized, badRequest } from 'next-basics';
|
||||||
|
import debug from 'debug';
|
||||||
import cors from 'cors';
|
import cors from 'cors';
|
||||||
import { getSession } from './session';
|
import { findSession } from 'lib/session';
|
||||||
import { parseAuthToken, parseShareToken } from './auth';
|
import { parseAuthToken, parseShareToken } from 'lib/auth';
|
||||||
|
import redis from 'lib/redis';
|
||||||
|
|
||||||
|
const log = debug('umami:middleware');
|
||||||
|
|
||||||
export const useCors = createMiddleware(cors());
|
export const useCors = createMiddleware(cors());
|
||||||
|
|
||||||
export const useSession = createMiddleware(async (req, res, next) => {
|
export const useSession = createMiddleware(async (req, res, next) => {
|
||||||
let session;
|
const session = await findSession(req);
|
||||||
|
|
||||||
try {
|
|
||||||
session = await getSession(req);
|
|
||||||
} catch (e) {
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.error(e);
|
|
||||||
|
|
||||||
return serverError(res, e.message);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!session) {
|
if (!session) {
|
||||||
|
log('useSession:session-not-found');
|
||||||
return badRequest(res);
|
return badRequest(res);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -29,10 +25,14 @@ export const useAuth = createMiddleware(async (req, res, next) => {
|
|||||||
const token = await parseAuthToken(req);
|
const token = await parseAuthToken(req);
|
||||||
const shareToken = await parseShareToken(req);
|
const shareToken = await parseShareToken(req);
|
||||||
|
|
||||||
if (!token && !shareToken) {
|
const key = `auth:${token?.authKey}`;
|
||||||
|
const data = redis.enabled ? await redis.get(key) : token;
|
||||||
|
|
||||||
|
if (!data && !shareToken) {
|
||||||
|
log('useAuth:user-not-authorized');
|
||||||
return unauthorized(res);
|
return unauthorized(res);
|
||||||
}
|
}
|
||||||
|
|
||||||
req.auth = { ...token, shareToken };
|
req.auth = { ...data, shareToken };
|
||||||
next();
|
next();
|
||||||
});
|
});
|
||||||
|
@ -32,13 +32,13 @@ function getClient() {
|
|||||||
async function get(key) {
|
async function get(key) {
|
||||||
await connect();
|
await connect();
|
||||||
|
|
||||||
return redis.get(key);
|
return JSON.parse(await redis.get(key));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function set(key, value) {
|
async function set(key, value) {
|
||||||
await connect();
|
await connect();
|
||||||
|
|
||||||
return redis.set(key, value);
|
return redis.set(key, JSON.stringify(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function del(key) {
|
async function del(key) {
|
||||||
|
@ -1,80 +1,73 @@
|
|||||||
import { parseToken } from 'next-basics';
|
import { parseToken } from 'next-basics';
|
||||||
import { validate } from 'uuid';
|
import { validate } from 'uuid';
|
||||||
import { secret, uuid } from 'lib/crypto';
|
import { secret, uuid } from 'lib/crypto';
|
||||||
import redis, { DELETED } from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
import clickhouse from 'lib/clickhouse';
|
|
||||||
import { getClientInfo, getJsonBody } from 'lib/request';
|
import { getClientInfo, getJsonBody } from 'lib/request';
|
||||||
import { createSession, getSession as getSessionPrisma, getWebsite } from 'queries';
|
import { createSession, getSession, getWebsite } from 'queries';
|
||||||
|
|
||||||
export async function getSession(req) {
|
export async function findSession(req) {
|
||||||
const { payload } = getJsonBody(req);
|
const { payload } = getJsonBody(req);
|
||||||
|
|
||||||
if (!payload) {
|
if (!payload) {
|
||||||
throw new Error('Invalid request');
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const cache = req.headers['x-umami-cache'];
|
// Check if cache token is passed
|
||||||
|
const cacheToken = req.headers['x-umami-cache'];
|
||||||
|
|
||||||
if (cache) {
|
if (cacheToken) {
|
||||||
const result = await parseToken(cache, secret());
|
const result = await parseToken(cacheToken, secret());
|
||||||
|
|
||||||
if (result) {
|
if (result) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Verify payload
|
||||||
const { website: websiteId, hostname, screen, language } = payload;
|
const { website: websiteId, hostname, screen, language } = payload;
|
||||||
|
|
||||||
if (!validate(websiteId)) {
|
if (!validate(websiteId)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
let isValidWebsite = null;
|
// Find website
|
||||||
|
let website;
|
||||||
|
|
||||||
// Check if website exists
|
if (cache.enabled) {
|
||||||
if (redis.enabled) {
|
website = await cache.fetchWebsite(websiteId);
|
||||||
isValidWebsite = await redis.get(`website:${websiteId}`);
|
} else {
|
||||||
|
website = await getWebsite({ id: websiteId });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check database if does not exists in Redis
|
if (!website || website.isDeleted) {
|
||||||
if (!isValidWebsite) {
|
|
||||||
const website = await getWebsite({ id: websiteId });
|
|
||||||
isValidWebsite = !!website;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isValidWebsite || isValidWebsite === DELETED) {
|
|
||||||
throw new Error(`Website not found: ${websiteId}`);
|
throw new Error(`Website not found: ${websiteId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);
|
const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);
|
||||||
const sessionId = uuid(websiteId, hostname, ip, userAgent);
|
const sessionId = uuid(websiteId, hostname, ip, userAgent);
|
||||||
|
|
||||||
let isValidSession = null;
|
// Find session
|
||||||
let session = null;
|
let session;
|
||||||
|
|
||||||
if (!clickhouse.enabled) {
|
if (cache.enabled) {
|
||||||
// Check if session exists
|
session = await cache.fetchSession(sessionId);
|
||||||
if (redis.enabled) {
|
} else {
|
||||||
isValidSession = await redis.get(`session:${sessionId}`);
|
session = await getSession({ id: sessionId });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check database if does not exists in Redis
|
// Create a session if not found
|
||||||
if (!isValidSession) {
|
if (!session) {
|
||||||
session = await getSessionPrisma({ id: sessionId });
|
|
||||||
isValidSession = !!session;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isValidSession) {
|
|
||||||
try {
|
try {
|
||||||
session = await createSession(websiteId, {
|
session = await createSession({
|
||||||
id: sessionId,
|
id: sessionId,
|
||||||
|
websiteId,
|
||||||
hostname,
|
hostname,
|
||||||
browser,
|
browser,
|
||||||
os,
|
os,
|
||||||
|
device,
|
||||||
screen,
|
screen,
|
||||||
language,
|
language,
|
||||||
country,
|
country,
|
||||||
device,
|
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (!e.message.toLowerCase().includes('unique constraint')) {
|
if (!e.message.toLowerCase().includes('unique constraint')) {
|
||||||
@ -82,21 +75,6 @@ export async function getSession(req) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
session = {
|
|
||||||
id: sessionId,
|
|
||||||
hostname,
|
|
||||||
browser,
|
|
||||||
os,
|
|
||||||
screen,
|
|
||||||
language,
|
|
||||||
country,
|
|
||||||
device,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return session;
|
||||||
websiteId,
|
|
||||||
session,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
@ -84,7 +84,7 @@
|
|||||||
"maxmind": "^4.3.6",
|
"maxmind": "^4.3.6",
|
||||||
"moment-timezone": "^0.5.35",
|
"moment-timezone": "^0.5.35",
|
||||||
"next": "^12.3.1",
|
"next": "^12.3.1",
|
||||||
"next-basics": "^0.20.0",
|
"next-basics": "^0.23.0",
|
||||||
"node-fetch": "^3.2.8",
|
"node-fetch": "^3.2.8",
|
||||||
"npm-run-all": "^4.1.5",
|
"npm-run-all": "^4.1.5",
|
||||||
"prop-types": "^15.7.2",
|
"prop-types": "^15.7.2",
|
||||||
|
@ -5,7 +5,7 @@ import { createToken, unauthorized, send, badRequest, forbidden } from 'next-bas
|
|||||||
import { savePageView, saveEvent } from 'queries';
|
import { savePageView, saveEvent } from 'queries';
|
||||||
import { useCors, useSession } from 'lib/middleware';
|
import { useCors, useSession } from 'lib/middleware';
|
||||||
import { getJsonBody, getIpAddress } from 'lib/request';
|
import { getJsonBody, getIpAddress } from 'lib/request';
|
||||||
import { secret, uuid } from 'lib/crypto';
|
import { secret } from 'lib/crypto';
|
||||||
|
|
||||||
export default async (req, res) => {
|
export default async (req, res) => {
|
||||||
await useCors(req, res);
|
await useCors(req, res);
|
||||||
@ -58,9 +58,10 @@ export default async (req, res) => {
|
|||||||
|
|
||||||
await useSession(req, res);
|
await useSession(req, res);
|
||||||
|
|
||||||
const { websiteId, session } = req.session;
|
const session = req.session;
|
||||||
|
|
||||||
const { type, payload } = getJsonBody(req);
|
const { type, payload } = getJsonBody(req);
|
||||||
|
const { websiteId } = session;
|
||||||
|
|
||||||
let { url, referrer, event_name: eventName, event_data: eventData } = payload;
|
let { url, referrer, event_name: eventName, event_data: eventData } = payload;
|
||||||
|
|
||||||
@ -69,10 +70,9 @@ export default async (req, res) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (type === 'pageview') {
|
if (type === 'pageview') {
|
||||||
await savePageView(websiteId, { pageViewId: uuid(), session, url, referrer });
|
await savePageView(websiteId, { session, url, referrer });
|
||||||
} else if (type === 'event') {
|
} else if (type === 'event') {
|
||||||
await saveEvent(websiteId, {
|
await saveEvent(websiteId, {
|
||||||
eventId: uuid(),
|
|
||||||
session,
|
session,
|
||||||
url,
|
url,
|
||||||
eventName,
|
eventName,
|
||||||
@ -82,13 +82,7 @@ export default async (req, res) => {
|
|||||||
return badRequest(res);
|
return badRequest(res);
|
||||||
}
|
}
|
||||||
|
|
||||||
const token = createToken(
|
const token = createToken(session, secret());
|
||||||
{
|
|
||||||
websiteId,
|
|
||||||
session,
|
|
||||||
},
|
|
||||||
secret(),
|
|
||||||
);
|
|
||||||
|
|
||||||
return send(res, token);
|
return send(res, token);
|
||||||
};
|
};
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis, { DELETED } from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function deleteUser(userId) {
|
export async function deleteUser(userId) {
|
||||||
const { client } = prisma;
|
const { client } = prisma;
|
||||||
@ -8,12 +8,6 @@ export async function deleteUser(userId) {
|
|||||||
where: { userId },
|
where: { userId },
|
||||||
});
|
});
|
||||||
|
|
||||||
let websiteIds = [];
|
|
||||||
|
|
||||||
if (websites.length > 0) {
|
|
||||||
websiteIds = websites.map(a => a.id);
|
|
||||||
}
|
|
||||||
|
|
||||||
return client
|
return client
|
||||||
.$transaction([
|
.$transaction([
|
||||||
client.pageview.deleteMany({
|
client.pageview.deleteMany({
|
||||||
@ -37,13 +31,15 @@ export async function deleteUser(userId) {
|
|||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
])
|
])
|
||||||
.then(async res => {
|
.then(async data => {
|
||||||
if (redis.enabled) {
|
if (cache.enabled) {
|
||||||
for (let i = 0; i < websiteIds.length; i++) {
|
const ids = websites.map(a => a.id);
|
||||||
await redis.set(`website:${websiteIds[i]}`, DELETED);
|
|
||||||
|
for (let i = 0; i < ids.length; i++) {
|
||||||
|
await cache.deleteWebsite(`website:${ids[i]}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return res;
|
return data;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function createWebsite(userId, data) {
|
export async function createWebsite(userId, data) {
|
||||||
return prisma.client.website
|
return prisma.client.website
|
||||||
@ -13,11 +13,11 @@ export async function createWebsite(userId, data) {
|
|||||||
...data,
|
...data,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.then(async res => {
|
.then(async data => {
|
||||||
if (redis.enabled && res) {
|
if (cache.enabled) {
|
||||||
await redis.set(`website:${res.id}`, res);
|
await cache.storeWebsite(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
return res;
|
return data;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis, { DELETED } from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function deleteWebsite(id) {
|
export async function deleteWebsite(id) {
|
||||||
const { client, transaction } = prisma;
|
const { client, transaction } = prisma;
|
||||||
@ -20,11 +20,11 @@ export async function deleteWebsite(id) {
|
|||||||
client.website.delete({
|
client.website.delete({
|
||||||
where: { id },
|
where: { id },
|
||||||
}),
|
}),
|
||||||
]).then(async res => {
|
]).then(async data => {
|
||||||
if (redis.enabled) {
|
if (cache.enabled) {
|
||||||
await redis.set(`website:${id}`, DELETED);
|
await cache.deleteWebsite(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
return res;
|
return data;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -1,16 +1,7 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis from 'lib/redis';
|
|
||||||
|
|
||||||
export async function getWebsite(where) {
|
export async function getWebsite(where) {
|
||||||
return prisma.client.website
|
return prisma.client.website.findUnique({
|
||||||
.findUnique({
|
|
||||||
where,
|
where,
|
||||||
})
|
|
||||||
.then(async data => {
|
|
||||||
if (redis.enabled && data) {
|
|
||||||
await redis.set(`website:${data.id}`, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
return data;
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import clickhouse from 'lib/clickhouse';
|
import clickhouse from 'lib/clickhouse';
|
||||||
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function getEventData(...args) {
|
export async function getEventData(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -41,7 +41,7 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, colu
|
|||||||
async function clickhouseQuery(websiteId, { startDate, endDate, event_name, columns, filters }) {
|
async function clickhouseQuery(websiteId, { startDate, endDate, event_name, columns, filters }) {
|
||||||
const { rawQuery, getBetweenDates, getEventDataColumnsQuery, getEventDataFilterQuery } =
|
const { rawQuery, getBetweenDates, getEventDataColumnsQuery, getEventDataFilterQuery } =
|
||||||
clickhouse;
|
clickhouse;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
const params = [websiteId, website?.revId || 0];
|
const params = [websiteId, website?.revId || 0];
|
||||||
|
|
||||||
return rawQuery(
|
return rawQuery(
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import clickhouse from 'lib/clickhouse';
|
import clickhouse from 'lib/clickhouse';
|
||||||
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function getEventMetrics(...args) {
|
export async function getEventMetrics(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -47,7 +47,7 @@ async function clickhouseQuery(
|
|||||||
filters = {},
|
filters = {},
|
||||||
) {
|
) {
|
||||||
const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse;
|
const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
const params = [websiteId, website?.revId || 0];
|
const params = [websiteId, website?.revId || 0];
|
||||||
|
|
||||||
return rawQuery(
|
return rawQuery(
|
||||||
|
@ -3,7 +3,7 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
|||||||
import kafka from 'lib/kafka';
|
import kafka from 'lib/kafka';
|
||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import { uuid } from 'lib/crypto';
|
import { uuid } from 'lib/crypto';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function saveEvent(...args) {
|
export async function saveEvent(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -12,52 +12,49 @@ export async function saveEvent(...args) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function relationalQuery(
|
async function relationalQuery(data) {
|
||||||
websiteId,
|
const { websiteId, sessionId, url, eventName, eventData } = data;
|
||||||
{ eventId, session: { id: sessionId }, eventUuid, url, eventName, eventData },
|
const eventId = uuid();
|
||||||
) {
|
|
||||||
const data = {
|
const params = {
|
||||||
id: eventId,
|
id: eventId,
|
||||||
websiteId,
|
websiteId,
|
||||||
sessionId,
|
sessionId,
|
||||||
url: url?.substring(0, URL_LENGTH),
|
url: url?.substring(0, URL_LENGTH),
|
||||||
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
|
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
|
||||||
eventUuid,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if (eventData) {
|
if (eventData) {
|
||||||
data.eventData = {
|
params.eventData = {
|
||||||
create: {
|
create: {
|
||||||
|
id: eventId,
|
||||||
eventData: eventData,
|
eventData: eventData,
|
||||||
id: uuid(),
|
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return prisma.client.event.create({
|
return prisma.client.event.create({
|
||||||
data,
|
data: params,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clickhouseQuery(
|
async function clickhouseQuery(data) {
|
||||||
websiteId,
|
const { websiteId, sessionId, url, eventName, eventData } = data;
|
||||||
{ session: { country, sessionUuid, ...sessionArgs }, eventUuid, url, eventName, eventData },
|
|
||||||
) {
|
|
||||||
const { getDateFormat, sendMessage } = kafka;
|
const { getDateFormat, sendMessage } = kafka;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
|
|
||||||
const params = {
|
const params = {
|
||||||
session_id: sessionUuid,
|
|
||||||
event_id: eventUuid,
|
|
||||||
website_id: websiteId,
|
website_id: websiteId,
|
||||||
rev_id: website?.revId || 0,
|
session_id: sessionId,
|
||||||
created_at: getDateFormat(new Date()),
|
event_id: uuid(),
|
||||||
url: url?.substring(0, URL_LENGTH),
|
url: url?.substring(0, URL_LENGTH),
|
||||||
event_name: eventName?.substring(0, EVENT_NAME_LENGTH),
|
event_name: eventName?.substring(0, EVENT_NAME_LENGTH),
|
||||||
event_data: eventData ? JSON.stringify(eventData) : null,
|
event_data: eventData ? JSON.stringify(eventData) : null,
|
||||||
...sessionArgs,
|
rev_id: website?.revId || 0,
|
||||||
country: country ? country : null,
|
created_at: getDateFormat(new Date()),
|
||||||
};
|
};
|
||||||
|
|
||||||
await sendMessage(params, 'event');
|
await sendMessage(params, 'event');
|
||||||
|
|
||||||
|
return data;
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import clickhouse from 'lib/clickhouse';
|
import clickhouse from 'lib/clickhouse';
|
||||||
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function getPageviewMetrics(...args) {
|
export async function getPageviewMetrics(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -38,7 +38,7 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, f
|
|||||||
|
|
||||||
async function clickhouseQuery(websiteId, { startDate, endDate, column, filters = {} }) {
|
async function clickhouseQuery(websiteId, { startDate, endDate, column, filters = {} }) {
|
||||||
const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
|
const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
const params = [websiteId, website?.revId || 0];
|
const params = [websiteId, website?.revId || 0];
|
||||||
const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params);
|
const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params);
|
||||||
|
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import clickhouse from 'lib/clickhouse';
|
import clickhouse from 'lib/clickhouse';
|
||||||
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function getPageviewStats(...args) {
|
export async function getPageviewStats(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -52,7 +52,7 @@ async function clickhouseQuery(
|
|||||||
{ start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} },
|
{ start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} },
|
||||||
) {
|
) {
|
||||||
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
|
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
const params = [websiteId, website?.revId || 0];
|
const params = [websiteId, website?.revId || 0];
|
||||||
const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
|
const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
|
||||||
|
|
||||||
|
@ -2,7 +2,8 @@ import { URL_LENGTH } from 'lib/constants';
|
|||||||
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
||||||
import kafka from 'lib/kafka';
|
import kafka from 'lib/kafka';
|
||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
import { uuid } from 'lib/crypto';
|
||||||
|
|
||||||
export async function savePageView(...args) {
|
export async function savePageView(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -11,13 +12,11 @@ export async function savePageView(...args) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function relationalQuery(
|
async function relationalQuery(data) {
|
||||||
websiteId,
|
const { websiteId, sessionId, url, referrer } = data;
|
||||||
{ pageViewId, session: { id: sessionId }, url, referrer },
|
|
||||||
) {
|
|
||||||
return prisma.client.pageview.create({
|
return prisma.client.pageview.create({
|
||||||
data: {
|
data: {
|
||||||
id: pageViewId,
|
id: uuid(),
|
||||||
websiteId,
|
websiteId,
|
||||||
sessionId,
|
sessionId,
|
||||||
url: url?.substring(0, URL_LENGTH),
|
url: url?.substring(0, URL_LENGTH),
|
||||||
@ -26,22 +25,21 @@ async function relationalQuery(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clickhouseQuery(
|
async function clickhouseQuery(data) {
|
||||||
websiteId,
|
const { websiteId, sessionId, url, referrer } = data;
|
||||||
{ session: { country, id: sessionId, ...sessionArgs }, url, referrer },
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
) {
|
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
|
||||||
const { getDateFormat, sendMessage } = kafka;
|
const { getDateFormat, sendMessage } = kafka;
|
||||||
const params = {
|
|
||||||
|
const msg = {
|
||||||
session_id: sessionId,
|
session_id: sessionId,
|
||||||
website_id: websiteId,
|
website_id: websiteId,
|
||||||
created_at: getDateFormat(new Date()),
|
|
||||||
url: url?.substring(0, URL_LENGTH),
|
url: url?.substring(0, URL_LENGTH),
|
||||||
referrer: referrer?.substring(0, URL_LENGTH),
|
referrer: referrer?.substring(0, URL_LENGTH),
|
||||||
rev_id: website?.revId || 0,
|
rev_id: website?.revId || 0,
|
||||||
...sessionArgs,
|
created_at: getDateFormat(new Date()),
|
||||||
country: country ? country : null,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
await sendMessage(params, 'event');
|
await sendMessage(msg, 'event');
|
||||||
|
|
||||||
|
return data;
|
||||||
}
|
}
|
||||||
|
@ -1,62 +1,45 @@
|
|||||||
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
||||||
import kafka from 'lib/kafka';
|
import kafka from 'lib/kafka';
|
||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function createSession(...args) {
|
export async function createSession(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
[PRISMA]: () => relationalQuery(...args),
|
[PRISMA]: () => relationalQuery(...args),
|
||||||
[CLICKHOUSE]: () => clickhouseQuery(...args),
|
[CLICKHOUSE]: () => clickhouseQuery(...args),
|
||||||
}).then(async data => {
|
}).then(async data => {
|
||||||
if (redis.enabled && data) {
|
if (cache.enabled) {
|
||||||
await redis.set(`session:${data.id}`, data);
|
await cache.storeSession(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
return data;
|
return data;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function relationalQuery(websiteId, data) {
|
async function relationalQuery(data) {
|
||||||
return prisma.client.session.create({
|
return prisma.client.session.create({ data });
|
||||||
data: {
|
|
||||||
websiteId,
|
|
||||||
...data,
|
|
||||||
},
|
|
||||||
select: {
|
|
||||||
id: true,
|
|
||||||
hostname: true,
|
|
||||||
browser: true,
|
|
||||||
os: true,
|
|
||||||
screen: true,
|
|
||||||
language: true,
|
|
||||||
country: true,
|
|
||||||
device: true,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clickhouseQuery(
|
async function clickhouseQuery(data) {
|
||||||
websiteId,
|
const { id, websiteId, hostname, browser, os, device, screen, language, country } = data;
|
||||||
{ sessionId, hostname, browser, os, screen, language, country, device },
|
|
||||||
) {
|
|
||||||
const { getDateFormat, sendMessage } = kafka;
|
const { getDateFormat, sendMessage } = kafka;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
|
|
||||||
const data = {
|
const msg = {
|
||||||
sessionId,
|
session_id: id,
|
||||||
website_id: websiteId,
|
website_id: websiteId,
|
||||||
rev_id: website?.revId || 0,
|
|
||||||
created_at: getDateFormat(new Date()),
|
|
||||||
hostname,
|
hostname,
|
||||||
browser,
|
browser,
|
||||||
os,
|
os,
|
||||||
device,
|
device,
|
||||||
screen,
|
screen,
|
||||||
language,
|
language,
|
||||||
country: country ? country : null,
|
country,
|
||||||
|
rev_id: website?.revId || 0,
|
||||||
|
created_at: getDateFormat(new Date()),
|
||||||
};
|
};
|
||||||
|
|
||||||
await sendMessage(data, 'event');
|
await sendMessage(msg, 'event');
|
||||||
|
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
@ -1,18 +1,11 @@
|
|||||||
import clickhouse from 'lib/clickhouse';
|
import clickhouse from 'lib/clickhouse';
|
||||||
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
|
||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import redis from 'lib/redis';
|
|
||||||
|
|
||||||
export async function getSession(...args) {
|
export async function getSession(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
[PRISMA]: () => relationalQuery(...args),
|
[PRISMA]: () => relationalQuery(...args),
|
||||||
[CLICKHOUSE]: () => clickhouseQuery(...args),
|
[CLICKHOUSE]: () => clickhouseQuery(...args),
|
||||||
}).then(async data => {
|
|
||||||
if (redis.enabled && data) {
|
|
||||||
await redis.set(`session:${data.id}`, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
return data;
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -27,7 +20,7 @@ async function clickhouseQuery(sessionId) {
|
|||||||
const params = [sessionId];
|
const params = [sessionId];
|
||||||
|
|
||||||
return rawQuery(
|
return rawQuery(
|
||||||
`select distinct
|
`select
|
||||||
session_id,
|
session_id,
|
||||||
website_id,
|
website_id,
|
||||||
created_at,
|
created_at,
|
||||||
@ -39,7 +32,8 @@ async function clickhouseQuery(sessionId) {
|
|||||||
language,
|
language,
|
||||||
country
|
country
|
||||||
from event
|
from event
|
||||||
where session_id = $1`,
|
where session_id = $1
|
||||||
|
limit 1`,
|
||||||
params,
|
params,
|
||||||
).then(result => findFirst(result));
|
).then(result => findFirst(result));
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import prisma from 'lib/prisma';
|
import prisma from 'lib/prisma';
|
||||||
import clickhouse from 'lib/clickhouse';
|
import clickhouse from 'lib/clickhouse';
|
||||||
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
|
||||||
import redis from 'lib/redis';
|
import cache from 'lib/cache';
|
||||||
|
|
||||||
export async function getSessionMetrics(...args) {
|
export async function getSessionMetrics(...args) {
|
||||||
return runQuery({
|
return runQuery({
|
||||||
@ -37,7 +37,7 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters =
|
|||||||
|
|
||||||
async function clickhouseQuery(websiteId, { startDate, endDate, field, filters = {} }) {
|
async function clickhouseQuery(websiteId, { startDate, endDate, field, filters = {} }) {
|
||||||
const { parseFilters, getBetweenDates, rawQuery } = clickhouse;
|
const { parseFilters, getBetweenDates, rawQuery } = clickhouse;
|
||||||
const website = await redis.get(`website:${websiteId}`);
|
const website = await cache.fetchWebsite(websiteId);
|
||||||
const params = [websiteId, website?.revId || 0];
|
const params = [websiteId, website?.revId || 0];
|
||||||
const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
|
const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
|
||||||
|
|
||||||
|
@ -5344,10 +5344,10 @@ natural-compare@^1.4.0:
|
|||||||
resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz"
|
resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz"
|
||||||
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
|
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
|
||||||
|
|
||||||
next-basics@^0.20.0:
|
next-basics@^0.23.0:
|
||||||
version "0.20.0"
|
version "0.23.0"
|
||||||
resolved "https://registry.yarnpkg.com/next-basics/-/next-basics-0.20.0.tgz#f88791dcc538438b5044695e37d58b4fdccc6dd8"
|
resolved "https://registry.yarnpkg.com/next-basics/-/next-basics-0.23.0.tgz#6ccc4b5f3cd87550b07d280c2a3bdee26f557912"
|
||||||
integrity sha512-KPqVVSzkKUvu9shvZt5Bp7Xv1nZ2xJRRqwqQ+a6a5JjsdE10Q3p07VLrT2ykl+v/CvR4sz98c0n+MaWpgO3Ckw==
|
integrity sha512-AwKipMFkxS84ZKFDVSo8ZHXoOVzF2sPZp21pgtGlqf9aUmj48+QtxdM6ZnJVRg8Savf48ENjeYzeiu5I8Z4l/w==
|
||||||
dependencies:
|
dependencies:
|
||||||
base-x "^4.0.0"
|
base-x "^4.0.0"
|
||||||
bcryptjs "^2.4.3"
|
bcryptjs "^2.4.3"
|
||||||
|
Loading…
Reference in New Issue
Block a user