add back lib folder

src/lib/auth.ts

@@ -0,0 +1,226 @@
+import { Report } from '@prisma/client';
+import debug from 'debug';
+import redis from '@umami/redis-client';
+import { PERMISSIONS, ROLE_PERMISSIONS, SHARE_TOKEN_HEADER } from 'lib/constants';
+import { secret } from 'lib/crypto';
+import { createSecureToken, ensureArray, getRandomChars, parseToken } from 'next-basics';
+import { findTeamWebsiteByUserId, getTeamUser, getTeamWebsite } from 'queries';
+import { loadWebsite } from './load';
+import { Auth } from './types';
+import { NextApiRequest } from 'next';
+
+const log = debug('umami:auth');
+const cloudMode = process.env.CLOUD_MODE;
+
+export async function saveAuth(data: any, expire = 0) {
+  const authKey = `auth:${getRandomChars(32)}`;
+
+  await redis.client.set(authKey, data);
+
+  if (expire) {
+    await redis.client.expire(authKey, expire);
+  }
+
+  return createSecureToken({ authKey }, secret());
+}
+
+export function getAuthToken(req: NextApiRequest) {
+  try {
+    return req.headers.authorization.split(' ')[1];
+  } catch {
+    return null;
+  }
+}
+
+export function parseShareToken(req: Request) {
+  try {
+    return parseToken(req.headers[SHARE_TOKEN_HEADER], secret());
+  } catch (e) {
+    log(e);
+    return null;
+  }
+}
+
+export async function canViewWebsite({ user, shareToken }: Auth, websiteId: string) {
+  if (user?.isAdmin) {
+    return true;
+  }
+
+  if (shareToken?.websiteId === websiteId) {
+    return true;
+  }
+
+  const website = await loadWebsite(websiteId);
+
+  if (user.id === website?.userId) {
+    return true;
+  }
+
+  return !!(await findTeamWebsiteByUserId(websiteId, user.id));
+}
+
+export async function canViewAllWebsites({ user }: Auth) {
+  return user.isAdmin;
+}
+
+export async function canCreateWebsite({ user, grant }: Auth) {
+  if (cloudMode) {
+    return !!grant?.find(a => a === PERMISSIONS.websiteCreate);
+  }
+
+  if (user.isAdmin) {
+    return true;
+  }
+
+  return hasPermission(user.role, PERMISSIONS.websiteCreate);
+}
+
+export async function canUpdateWebsite({ user }: Auth, websiteId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  const website = await loadWebsite(websiteId);
+
+  return user.id === website?.userId;
+}
+
+export async function canDeleteWebsite({ user }: Auth, websiteId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  const website = await loadWebsite(websiteId);
+
+  return user.id === website?.userId;
+}
+
+export async function canViewReport(auth: Auth, report: Report) {
+  if (auth.user.isAdmin) {
+    return true;
+  }
+
+  if (auth.user.id == report.userId) {
+    return true;
+  }
+
+  return !!(await canViewWebsite(auth, report.websiteId));
+}
+
+export async function canUpdateReport({ user }: Auth, report: Report) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  return user.id == report.userId;
+}
+
+export async function canDeleteReport(auth: Auth, report: Report) {
+  return canUpdateReport(auth, report);
+}
+
+export async function canCreateTeam({ user, grant }: Auth) {
+  if (cloudMode) {
+    return !!grant?.find(a => a === PERMISSIONS.teamCreate);
+  }
+
+  if (user.isAdmin) {
+    return true;
+  }
+
+  return !!user;
+}
+
+export async function canViewTeam({ user }: Auth, teamId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  return getTeamUser(teamId, user.id);
+}
+
+export async function canUpdateTeam({ user }: Auth, teamId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  const teamUser = await getTeamUser(teamId, user.id);
+
+  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamUpdate);
+}
+
+export async function canDeleteTeam({ user }: Auth, teamId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  const teamUser = await getTeamUser(teamId, user.id);
+
+  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamDelete);
+}
+
+export async function canDeleteTeamUser({ user }: Auth, teamId: string, removeUserId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  if (removeUserId === user.id) {
+    return true;
+  }
+
+  const teamUser = await getTeamUser(teamId, user.id);
+
+  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamUpdate);
+}
+
+export async function canDeleteTeamWebsite({ user }: Auth, teamId: string, websiteId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  const teamWebsite = await getTeamWebsite(teamId, websiteId);
+
+  if (teamWebsite?.website?.userId === user.id) {
+    return true;
+  }
+
+  if (teamWebsite) {
+    const teamUser = await getTeamUser(teamWebsite.teamId, user.id);
+
+    return hasPermission(teamUser.role, PERMISSIONS.teamUpdate);
+  }
+
+  return false;
+}
+
+export async function canCreateUser({ user }: Auth) {
+  return user.isAdmin;
+}
+
+export async function canViewUser({ user }: Auth, viewedUserId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  return user.id === viewedUserId;
+}
+
+export async function canViewUsers({ user }: Auth) {
+  return user.isAdmin;
+}
+
+export async function canUpdateUser({ user }: Auth, viewedUserId: string) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  return user.id === viewedUserId;
+}
+
+export async function canDeleteUser({ user }: Auth) {
+  return user.isAdmin;
+}
+
+export async function hasPermission(role: string, permission: string | string[]) {
+  return ensureArray(permission).some(e => ROLE_PERMISSIONS[role]?.includes(e));
+}

src/lib/cache.ts

@@ -0,0 +1,86 @@
+import { User, Website } from '@prisma/client';
+import redis from '@umami/redis-client';
+import { getSession, getUserById, getWebsiteById } from '../queries';
+
+async function fetchWebsite(id): Promise<Website> {
+  return redis.client.getCache(`website:${id}`, () => getWebsiteById(id), 86400);
+}
+
+async function storeWebsite(data) {
+  const { id } = data;
+  const key = `website:${id}`;
+
+  const obj = await redis.client.setCache(key, data);
+  await redis.client.expire(key, 86400);
+
+  return obj;
+}
+
+async function deleteWebsite(id) {
+  return redis.client.deleteCache(`website:${id}`);
+}
+
+async function fetchUser(id): Promise<User> {
+  return redis.client.getCache(
+    `user:${id}`,
+    () => getUserById(id, { includePassword: true }),
+    86400,
+  );
+}
+
+async function storeUser(data) {
+  const { id } = data;
+  const key = `user:${id}`;
+
+  const obj = await redis.client.setCache(key, data);
+  await redis.client.expire(key, 86400);
+
+  return obj;
+}
+
+async function deleteUser(id) {
+  return redis.client.deleteCache(`user:${id}`);
+}
+
+async function fetchSession(id) {
+  return redis.client.getCache(`session:${id}`, () => getSession(id), 86400);
+}
+
+async function storeSession(data) {
+  const { id } = data;
+  const key = `session:${id}`;
+
+  const obj = await redis.client.setCache(key, data);
+  await redis.client.expire(key, 86400);
+
+  return obj;
+}
+
+async function deleteSession(id) {
+  return redis.client.deleteCache(`session:${id}`);
+}
+
+async function fetchUserBlock(userId: string) {
+  const key = `user:block:${userId}`;
+  return redis.client.get(key);
+}
+
+async function incrementUserBlock(userId: string) {
+  const key = `user:block:${userId}`;
+  return redis.client.incr(key);
+}
+
+export default {
+  fetchWebsite,
+  storeWebsite,
+  deleteWebsite,
+  fetchUser,
+  storeUser,
+  deleteUser,
+  fetchSession,
+  storeSession,
+  deleteSession,
+  fetchUserBlock,
+  incrementUserBlock,
+  enabled: !!redis.enabled,
+};

src/lib/charts.tsx

@@ -0,0 +1,62 @@
+import { StatusLight } from 'react-basics';
+import { formatDate } from 'lib/date';
+import { formatLongNumber } from 'lib/format';
+
+export function renderNumberLabels(label: string) {
+  return +label > 1000 ? formatLongNumber(+label) : label;
+}
+
+export function renderDateLabels(unit: string, locale: string) {
+  return (label: string, index: number, values: any[]) => {
+    const d = new Date(values[index].value);
+
+    switch (unit) {
+      case 'minute':
+        return formatDate(d, 'h:mm', locale);
+      case 'hour':
+        return formatDate(d, 'p', locale);
+      case 'day':
+        return formatDate(d, 'MMM d', locale);
+      case 'month':
+        return formatDate(d, 'MMM', locale);
+      case 'year':
+        return formatDate(d, 'YYY', locale);
+      default:
+        return label;
+    }
+  };
+}
+
+export function renderStatusTooltipPopup(unit: string, locale: string) {
+  return (setTooltipPopup: (data: any) => void, model: any) => {
+    const { opacity, labelColors, dataPoints } = model.tooltip;
+
+    if (!dataPoints?.length || !opacity) {
+      setTooltipPopup(null);
+      return;
+    }
+
+    const formats = {
+      millisecond: 'T',
+      second: 'pp',
+      minute: 'p',
+      hour: 'h:mm aaa - PP',
+      day: 'PPPP',
+      week: 'PPPP',
+      month: 'LLLL yyyy',
+      quarter: 'qqq',
+      year: 'yyyy',
+    };
+
+    setTooltipPopup(
+      <>
+        <div>{formatDate(new Date(dataPoints[0].raw.x), formats[unit], locale)}</div>
+        <div>
+          <StatusLight color={labelColors?.[0]?.backgroundColor}>
+            {formatLongNumber(dataPoints[0].raw.y)} {dataPoints[0].dataset.label}
+          </StatusLight>
+        </div>
+      </>,
+    );
+  };
+}

src/lib/clickhouse.ts

@@ -0,0 +1,171 @@
+import { ClickHouseClient, createClient } from '@clickhouse/client';
+import dateFormat from 'dateformat';
+import debug from 'debug';
+import { CLICKHOUSE } from 'lib/db';
+import { QueryFilters, QueryOptions } from './types';
+import { FILTER_COLUMNS, OPERATORS } from './constants';
+import { loadWebsite } from './load';
+import { maxDate } from './date';
+
+export const CLICKHOUSE_DATE_FORMATS = {
+  minute: '%Y-%m-%d %H:%M:00',
+  hour: '%Y-%m-%d %H:00:00',
+  day: '%Y-%m-%d',
+  month: '%Y-%m-01',
+  year: '%Y-01-01',
+};
+
+const log = debug('umami:clickhouse');
+
+let clickhouse: ClickHouseClient;
+const enabled = Boolean(process.env.CLICKHOUSE_URL);
+
+function getClient() {
+  const {
+    hostname,
+    port,
+    pathname,
+    protocol,
+    username = 'default',
+    password,
+  } = new URL(process.env.CLICKHOUSE_URL);
+
+  const client = createClient({
+    host: `${protocol}//${hostname}:${port}`,
+    database: pathname.replace('/', ''),
+    username: username,
+    password,
+  });
+
+  if (process.env.NODE_ENV !== 'production') {
+    global[CLICKHOUSE] = client;
+  }
+
+  log('Clickhouse initialized');
+
+  return client;
+}
+
+function getDateStringQuery(data, unit) {
+  return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
+}
+
+function getDateQuery(field, unit, timezone?) {
+  if (timezone) {
+    return `date_trunc('${unit}', ${field}, '${timezone}')`;
+  }
+  return `date_trunc('${unit}', ${field})`;
+}
+
+function getDateFormat(date) {
+  return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
+}
+
+function mapFilter(column, operator, name, type = 'String') {
+  switch (operator) {
+    case OPERATORS.equals:
+      return `${column} = {${name}:${type}}`;
+    case OPERATORS.notEquals:
+      return `${column} != {${name}:${type}}`;
+    default:
+      return '';
+  }
+}
+
+function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
+  const query = Object.keys(filters).reduce((arr, name) => {
+    const value = filters[name];
+    const operator = value?.filter ?? OPERATORS.equals;
+    const column = FILTER_COLUMNS[name] ?? options?.columns?.[name];
+
+    if (value !== undefined && column) {
+      arr.push(`and ${mapFilter(column, operator, name)}`);
+
+      if (name === 'referrer') {
+        arr.push('and referrer_domain != {websiteDomain:String}');
+      }
+    }
+
+    return arr;
+  }, []);
+
+  return query.join('\n');
+}
+
+function normalizeFilters(filters = {}) {
+  return Object.keys(filters).reduce((obj, key) => {
+    const value = filters[key];
+
+    obj[key] = value?.value ?? value;
+
+    return obj;
+  }, {});
+}
+
+async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) {
+  const website = await loadWebsite(websiteId);
+
+  return {
+    filterQuery: getFilterQuery(filters, options),
+    params: {
+      ...normalizeFilters(filters),
+      websiteId,
+      startDate: maxDate(filters.startDate, new Date(website.resetAt)),
+      websiteDomain: website.domain,
+    },
+  };
+}
+
+async function rawQuery(query: string, params: Record<string, unknown> = {}): Promise<unknown> {
+  if (process.env.LOG_QUERY) {
+    log('QUERY:\n', query);
+    log('PARAMETERS:\n', params);
+  }
+
+  await connect();
+
+  const resultSet = await clickhouse.query({
+    query: query,
+    query_params: params,
+    format: 'JSONEachRow',
+  });
+
+  const data = await resultSet.json();
+
+  return data;
+}
+
+async function findUnique(data) {
+  if (data.length > 1) {
+    throw `${data.length} records found when expecting 1.`;
+  }
+
+  return findFirst(data);
+}
+
+async function findFirst(data) {
+  return data[0] ?? null;
+}
+
+async function connect() {
+  if (enabled && !clickhouse) {
+    clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient());
+  }
+
+  return clickhouse;
+}
+
+export default {
+  enabled,
+  client: clickhouse,
+  log,
+  connect,
+  getDateStringQuery,
+  getDateQuery,
+  getDateFormat,
+  getFilterQuery,
+  parseFilters,
+  findUnique,
+  findFirst,
+  rawQuery,
+};

src/lib/client.ts

@@ -0,0 +1,14 @@
+import { getItem, setItem, removeItem } from 'next-basics';
+import { AUTH_TOKEN } from './constants';
+
+export function getClientAuthToken() {
+  return getItem(AUTH_TOKEN);
+}
+
+export function setClientAuthToken(token: string) {
+  setItem(AUTH_TOKEN, token);
+}
+
+export function removeClientAuthToken() {
+  removeItem(AUTH_TOKEN);
+}

src/lib/constants.ts

@@ -0,0 +1,521 @@
+/* eslint-disable no-unused-vars */
+export const CURRENT_VERSION = process.env.currentVersion;
+export const AUTH_TOKEN = 'umami.auth';
+export const LOCALE_CONFIG = 'umami.locale';
+export const TIMEZONE_CONFIG = 'umami.timezone';
+export const DATE_RANGE_CONFIG = 'umami.date-range';
+export const THEME_CONFIG = 'umami.theme';
+export const DASHBOARD_CONFIG = 'umami.dashboard';
+export const VERSION_CHECK = 'umami.version-check';
+export const SHARE_TOKEN_HEADER = 'x-umami-share-token';
+export const HOMEPAGE_URL = 'https://umami.is';
+export const REPO_URL = 'https://github.com/umami-software/umami';
+export const UPDATES_URL = 'https://api.umami.is/v1/updates';
+export const TELEMETRY_PIXEL = 'https://i.umami.is/a.png';
+
+export const DEFAULT_LOCALE = process.env.defaultLocale || 'en-US';
+export const DEFAULT_THEME = 'light';
+export const DEFAULT_ANIMATION_DURATION = 300;
+export const DEFAULT_DATE_RANGE = '24hour';
+export const DEFAULT_WEBSITE_LIMIT = 10;
+export const DEFAULT_RESET_DATE = '2000-01-01';
+export const DEFAULT_PAGE_SIZE = 10;
+
+export const REALTIME_RANGE = 30;
+export const REALTIME_INTERVAL = 5000;
+
+export const FILTER_COMBINED = 'filter-combined';
+export const FILTER_RAW = 'filter-raw';
+export const FILTER_DAY = 'filter-day';
+export const FILTER_RANGE = 'filter-range';
+export const FILTER_REFERRERS = 'filter-referrers';
+export const FILTER_PAGES = 'filter-pages';
+export const UNIT_TYPES = ['year', 'month', 'hour', 'day'];
+export const EVENT_COLUMNS = ['url', 'referrer', 'title', 'query', 'event'];
+
+export const SESSION_COLUMNS = [
+  'browser',
+  'os',
+  'device',
+  'screen',
+  'language',
+  'country',
+  'region',
+  'city',
+];
+
+export const FILTER_COLUMNS = {
+  url: 'url_path',
+  referrer: 'referrer_domain',
+  title: 'page_title',
+  query: 'url_query',
+  os: 'os',
+  browser: 'browser',
+  device: 'device',
+  country: 'country',
+  region: 'subdivision1',
+  city: 'city',
+  language: 'language',
+  event: 'event_name',
+};
+
+export const COLLECTION_TYPE = {
+  event: 'event',
+  identify: 'identify',
+};
+
+export const EVENT_TYPE = {
+  pageView: 1,
+  customEvent: 2,
+} as const;
+
+export const DATA_TYPE = {
+  string: 1,
+  number: 2,
+  boolean: 3,
+  date: 4,
+  array: 5,
+} as const;
+
+export const OPERATORS = {
+  equals: 'eq',
+  notEquals: 'neq',
+  set: 's',
+  notSet: 'ns',
+  contains: 'c',
+  doesNotContain: 'dnc',
+  true: 't',
+  false: 'f',
+  greaterThan: 'gt',
+  lessThan: 'lt',
+  greaterThanEquals: 'gte',
+  lessThanEquals: 'lte',
+  before: 'bf',
+  after: 'af',
+} as const;
+
+export const DATA_TYPES = {
+  [DATA_TYPE.string]: 'string',
+  [DATA_TYPE.number]: 'number',
+  [DATA_TYPE.boolean]: 'boolean',
+  [DATA_TYPE.date]: 'date',
+  [DATA_TYPE.array]: 'array',
+};
+
+export const REPORT_TYPES = {
+  funnel: 'funnel',
+  insights: 'insights',
+  retention: 'retention',
+} as const;
+
+export const REPORT_PARAMETERS = {
+  fields: 'fields',
+  filters: 'filters',
+  groups: 'groups',
+} as const;
+
+export const KAFKA_TOPIC = {
+  event: 'event',
+  eventData: 'event_data',
+} as const;
+
+export const ROLES = {
+  admin: 'admin',
+  user: 'user',
+  viewOnly: 'view-only',
+  teamOwner: 'team-owner',
+  teamMember: 'team-member',
+} as const;
+
+export const PERMISSIONS = {
+  all: 'all',
+  websiteCreate: 'website:create',
+  websiteUpdate: 'website:update',
+  websiteDelete: 'website:delete',
+  teamCreate: 'team:create',
+  teamUpdate: 'team:update',
+  teamDelete: 'team:delete',
+} as const;
+
+export const ROLE_PERMISSIONS = {
+  [ROLES.admin]: [PERMISSIONS.all],
+  [ROLES.user]: [
+    PERMISSIONS.websiteCreate,
+    PERMISSIONS.websiteUpdate,
+    PERMISSIONS.websiteDelete,
+    PERMISSIONS.teamCreate,
+  ],
+  [ROLES.viewOnly]: [],
+  [ROLES.teamOwner]: [PERMISSIONS.teamUpdate, PERMISSIONS.teamDelete],
+  [ROLES.teamMember]: [],
+} as const;
+
+export const THEME_COLORS = {
+  light: {
+    primary: '#2680eb',
+    gray50: '#ffffff',
+    gray75: '#fafafa',
+    gray100: '#f5f5f5',
+    gray200: '#eaeaea',
+    gray300: '#e1e1e1',
+    gray400: '#cacaca',
+    gray500: '#b3b3b3',
+    gray600: '#8e8e8e',
+    gray700: '#6e6e6e',
+    gray800: '#4b4b4b',
+    gray900: '#2c2c2c',
+  },
+  dark: {
+    primary: '#2680eb',
+    gray50: '#252525',
+    gray75: '#2f2f2f',
+    gray100: '#323232',
+    gray200: '#3e3e3e',
+    gray300: '#4a4a4a',
+    gray400: '#5a5a5a',
+    gray500: '#6e6e6e',
+    gray600: '#909090',
+    gray700: '#b9b9b9',
+    gray800: '#e3e3e3',
+    gray900: '#ffffff',
+  },
+};
+
+export const EVENT_COLORS = [
+  '#2680eb',
+  '#9256d9',
+  '#44b556',
+  '#e68619',
+  '#e34850',
+  '#f7bd12',
+  '#01bad7',
+  '#6734bc',
+  '#89c541',
+  '#ffc301',
+  '#ec1562',
+  '#ffec16',
+];
+
+export const DOMAIN_REGEX =
+  /^(localhost(:[1-9]\d{0,4})?|((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9-]+(-[a-z0-9-]+)*\.)+(xn--)?[a-z0-9-]{2,63})$/;
+export const SHARE_ID_REGEX = /^[a-zA-Z0-9]{8,16}$/;
+export const UUID_REGEX =
+  /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/;
+export const HOSTNAME_REGEX =
+  /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/;
+
+export const DESKTOP_SCREEN_WIDTH = 1920;
+export const LAPTOP_SCREEN_WIDTH = 1024;
+export const MOBILE_SCREEN_WIDTH = 479;
+
+export const URL_LENGTH = 500;
+export const EVENT_NAME_LENGTH = 50;
+
+export const DESKTOP_OS = [
+  'BeOS',
+  'Chrome OS',
+  'Linux',
+  'Mac OS',
+  'Open BSD',
+  'OS/2',
+  'QNX',
+  'Sun OS',
+  'Windows 10',
+  'Windows 2000',
+  'Windows 3.11',
+  'Windows 7',
+  'Windows 8',
+  'Windows 8.1',
+  'Windows 95',
+  'Windows 98',
+  'Windows ME',
+  'Windows Server 2003',
+  'Windows Vista',
+  'Windows XP',
+];
+
+export const MOBILE_OS = ['Amazon OS', 'Android OS', 'BlackBerry OS', 'iOS', 'Windows Mobile'];
+
+export const BROWSERS = {
+  android: 'Android',
+  aol: 'AOL',
+  beaker: 'Beaker',
+  bb10: 'BlackBerry 10',
+  chrome: 'Chrome',
+  'chromium-webview': 'Chrome (webview)',
+  crios: 'Chrome (iOS)',
+  curl: 'Curl',
+  edge: 'Edge',
+  'edge-chromium': 'Edge (Chromium)',
+  'edge-ios': 'Edge (iOS)',
+  facebook: 'Facebook',
+  firefox: 'Firefox',
+  fxios: 'Firefox (iOS)',
+  ie: 'IE',
+  instagram: 'Instagram',
+  ios: 'iOS',
+  'ios-webview': 'iOS (webview)',
+  kakaotalk: 'KaKaoTalk',
+  miui: 'MIUI',
+  opera: 'Opera',
+  'opera-mini': 'Opera Mini',
+  phantomjs: 'PhantomJS',
+  safari: 'Safari',
+  samsung: 'Samsung',
+  silk: 'Silk',
+  searchbot: 'Searchbot',
+  yandexbrowser: 'Yandex',
+};
+
+export const MAP_FILE = '/datamaps.world.json';
+
+export const ISO_COUNTRIES = {
+  AFG: 'AF',
+  ALA: 'AX',
+  ALB: 'AL',
+  DZA: 'DZ',
+  ASM: 'AS',
+  AND: 'AD',
+  AGO: 'AO',
+  AIA: 'AI',
+  ATA: 'AQ',
+  ATG: 'AG',
+  ARG: 'AR',
+  ARM: 'AM',
+  ABW: 'AW',
+  AUS: 'AU',
+  AUT: 'AT',
+  AZE: 'AZ',
+  BHS: 'BS',
+  BHR: 'BH',
+  BGD: 'BD',
+  BRB: 'BB',
+  BLR: 'BY',
+  BEL: 'BE',
+  BLZ: 'BZ',
+  BEN: 'BJ',
+  BMU: 'BM',
+  BTN: 'BT',
+  BOL: 'BO',
+  BIH: 'BA',
+  BWA: 'BW',
+  BVT: 'BV',
+  BRA: 'BR',
+  VGB: 'VG',
+  IOT: 'IO',
+  BRN: 'BN',
+  BGR: 'BG',
+  BFA: 'BF',
+  BDI: 'BI',
+  KHM: 'KH',
+  CMR: 'CM',
+  CAN: 'CA',
+  CPV: 'CV',
+  CYM: 'KY',
+  CAF: 'CF',
+  TCD: 'TD',
+  CHL: 'CL',
+  CHN: 'CN',
+  HKG: 'HK',
+  MAC: 'MO',
+  CXR: 'CX',
+  CCK: 'CC',
+  COL: 'CO',
+  COM: 'KM',
+  COG: 'CG',
+  COD: 'CD',
+  COK: 'CK',
+  CRI: 'CR',
+  CIV: 'CI',
+  HRV: 'HR',
+  CUB: 'CU',
+  CYP: 'CY',
+  CZE: 'CZ',
+  DNK: 'DK',
+  DJI: 'DJ',
+  DMA: 'DM',
+  DOM: 'DO',
+  ECU: 'EC',
+  EGY: 'EG',
+  SLV: 'SV',
+  GNQ: 'GQ',
+  ERI: 'ER',
+  EST: 'EE',
+  ETH: 'ET',
+  FLK: 'FK',
+  FRO: 'FO',
+  FJI: 'FJ',
+  FIN: 'FI',
+  FRA: 'FR',
+  GUF: 'GF',
+  PYF: 'PF',
+  ATF: 'TF',
+  GAB: 'GA',
+  GMB: 'GM',
+  GEO: 'GE',
+  DEU: 'DE',
+  GHA: 'GH',
+  GIB: 'GI',
+  GRC: 'GR',
+  GRL: 'GL',
+  GRD: 'GD',
+  GLP: 'GP',
+  GUM: 'GU',
+  GTM: 'GT',
+  GGY: 'GG',
+  GIN: 'GN',
+  GNB: 'GW',
+  GUY: 'GY',
+  HTI: 'HT',
+  HMD: 'HM',
+  VAT: 'VA',
+  HND: 'HN',
+  HUN: 'HU',
+  ISL: 'IS',
+  IND: 'IN',
+  IDN: 'ID',
+  IRN: 'IR',
+  IRQ: 'IQ',
+  IRL: 'IE',
+  IMN: 'IM',
+  ISR: 'IL',
+  ITA: 'IT',
+  JAM: 'JM',
+  JPN: 'JP',
+  JEY: 'JE',
+  JOR: 'JO',
+  KAZ: 'KZ',
+  KEN: 'KE',
+  KIR: 'KI',
+  PRK: 'KP',
+  KOR: 'KR',
+  KWT: 'KW',
+  KGZ: 'KG',
+  LAO: 'LA',
+  LVA: 'LV',
+  LBN: 'LB',
+  LSO: 'LS',
+  LBR: 'LR',
+  LBY: 'LY',
+  LIE: 'LI',
+  LTU: 'LT',
+  LUX: 'LU',
+  MKD: 'MK',
+  MDG: 'MG',
+  MWI: 'MW',
+  MYS: 'MY',
+  MDV: 'MV',
+  MLI: 'ML',
+  MLT: 'MT',
+  MHL: 'MH',
+  MTQ: 'MQ',
+  MRT: 'MR',
+  MUS: 'MU',
+  MYT: 'YT',
+  MEX: 'MX',
+  FSM: 'FM',
+  MDA: 'MD',
+  MCO: 'MC',
+  MNG: 'MN',
+  MNE: 'ME',
+  MSR: 'MS',
+  MAR: 'MA',
+  MOZ: 'MZ',
+  MMR: 'MM',
+  NAM: 'NA',
+  NRU: 'NR',
+  NPL: 'NP',
+  NLD: 'NL',
+  ANT: 'AN',
+  NCL: 'NC',
+  NZL: 'NZ',
+  NIC: 'NI',
+  NER: 'NE',
+  NGA: 'NG',
+  NIU: 'NU',
+  NFK: 'NF',
+  MNP: 'MP',
+  NOR: 'NO',
+  OMN: 'OM',
+  PAK: 'PK',
+  PLW: 'PW',
+  PSE: 'PS',
+  PAN: 'PA',
+  PNG: 'PG',
+  PRY: 'PY',
+  PER: 'PE',
+  PHL: 'PH',
+  PCN: 'PN',
+  POL: 'PL',
+  PRT: 'PT',
+  PRI: 'PR',
+  QAT: 'QA',
+  REU: 'RE',
+  ROU: 'RO',
+  RUS: 'RU',
+  RWA: 'RW',
+  BLM: 'BL',
+  SHN: 'SH',
+  KNA: 'KN',
+  LCA: 'LC',
+  MAF: 'MF',
+  SPM: 'PM',
+  VCT: 'VC',
+  WSM: 'WS',
+  SMR: 'SM',
+  STP: 'ST',
+  SAU: 'SA',
+  SEN: 'SN',
+  SRB: 'RS',
+  SYC: 'SC',
+  SLE: 'SL',
+  SGP: 'SG',
+  SVK: 'SK',
+  SVN: 'SI',
+  SLB: 'SB',
+  SOM: 'SO',
+  ZAF: 'ZA',
+  SGS: 'GS',
+  SSD: 'SS',
+  ESP: 'ES',
+  LKA: 'LK',
+  SDN: 'SD',
+  SUR: 'SR',
+  SJM: 'SJ',
+  SWZ: 'SZ',
+  SWE: 'SE',
+  CHE: 'CH',
+  SYR: 'SY',
+  TWN: 'TW',
+  TJK: 'TJ',
+  TZA: 'TZ',
+  THA: 'TH',
+  TLS: 'TL',
+  TGO: 'TG',
+  TKL: 'TK',
+  TON: 'TO',
+  TTO: 'TT',
+  TUN: 'TN',
+  TUR: 'TR',
+  TKM: 'TM',
+  TCA: 'TC',
+  TUV: 'TV',
+  UGA: 'UG',
+  UKR: 'UA',
+  ARE: 'AE',
+  GBR: 'GB',
+  USA: 'US',
+  UMI: 'UM',
+  URY: 'UY',
+  UZB: 'UZ',
+  VUT: 'VU',
+  VEN: 'VE',
+  VNM: 'VN',
+  VIR: 'VI',
+  WLF: 'WF',
+  ESH: 'EH',
+  YEM: 'YE',
+  ZMB: 'ZM',
+  ZWE: 'ZW',
+  XKX: 'XK',
+};

src/lib/crypto.ts

@@ -0,0 +1,23 @@
+import { startOfMonth } from 'date-fns';
+import { hash } from 'next-basics';
+import { v4, v5, validate } from 'uuid';
+
+export function secret() {
+  return hash(process.env.APP_SECRET || process.env.DATABASE_URL);
+}
+
+export function salt() {
+  const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString());
+
+  return hash(secret(), ROTATING_SALT);
+}
+
+export function uuid(...args: any) {
+  if (!args.length) return v4();
+
+  return v5(hash(...args, salt()), v5.DNS);
+}
+
+export function isUuid(value: string) {
+  return validate(value);
+}

src/lib/data.ts

@@ -0,0 +1,75 @@
+import { isValid, parseISO } from 'date-fns';
+import { DATA_TYPE } from './constants';
+import { DynamicDataType } from './types';
+
+export function flattenJSON(
+  eventData: { [key: string]: any },
+  keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],
+  parentKey = '',
+): { key: string; value: any; dynamicDataType: DynamicDataType }[] {
+  return Object.keys(eventData).reduce(
+    (acc, key) => {
+      const value = eventData[key];
+      const type = typeof eventData[key];
+
+      // nested object
+      if (value && type === 'object' && !Array.isArray(value) && !isValid(value)) {
+        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));
+      } else {
+        createKey(getKeyName(key, parentKey), value, acc);
+      }
+
+      return acc;
+    },
+    { keyValues, parentKey },
+  ).keyValues;
+}
+
+export function getDataType(value: any): string {
+  let type: string = typeof value;
+
+  if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
+    type = 'date';
+  }
+
+  return type;
+}
+
+function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
+  const type = getDataType(value);
+
+  let dynamicDataType = null;
+
+  switch (type) {
+    case 'number':
+      dynamicDataType = DATA_TYPE.number;
+      break;
+    case 'string':
+      dynamicDataType = DATA_TYPE.string;
+      break;
+    case 'boolean':
+      dynamicDataType = DATA_TYPE.boolean;
+      value = value ? 'true' : 'false';
+      break;
+    case 'date':
+      dynamicDataType = DATA_TYPE.date;
+      break;
+    case 'object':
+      dynamicDataType = DATA_TYPE.array;
+      value = JSON.stringify(value);
+      break;
+    default:
+      dynamicDataType = DATA_TYPE.string;
+      break;
+  }
+
+  acc.keyValues.push({ key, value, dynamicDataType });
+}
+
+function getKeyName(key, parentKey) {
+  if (!parentKey) {
+    return key;
+  }
+
+  return `${parentKey}.${key}`;
+}

src/lib/date.ts

@@ -0,0 +1,333 @@
+import moment from 'moment-timezone';
+import {
+  addMinutes,
+  addHours,
+  addDays,
+  addMonths,
+  addYears,
+  subHours,
+  subDays,
+  subMonths,
+  subYears,
+  startOfMinute,
+  startOfHour,
+  startOfDay,
+  startOfWeek,
+  startOfMonth,
+  startOfYear,
+  endOfHour,
+  endOfDay,
+  endOfWeek,
+  endOfMonth,
+  endOfYear,
+  differenceInMinutes,
+  differenceInHours,
+  differenceInCalendarDays,
+  differenceInCalendarMonths,
+  differenceInCalendarYears,
+  format,
+  max,
+  min,
+  isDate,
+  subWeeks,
+} from 'date-fns';
+import { getDateLocale } from 'lib/lang';
+import { DateRange } from 'lib/types';
+
+export const TIME_UNIT = {
+  minute: 'minute',
+  hour: 'hour',
+  day: 'day',
+  week: 'week',
+  month: 'month',
+  year: 'year',
+};
+
+const dateFuncs = {
+  minute: [differenceInMinutes, addMinutes, startOfMinute],
+  hour: [differenceInHours, addHours, startOfHour],
+  day: [differenceInCalendarDays, addDays, startOfDay],
+  month: [differenceInCalendarMonths, addMonths, startOfMonth],
+  year: [differenceInCalendarYears, addYears, startOfYear],
+};
+
+export function getTimezone() {
+  return moment.tz.guess();
+}
+
+export function getLocalTime(t: string | number | Date) {
+  return addMinutes(new Date(t), new Date().getTimezoneOffset());
+}
+
+export function parseDateRange(value: string | object, locale = 'en-US'): DateRange {
+  if (typeof value === 'object') {
+    return value as DateRange;
+  }
+
+  if (value === 'all') {
+    return {
+      startDate: new Date(0),
+      endDate: new Date(1),
+      value,
+    };
+  }
+
+  if (value?.startsWith?.('range')) {
+    const [, startTime, endTime] = value.split(':');
+
+    const startDate = new Date(+startTime);
+    const endDate = new Date(+endTime);
+
+    return {
+      startDate,
+      endDate,
+      unit: getMinimumUnit(startDate, endDate),
+      value,
+    };
+  }
+
+  const now = new Date();
+  const dateLocale = getDateLocale(locale);
+
+  const match = value?.match?.(/^(?<num>[0-9-]+)(?<unit>hour|day|week|month|year)$/);
+
+  if (!match) return null;
+
+  const { num, unit } = match.groups;
+  const selectedUnit = { num: +num, unit };
+
+  if (+num === 1) {
+    switch (unit) {
+      case 'day':
+        return {
+          startDate: startOfDay(now),
+          endDate: endOfDay(now),
+          unit: 'hour',
+          value,
+          selectedUnit,
+        };
+      case 'week':
+        return {
+          startDate: startOfWeek(now, { locale: dateLocale }),
+          endDate: endOfWeek(now, { locale: dateLocale }),
+          unit: 'day',
+          value,
+          selectedUnit,
+        };
+      case 'month':
+        return {
+          startDate: startOfMonth(now),
+          endDate: endOfMonth(now),
+          unit: 'day',
+          value,
+          selectedUnit,
+        };
+      case 'year':
+        return {
+          startDate: startOfYear(now),
+          endDate: endOfYear(now),
+          unit: 'month',
+          value,
+          selectedUnit,
+        };
+    }
+  }
+
+  if (+num === -1) {
+    switch (unit) {
+      case 'day':
+        return {
+          startDate: subDays(startOfDay(now), 1),
+          endDate: subDays(endOfDay(now), 1),
+          unit: 'hour',
+          value,
+          selectedUnit,
+        };
+      case 'week':
+        return {
+          startDate: subDays(startOfWeek(now, { locale: dateLocale }), 7),
+          endDate: subDays(endOfWeek(now, { locale: dateLocale }), 1),
+          unit: 'day',
+          value,
+          selectedUnit,
+        };
+      case 'month':
+        return {
+          startDate: subMonths(startOfMonth(now), 1),
+          endDate: subMonths(endOfMonth(now), 1),
+          unit: 'day',
+          value,
+          selectedUnit,
+        };
+      case 'year':
+        return {
+          startDate: subYears(startOfYear(now), 1),
+          endDate: subYears(endOfYear(now), 1),
+          unit: 'month',
+          value,
+          selectedUnit,
+        };
+    }
+  }
+
+  switch (unit) {
+    case 'day':
+      return {
+        startDate: subDays(startOfDay(now), +num - 1),
+        endDate: endOfDay(now),
+        unit,
+        value,
+        selectedUnit,
+      };
+    case 'hour':
+      return {
+        startDate: subHours(startOfHour(now), +num - 1),
+        endDate: endOfHour(now),
+        unit,
+        value,
+        selectedUnit,
+      };
+  }
+}
+
+export function incrementDateRange(
+  value: { startDate: any; endDate: any; selectedUnit: any },
+  increment: number,
+) {
+  const { startDate, endDate, selectedUnit } = value;
+
+  const { num, unit } = selectedUnit;
+
+  const sub = Math.abs(num) * increment;
+
+  switch (unit) {
+    case 'hour':
+      return {
+        ...value,
+        startDate: subHours(startDate, sub),
+        endDate: subHours(endDate, sub),
+        value: 'range',
+      };
+    case 'day':
+      return {
+        ...value,
+        startDate: subDays(startDate, sub),
+        endDate: subDays(endDate, sub),
+        value: 'range',
+      };
+    case 'week':
+      return {
+        ...value,
+        startDate: subWeeks(startDate, sub),
+        endDate: subWeeks(endDate, sub),
+        value: 'range',
+      };
+    case 'month':
+      return {
+        ...value,
+        startDate: subMonths(startDate, sub),
+        endDate: subMonths(endDate, sub),
+        value: 'range',
+      };
+    case 'year':
+      return {
+        ...value,
+        startDate: subYears(startDate, sub),
+        endDate: subYears(endDate, sub),
+        value: 'range',
+      };
+  }
+}
+
+export function getAllowedUnits(startDate: Date, endDate: Date) {
+  const units = ['minute', 'hour', 'day', 'month', 'year'];
+  const minUnit = getMinimumUnit(startDate, endDate);
+  const index = units.indexOf(minUnit === 'year' ? 'month' : minUnit);
+
+  return index >= 0 ? units.splice(index) : [];
+}
+
+export function getMinimumUnit(startDate: number | Date, endDate: number | Date) {
+  if (differenceInMinutes(endDate, startDate) <= 60) {
+    return 'minute';
+  } else if (differenceInHours(endDate, startDate) <= 48) {
+    return 'hour';
+  } else if (differenceInCalendarMonths(endDate, startDate) <= 12) {
+    return 'day';
+  } else if (differenceInCalendarMonths(endDate, startDate) <= 24) {
+    return 'month';
+  }
+
+  return 'year';
+}
+
+export function getDateFromString(str: string) {
+  const [ymd, hms] = str.split(' ');
+  const [year, month, day] = ymd.split('-');
+
+  if (hms) {
+    const [hour, min, sec] = hms.split(':');
+
+    return new Date(+year, +month - 1, +day, +hour, +min, +sec);
+  }
+
+  return new Date(+year, +month - 1, +day);
+}
+
+export function getDateArray(data: any[], startDate: Date, endDate: Date, unit: string) {
+  const arr = [];
+  const [diff, add, normalize] = dateFuncs[unit];
+  const n = diff(endDate, startDate) + 1;
+
+  function findData(date: Date) {
+    const d = data.find(({ x }) => {
+      return normalize(getDateFromString(x)).getTime() === date.getTime();
+    });
+
+    return d?.y || 0;
+  }
+
+  for (let i = 0; i < n; i++) {
+    const t = normalize(add(startDate, i));
+    const y = findData(t);
+
+    arr.push({ x: t, y });
+  }
+
+  return arr;
+}
+
+export function getDateLength(startDate: Date, endDate: Date, unit: string | number) {
+  const [diff] = dateFuncs[unit];
+  return diff(endDate, startDate) + 1;
+}
+
+export const CUSTOM_FORMATS = {
+  'en-US': {
+    p: 'ha',
+    pp: 'h:mm:ss',
+  },
+  'fr-FR': {
+    'M/d': 'd/M',
+    'MMM d': 'd MMM',
+    'EEE M/d': 'EEE d/M',
+  },
+};
+
+export function formatDate(date: string | number | Date, str: string, locale = 'en-US') {
+  return format(
+    typeof date === 'string' ? new Date(date) : date,
+    CUSTOM_FORMATS?.[locale]?.[str] || str,
+    {
+      locale: getDateLocale(locale),
+    },
+  );
+}
+
+export function maxDate(...args: Date[]) {
+  return max(args.filter(n => isDate(n)));
+}
+
+export function minDate(...args: any[]) {
+  return min(args.filter(n => isDate(n)));
+}

src/lib/db.ts

@@ -0,0 +1,45 @@
+export const PRISMA = 'prisma';
+export const POSTGRESQL = 'postgresql';
+export const MYSQL = 'mysql';
+export const CLICKHOUSE = 'clickhouse';
+export const KAFKA = 'kafka';
+export const KAFKA_PRODUCER = 'kafka-producer';
+
+// Fixes issue with converting bigint values
+BigInt.prototype['toJSON'] = function () {
+  return Number(this);
+};
+
+export function getDatabaseType(url = process.env.DATABASE_URL) {
+  const type = url && url.split(':')[0];
+
+  if (type === 'postgres') {
+    return POSTGRESQL;
+  }
+
+  if (process.env.CLICKHOUSE_URL) {
+    return CLICKHOUSE;
+  }
+
+  return type;
+}
+
+export async function runQuery(queries: any) {
+  const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL);
+
+  if (db === POSTGRESQL || db === MYSQL) {
+    return queries[PRISMA]();
+  }
+
+  if (db === CLICKHOUSE) {
+    if (queries[KAFKA]) {
+      return queries[KAFKA]();
+    }
+
+    return queries[CLICKHOUSE]();
+  }
+}
+
+export function notImplemented() {
+  throw new Error('Not implemented.');
+}

src/lib/detect.ts

@@ -0,0 +1,137 @@
+import path from 'path';
+import { getClientIp } from 'request-ip';
+import { browserName, detectOS } from 'detect-browser';
+import isLocalhost from 'is-localhost-ip';
+import maxmind from 'maxmind';
+import { safeDecodeURIComponent } from 'next-basics';
+
+import {
+  DESKTOP_OS,
+  MOBILE_OS,
+  DESKTOP_SCREEN_WIDTH,
+  LAPTOP_SCREEN_WIDTH,
+  MOBILE_SCREEN_WIDTH,
+} from './constants';
+import { NextApiRequestCollect } from 'pages/api/send';
+
+let lookup;
+
+export function getIpAddress(req) {
+  // Custom header
+  if (req.headers[process.env.CLIENT_IP_HEADER]) {
+    return req.headers[process.env.CLIENT_IP_HEADER];
+  }
+  // Cloudflare
+  else if (req.headers['cf-connecting-ip']) {
+    return req.headers['cf-connecting-ip'];
+  }
+
+  return getClientIp(req);
+}
+
+export function getDevice(screen, os) {
+  if (!screen) return;
+
+  const [width] = screen.split('x');
+
+  if (DESKTOP_OS.includes(os)) {
+    if (os === 'Chrome OS' || width < DESKTOP_SCREEN_WIDTH) {
+      return 'laptop';
+    }
+    return 'desktop';
+  } else if (MOBILE_OS.includes(os)) {
+    if (os === 'Amazon OS' || width > MOBILE_SCREEN_WIDTH) {
+      return 'tablet';
+    }
+    return 'mobile';
+  }
+
+  if (width >= DESKTOP_SCREEN_WIDTH) {
+    return 'desktop';
+  } else if (width >= LAPTOP_SCREEN_WIDTH) {
+    return 'laptop';
+  } else if (width >= MOBILE_SCREEN_WIDTH) {
+    return 'tablet';
+  } else {
+    return 'mobile';
+  }
+}
+
+function getRegionCode(country, region) {
+  if (!country || !region) {
+    return undefined;
+  }
+
+  return region.includes('-') ? region : `${country}-${region}`;
+}
+
+export async function getLocation(ip, req) {
+  // Ignore local ips
+  if (await isLocalhost(ip)) {
+    return;
+  }
+
+  // Cloudflare headers
+  if (req.headers['cf-ipcountry']) {
+    const country = safeDecodeURIComponent(req.headers['cf-ipcountry']);
+    const subdivision1 = safeDecodeURIComponent(req.headers['cf-region-code']);
+    const city = safeDecodeURIComponent(req.headers['cf-ipcity']);
+
+    return {
+      country,
+      subdivision1: getRegionCode(country, subdivision1),
+      city,
+    };
+  }
+
+  // Vercel headers
+  if (req.headers['x-vercel-ip-country']) {
+    const country = safeDecodeURIComponent(req.headers['x-vercel-ip-country']);
+    const subdivision1 = safeDecodeURIComponent(req.headers['x-vercel-ip-country-region']);
+    const city = safeDecodeURIComponent(req.headers['x-vercel-ip-city']);
+
+    return {
+      country,
+      subdivision1: getRegionCode(country, subdivision1),
+      city,
+    };
+  }
+
+  // Database lookup
+  if (!lookup) {
+    const dir = path.join(process.cwd(), 'geo');
+
+    lookup = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb'));
+  }
+
+  const result = lookup.get(ip);
+
+  if (result) {
+    const country = result.country?.iso_code ?? result?.registered_country?.iso_code;
+    const subdivision1 = result.subdivisions?.[0]?.iso_code;
+    const subdivision2 = result.subdivisions?.[1]?.names?.en;
+    const city = result.city?.names?.en;
+
+    return {
+      country,
+      subdivision1: getRegionCode(country, subdivision1),
+      subdivision2,
+      city,
+    };
+  }
+}
+
+export async function getClientInfo(req: NextApiRequestCollect, { screen }) {
+  const userAgent = req.headers['user-agent'];
+  const ip = getIpAddress(req);
+  const location = await getLocation(ip, req);
+  const country = location?.country;
+  const subdivision1 = location?.subdivision1;
+  const subdivision2 = location?.subdivision2;
+  const city = location?.city;
+  const browser = browserName(userAgent);
+  const os = detectOS(userAgent);
+  const device = getDevice(screen, os);
+
+  return { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device };
+}

src/lib/filters.ts

@@ -0,0 +1,78 @@
+export const urlFilter = (data: any[]) => {
+  const map = data.reduce((obj, { x, y }) => {
+    if (x) {
+      if (!obj[x]) {
+        obj[x] = y;
+      } else {
+        obj[x] += y;
+      }
+    }
+
+    return obj;
+  }, {});
+
+  return Object.keys(map).map(key => ({ x: key, y: map[key] }));
+};
+
+export const refFilter = (data: any[]) => {
+  const links = {};
+
+  const map = data.reduce((obj, { x, y }) => {
+    let id;
+
+    try {
+      const url = new URL(x);
+
+      id = url.hostname.replace(/www\./, '') || url.href;
+    } catch {
+      id = '';
+    }
+
+    links[id] = x;
+
+    if (!obj[id]) {
+      obj[id] = y;
+    } else {
+      obj[id] += y;
+    }
+
+    return obj;
+  }, {});
+
+  return Object.keys(map).map(key => ({ x: key, y: map[key], w: links[key] }));
+};
+
+export const emptyFilter = (data: any[]) => {
+  return data.map(item => (item.x ? item : null)).filter(n => n);
+};
+
+export const percentFilter = (data: any[]) => {
+  const total = data.reduce((n, { y }) => n + y, 0);
+  return data.map(({ x, y, ...props }) => ({ x, y, z: total ? (y / total) * 100 : 0, ...props }));
+};
+
+export const paramFilter = (data: any[]) => {
+  const map = data.reduce((obj, { x, y }) => {
+    try {
+      const searchParams = new URLSearchParams(x);
+
+      for (const [key, value] of searchParams) {
+        if (!obj[key]) {
+          obj[key] = { [value]: y };
+        } else if (!obj[key][value]) {
+          obj[key][value] = y;
+        } else {
+          obj[key][value] += y;
+        }
+      }
+    } catch {
+      // Ignore
+    }
+
+    return obj;
+  }, {});
+
+  return Object.keys(map).flatMap(key =>
+    Object.keys(map[key]).map(n => ({ x: `${key}=${n}`, p: key, v: n, y: map[key][n] })),
+  );
+};

src/lib/format.ts

@@ -0,0 +1,80 @@
+export function parseTime(val: number) {
+  const days = ~~(val / 86400);
+  const hours = ~~(val / 3600) - days * 24;
+  const minutes = ~~(val / 60) - days * 1440 - hours * 60;
+  const seconds = ~~val - days * 86400 - hours * 3600 - minutes * 60;
+  const ms = (val - ~~val) * 1000;
+
+  return {
+    days,
+    hours,
+    minutes,
+    seconds,
+    ms,
+  };
+}
+
+export function formatTime(val: number) {
+  const { hours, minutes, seconds } = parseTime(val);
+  const h = hours > 0 ? `${hours}:` : '';
+  const m = hours > 0 ? minutes.toString().padStart(2, '0') : minutes;
+  const s = seconds.toString().padStart(2, '0');
+
+  return `${h}${m}:${s}`;
+}
+
+export function formatShortTime(val: number, formats = ['m', 's'], space = '') {
+  const { days, hours, minutes, seconds, ms } = parseTime(val);
+  let t = '';
+
+  if (days > 0 && formats.indexOf('d') !== -1) t += `${days}d${space}`;
+  if (hours > 0 && formats.indexOf('h') !== -1) t += `${hours}h${space}`;
+  if (minutes > 0 && formats.indexOf('m') !== -1) t += `${minutes}m${space}`;
+  if (seconds > 0 && formats.indexOf('s') !== -1) t += `${seconds}s${space}`;
+  if (ms > 0 && formats.indexOf('ms') !== -1) t += `${ms}ms`;
+
+  if (!t) {
+    return `0${formats[formats.length - 1]}`;
+  }
+
+  return t;
+}
+
+export function formatNumber(n: string | number) {
+  return Number(n).toFixed(0);
+}
+
+export function formatLongNumber(value: number) {
+  const n = Number(value);
+
+  if (n >= 1000000) {
+    return `${(n / 1000000).toFixed(1)}m`;
+  }
+  if (n >= 100000) {
+    return `${(n / 1000).toFixed(0)}k`;
+  }
+  if (n >= 10000) {
+    return `${(n / 1000).toFixed(1)}k`;
+  }
+  if (n >= 1000) {
+    return `${(n / 1000).toFixed(2)}k`;
+  }
+
+  return formatNumber(n);
+}
+
+export function stringToColor(str: string) {
+  if (!str) {
+    return '#ffffff';
+  }
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = str.charCodeAt(i) + ((hash << 5) - hash);
+  }
+  let color = '#';
+  for (let i = 0; i < 3; i++) {
+    const value = (hash >> (i * 8)) & 0xff;
+    color += ('00' + value.toString(16)).slice(-2);
+  }
+  return color;
+}

src/lib/kafka.ts

@@ -0,0 +1,118 @@
+import dateFormat from 'dateformat';
+import debug from 'debug';
+import { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs';
+import { KAFKA, KAFKA_PRODUCER } from 'lib/db';
+import * as tls from 'tls';
+
+const log = debug('umami:kafka');
+
+let kafka: Kafka;
+let producer: Producer;
+const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);
+
+function getClient() {
+  const { username, password } = new URL(process.env.KAFKA_URL);
+  const brokers = process.env.KAFKA_BROKER.split(',');
+
+  const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } =
+    username && password
+      ? {
+          ssl: {
+            checkServerIdentity: () => undefined,
+            ca: [process.env.CA_CERT],
+            key: process.env.CLIENT_KEY,
+            cert: process.env.CLIENT_CERT,
+          },
+          sasl: {
+            mechanism: 'plain',
+            username,
+            password,
+          },
+        }
+      : {};
+
+  const client: Kafka = new Kafka({
+    clientId: 'umami',
+    brokers: brokers,
+    connectionTimeout: 3000,
+    logLevel: logLevel.ERROR,
+    ...ssl,
+  });
+
+  if (process.env.NODE_ENV !== 'production') {
+    global[KAFKA] = client;
+  }
+
+  log('Kafka initialized');
+
+  return client;
+}
+
+async function getProducer(): Promise<Producer> {
+  const producer = kafka.producer();
+  await producer.connect();
+
+  if (process.env.NODE_ENV !== 'production') {
+    global[KAFKA_PRODUCER] = producer;
+  }
+
+  log('Kafka producer initialized');
+
+  return producer;
+}
+
+function getDateFormat(date: Date, format?: string): string {
+  return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');
+}
+
+async function sendMessage(
+  message: { [key: string]: string | number },
+  topic: string,
+): Promise<RecordMetadata[]> {
+  await connect();
+
+  return producer.send({
+    topic,
+    messages: [
+      {
+        value: JSON.stringify(message),
+      },
+    ],
+    acks: -1,
+  });
+}
+
+async function sendMessages(messages: { [key: string]: string | number }[], topic: string) {
+  await connect();
+
+  await producer.send({
+    topic,
+    messages: messages.map(a => {
+      return { value: JSON.stringify(a) };
+    }),
+    acks: 1,
+  });
+}
+
+async function connect(): Promise<Kafka> {
+  if (!kafka) {
+    kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient());
+
+    if (kafka) {
+      producer = global[KAFKA_PRODUCER] || (await getProducer());
+    }
+  }
+
+  return kafka;
+}
+
+export default {
+  enabled,
+  client: kafka,
+  producer,
+  log,
+  connect,
+  getDateFormat,
+  sendMessage,
+  sendMessages,
+};

src/lib/lang.ts

@@ -0,0 +1,105 @@
+import {
+  arSA,
+  be,
+  bn,
+  cs,
+  sk,
+  da,
+  de,
+  el,
+  enUS,
+  enGB,
+  es,
+  fi,
+  fr,
+  faIR,
+  he,
+  hi,
+  hr,
+  id,
+  it,
+  ja,
+  km,
+  ko,
+  lt,
+  mn,
+  ms,
+  nb,
+  nl,
+  pl,
+  pt,
+  ptBR,
+  ro,
+  ru,
+  sl,
+  sv,
+  ta,
+  th,
+  tr,
+  uk,
+  zhCN,
+  zhTW,
+  ca,
+  hu,
+  vi,
+} from 'date-fns/locale';
+
+export const languages = {
+  'ar-SA': { label: 'العربية', dateLocale: arSA, dir: 'rtl' },
+  'be-BY': { label: 'Беларуская', dateLocale: be },
+  'bn-BD': { label: 'বাংলা', dateLocale: bn },
+  'ca-ES': { label: 'Català', dateLocale: ca },
+  'cs-CZ': { label: 'Čeština', dateLocale: cs },
+  'da-DK': { label: 'Dansk', dateLocale: da },
+  'de-CH': { label: 'Schwiizerdütsch', dateLocale: de },
+  'de-DE': { label: 'Deutsch', dateLocale: de },
+  'el-GR': { label: 'Ελληνικά', dateLocale: el },
+  'en-GB': { label: 'English (UK)', dateLocale: enGB },
+  'en-US': { label: 'English (US)', dateLocale: enUS },
+  'es-MX': { label: 'Español', dateLocale: es },
+  'fa-IR': { label: 'فارسی', dateLocale: faIR, dir: 'rtl' },
+  'fi-FI': { label: 'Suomi', dateLocale: fi },
+  'fo-FO': { label: 'Føroyskt' },
+  'fr-FR': { label: 'Français', dateLocale: fr },
+  'ga-ES': { label: 'Galacian (Spain)', dateLocale: es },
+  'he-IL': { label: 'עברית', dateLocale: he },
+  'hi-IN': { label: 'हिन्दी', dateLocale: hi },
+  'hr-HR': { label: 'Hrvatski', dateLocale: hr },
+  'hu-HU': { label: 'Hungarian', dateLocale: hu },
+  'id-ID': { label: 'Bahasa Indonesia', dateLocale: id },
+  'it-IT': { label: 'Italiano', dateLocale: it },
+  'ja-JP': { label: '日本語', dateLocale: ja },
+  'km-KH': { label: 'ភាសាខ្មែរ', dateLocale: km },
+  'ko-KR': { label: '한국어', dateLocale: ko },
+  'lt-LT': { label: 'Lietuvių', dateLocale: lt },
+  'mn-MN': { label: 'Монгол', dateLocale: mn },
+  'ms-MY': { label: 'Malay', dateLocale: ms },
+  'my-MM': { label: 'မြန်မာဘာသာ', dateLocale: enUS },
+  'nl-NL': { label: 'Nederlands', dateLocale: nl },
+  'nb-NO': { label: 'Norsk Bokmål', dateLocale: nb },
+  'pl-PL': { label: 'Polski', dateLocale: pl },
+  'pt-BR': { label: 'Português do Brasil', dateLocale: ptBR },
+  'pt-PT': { label: 'Português', dateLocale: pt },
+  'ro-RO': { label: 'Română', dateLocale: ro },
+  'ru-RU': { label: 'Русский', dateLocale: ru },
+  'si-LK': { label: 'සිංහල', dateLocale: id },
+  'sk-SK': { label: 'Slovenčina', dateLocale: sk },
+  'sl-SI': { label: 'Slovenščina', dateLocale: sl },
+  'sv-SE': { label: 'Svenska', dateLocale: sv },
+  'ta-IN': { label: 'தமிழ்', dateLocale: ta },
+  'th-TH': { label: 'ภาษาไทย', dateLocale: th },
+  'tr-TR': { label: 'Türkçe', dateLocale: tr },
+  'uk-UA': { label: 'українська', dateLocale: uk },
+  'ur-PK': { label: 'Urdu (Pakistan)', dateLocale: uk, dir: 'rtl' },
+  'vi-VN': { label: 'Tiếng Việt', dateLocale: vi },
+  'zh-CN': { label: '中文', dateLocale: zhCN },
+  'zh-TW': { label: '中文(繁體)', dateLocale: zhTW },
+};
+
+export function getDateLocale(locale: string) {
+  return languages[locale]?.dateLocale || enUS;
+}
+
+export function getTextDirection(locale: string) {
+  return languages[locale]?.dir || 'ltr';
+}

src/lib/load.ts

@@ -0,0 +1,51 @@
+import cache from 'lib/cache';
+import { getSession, getUserById, getWebsiteById } from 'queries';
+import { User, Website, Session } from '@prisma/client';
+
+export async function loadWebsite(websiteId: string): Promise<Website> {
+  let website;
+
+  if (cache.enabled) {
+    website = await cache.fetchWebsite(websiteId);
+  } else {
+    website = await getWebsiteById(websiteId);
+  }
+
+  if (!website || website.deletedAt) {
+    return null;
+  }
+
+  return website;
+}
+
+export async function loadSession(sessionId: string): Promise<Session> {
+  let session;
+
+  if (cache.enabled) {
+    session = await cache.fetchSession(sessionId);
+  } else {
+    session = await getSession(sessionId);
+  }
+
+  if (!session) {
+    return null;
+  }
+
+  return session;
+}
+
+export async function loadUser(userId: string): Promise<User> {
+  let user;
+
+  if (cache.enabled) {
+    user = await cache.fetchUser(userId);
+  } else {
+    user = await getUserById(userId);
+  }
+
+  if (!user || user.deletedAt) {
+    return null;
+  }
+
+  return user;
+}

src/lib/middleware.ts

@@ -0,0 +1,107 @@
+import cors from 'cors';
+import debug from 'debug';
+import redis from '@umami/redis-client';
+import { getAuthToken, parseShareToken } from 'lib/auth';
+import { ROLES } from 'lib/constants';
+import { secret } from 'lib/crypto';
+import { findSession } from 'lib/session';
+import {
+  badRequest,
+  createMiddleware,
+  forbidden,
+  parseSecureToken,
+  tooManyRequest,
+  unauthorized,
+} from 'next-basics';
+import { NextApiRequestCollect } from 'pages/api/send';
+import { getUserById } from '../queries';
+
+const log = debug('umami:middleware');
+
+export const useCors = createMiddleware(
+  cors({
+    // Cache CORS preflight request 24 hours by default
+    maxAge: Number(process.env.CORS_MAX_AGE) || 86400,
+  }),
+);
+
+export const useSession = createMiddleware(async (req, res, next) => {
+  try {
+    const session = await findSession(req as NextApiRequestCollect);
+
+    if (!session) {
+      log('useSession: Session not found');
+      return badRequest(res, 'Session not found.');
+    }
+
+    (req as any).session = session;
+  } catch (e: any) {
+    if (e.message === 'Usage Limit.') {
+      return tooManyRequest(res, e.message);
+    }
+    if (e.message.startsWith('Website not found:')) {
+      return forbidden(res, e.message);
+    }
+    return badRequest(res, e.message);
+  }
+
+  next();
+});
+
+export const useAuth = createMiddleware(async (req, res, next) => {
+  const token = getAuthToken(req);
+  const payload = parseSecureToken(token, secret());
+  const shareToken = await parseShareToken(req as any);
+
+  let user = null;
+  const { userId, authKey, grant } = payload || {};
+
+  if (userId) {
+    user = await getUserById(userId);
+  } else if (redis.enabled && authKey) {
+    const key = await redis.client.get(authKey);
+
+    if (key?.userId) {
+      user = await getUserById(key.userId);
+    }
+  }
+
+  if (process.env.NODE_ENV === 'development') {
+    log('useAuth:', { token, shareToken, payload, user, grant });
+  }
+
+  if (!user?.id && !shareToken) {
+    log('useAuth: User not authorized');
+    return unauthorized(res);
+  }
+
+  if (user) {
+    user.isAdmin = user.role === ROLES.admin;
+  }
+
+  (req as any).auth = {
+    user,
+    grant,
+    token,
+    shareToken,
+    authKey,
+  };
+
+  next();
+});
+
+export const useValidate = async (schema, req, res) => {
+  return createMiddleware(async (req: any, res, next) => {
+    try {
+      const rules = schema[req.method];
+
+      if (rules) {
+        rules.validateSync({ ...req.query, ...req.body });
+      }
+    } catch (e: any) {
+      return badRequest(res, e.message);
+    }
+
+    next();
+  })(req, res);
+};

src/lib/prisma.ts

@@ -0,0 +1,233 @@
+import { Prisma } from '@prisma/client';
+import prisma from '@umami/prisma-client';
+import moment from 'moment-timezone';
+import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
+import { FILTER_COLUMNS, SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants';
+import { loadWebsite } from './load';
+import { maxDate } from './date';
+import { QueryFilters, QueryOptions, SearchFilter } from './types';
+
+const MYSQL_DATE_FORMATS = {
+  minute: '%Y-%m-%d %H:%i:00',
+  hour: '%Y-%m-%d %H:00:00',
+  day: '%Y-%m-%d',
+  month: '%Y-%m-01',
+  year: '%Y-01-01',
+};
+
+const POSTGRESQL_DATE_FORMATS = {
+  minute: 'YYYY-MM-DD HH24:MI:00',
+  hour: 'YYYY-MM-DD HH24:00:00',
+  day: 'YYYY-MM-DD',
+  month: 'YYYY-MM-01',
+  year: 'YYYY-01-01',
+};
+
+function getAddIntervalQuery(field: string, interval: string): string {
+  const db = getDatabaseType(process.env.DATABASE_URL);
+
+  if (db === POSTGRESQL) {
+    return `${field} + interval '${interval}'`;
+  }
+
+  if (db === MYSQL) {
+    return `DATE_ADD(${field}, interval ${interval})`;
+  }
+}
+
+function getDayDiffQuery(field1: string, field2: string): string {
+  const db = getDatabaseType(process.env.DATABASE_URL);
+
+  if (db === POSTGRESQL) {
+    return `${field1}::date - ${field2}::date`;
+  }
+
+  if (db === MYSQL) {
+    return `DATEDIFF(${field1}, ${field2})`;
+  }
+}
+
+function getCastColumnQuery(field: string, type: string): string {
+  const db = getDatabaseType(process.env.DATABASE_URL);
+
+  if (db === POSTGRESQL) {
+    return `${field}::${type}`;
+  }
+
+  if (db === MYSQL) {
+    return `${field}`;
+  }
+}
+
+function getDateQuery(field: string, unit: string, timezone?: string): string {
+  const db = getDatabaseType();
+
+  if (db === POSTGRESQL) {
+    if (timezone) {
+      return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
+    }
+    return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
+  }
+
+  if (db === MYSQL) {
+    if (timezone) {
+      const tz = moment.tz(timezone).format('Z');
+
+      return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
+    }
+
+    return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
+  }
+}
+
+function getTimestampDiffQuery(field1: string, field2: string): string {
+  const db = getDatabaseType();
+
+  if (db === POSTGRESQL) {
+    return `floor(extract(epoch from (${field2} - ${field1})))`;
+  }
+
+  if (db === MYSQL) {
+    return `timestampdiff(second, ${field1}, ${field2})`;
+  }
+}
+
+function mapFilter(column, operator, name, type = 'varchar') {
+  switch (operator) {
+    case OPERATORS.equals:
+      return `${column} = {{${name}::${type}}}`;
+    case OPERATORS.notEquals:
+      return `${column} != {{${name}::${type}}}`;
+    default:
+      return '';
+  }
+}
+
+function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {
+  const query = Object.keys(filters).reduce((arr, name) => {
+    const value = filters[name];
+    const operator = value?.filter ?? OPERATORS.equals;
+    const column = FILTER_COLUMNS[name] ?? options?.columns?.[name];
+
+    if (value !== undefined && column) {
+      arr.push(`and ${mapFilter(column, operator, name)}`);
+
+      if (name === 'referrer') {
+        arr.push(
+          'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',
+        );
+      }
+    }
+
+    return arr;
+  }, []);
+
+  return query.join('\n');
+}
+
+function normalizeFilters(filters = {}) {
+  return Object.keys(filters).reduce((obj, key) => {
+    const value = filters[key];
+
+    obj[key] = value?.value ?? value;
+
+    return obj;
+  }, {});
+}
+
+async function parseFilters(
+  websiteId: string,
+  filters: QueryFilters = {},
+  options: QueryOptions = {},
+) {
+  const website = await loadWebsite(websiteId);
+
+  return {
+    joinSession:
+      options?.joinSession || Object.keys(filters).find(key => SESSION_COLUMNS.includes(key))
+        ? `inner join session on website_event.session_id = session.session_id`
+        : '',
+    filterQuery: getFilterQuery(filters, options),
+    params: {
+      ...normalizeFilters(filters),
+      websiteId,
+      startDate: maxDate(filters.startDate, website.resetAt),
+      websiteDomain: website.domain,
+    },
+  };
+}
+
+async function rawQuery(sql: string, data: object): Promise<any> {
+  const db = getDatabaseType();
+  const params = [];
+
+  if (db !== POSTGRESQL && db !== MYSQL) {
+    return Promise.reject(new Error('Unknown database.'));
+  }
+
+  const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
+    const [, name, type] = args;
+    params.push(data[name]);
+
+    return db === MYSQL ? '?' : `$${params.length}${type ?? ''}`;
+  });
+
+  return prisma.rawQuery(query, params);
+}
+
+function getPageFilters(filters: SearchFilter): [
+  {
+    orderBy: {
+      [x: string]: string;
+    }[];
+    take: number;
+    skip: number;
+  },
+  {
+    pageSize: number;
+    page: number;
+    orderBy: string;
+  },
+] {
+  const { page = 1, pageSize = DEFAULT_PAGE_SIZE, orderBy, sortDescending = false } = filters || {};
+
+  return [
+    {
+      ...(pageSize > 0 && { take: +pageSize, skip: +pageSize * (page - 1) }),
+      ...(orderBy && {
+        orderBy: [
+          {
+            [orderBy]: sortDescending ? 'desc' : 'asc',
+          },
+        ],
+      }),
+    },
+    { page: +page, pageSize, orderBy },
+  ];
+}
+
+function getSearchMode(): { mode?: Prisma.QueryMode } {
+  const db = getDatabaseType();
+
+  if (db === POSTGRESQL) {
+    return {
+      mode: 'insensitive',
+    };
+  }
+
+  return {};
+}
+
+export default {
+  ...prisma,
+  getAddIntervalQuery,
+  getDayDiffQuery,
+  getCastColumnQuery,
+  getDateQuery,
+  getTimestampDiffQuery,
+  getFilterQuery,
+  parseFilters,
+  getPageFilters,
+  getSearchMode,
+  rawQuery,
+};

src/lib/query.ts

@@ -0,0 +1,31 @@
+import { NextApiRequest } from 'next';
+import { getAllowedUnits, getMinimumUnit } from './date';
+import { getWebsiteDateRange } from '../queries';
+
+export async function parseDateRangeQuery(req: NextApiRequest) {
+  const { id: websiteId, startAt, endAt, unit } = req.query;
+
+  // All-time
+  if (+startAt === 0 && +endAt === 1) {
+    const result = await getWebsiteDateRange(websiteId as string);
+    const { min, max } = result[0];
+    const startDate = new Date(min);
+    const endDate = new Date(max);
+
+    return {
+      startDate,
+      endDate,
+      unit: getMinimumUnit(startDate, endDate),
+    };
+  }
+
+  const startDate = new Date(+startAt);
+  const endDate = new Date(+endAt);
+  const minUnit = getMinimumUnit(startDate, endDate);
+
+  return {
+    startDate,
+    endDate,
+    unit: (getAllowedUnits(startDate, endDate).includes(unit as string) ? unit : minUnit) as string,
+  };
+}

src/lib/schema.ts

@@ -0,0 +1,13 @@
+import * as yup from 'yup';
+
+export const dateRange = {
+  startAt: yup.number().integer().required(),
+  endAt: yup.number().integer().moreThan(yup.ref('startAt')).required(),
+};
+
+export const pageInfo = {
+  query: yup.string(),
+  page: yup.number().integer().positive(),
+  pageSize: yup.number().integer().positive().min(1).max(200),
+  orderBy: yup.string(),
+};

src/lib/session.ts

@@ -0,0 +1,126 @@
+import { isUuid, secret, uuid } from 'lib/crypto';
+import { getClientInfo } from 'lib/detect';
+import { parseToken } from 'next-basics';
+import { NextApiRequestCollect } from 'pages/api/send';
+import { createSession } from 'queries';
+import cache from './cache';
+import clickhouse from './clickhouse';
+import { loadSession, loadWebsite } from './load';
+
+export async function findSession(req: NextApiRequestCollect): Promise<{
+  id: any;
+  websiteId: string;
+  hostname: string;
+  browser: string;
+  os: any;
+  device: string;
+  screen: string;
+  language: string;
+  country: any;
+  subdivision1: any;
+  subdivision2: any;
+  city: any;
+  ownerId: string;
+}> {
+  const { payload } = req.body;
+
+  if (!payload) {
+    throw new Error('Invalid payload.');
+  }
+
+  // Check if cache token is passed
+  const cacheToken = req.headers['x-umami-cache'];
+
+  if (cacheToken) {
+    const result = await parseToken(cacheToken, secret());
+
+    if (result) {
+      await checkUserBlock(result?.ownerId);
+
+      return result;
+    }
+  }
+
+  // Verify payload
+  const { website: websiteId, hostname, screen, language } = payload;
+
+  // Check the hostname value for legality to eliminate dirty data
+  const validHostnameRegex = /^[\w-.]+$/;
+  if (!validHostnameRegex.test(hostname)) {
+    throw new Error('Invalid hostname.');
+  }
+
+  if (!isUuid(websiteId)) {
+    throw new Error('Invalid website ID.');
+  }
+
+  // Find website
+  const website = await loadWebsite(websiteId);
+
+  if (!website) {
+    throw new Error(`Website not found: ${websiteId}.`);
+  }
+
+  await checkUserBlock(website.userId);
+
+  const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
+    await getClientInfo(req, payload);
+
+  const sessionId = uuid(websiteId, hostname, ip, userAgent);
+
+  // Clickhouse does not require session lookup
+  if (clickhouse.enabled) {
+    return {
+      id: sessionId,
+      websiteId,
+      hostname,
+      browser,
+      os: os as any,
+      device,
+      screen,
+      language,
+      country,
+      subdivision1,
+      subdivision2,
+      city,
+      ownerId: website.userId,
+    };
+  }
+
+  // Find session
+  let session = await loadSession(sessionId);
+
+  // Create a session if not found
+  if (!session) {
+    try {
+      session = await createSession({
+        id: sessionId,
+        websiteId,
+        hostname,
+        browser,
+        os,
+        device,
+        screen,
+        language,
+        country,
+        subdivision1,
+        subdivision2,
+        city,
+      });
+    } catch (e: any) {
+      if (!e.message.toLowerCase().includes('unique constraint')) {
+        throw e;
+      }
+    }
+  }
+
+  return { ...session, ownerId: website.userId };
+}
+
+async function checkUserBlock(userId: string) {
+  if (process.env.ENABLE_BLOCKER && (await cache.fetchUserBlock(userId))) {
+    await cache.incrementUserBlock(userId);
+
+    throw new Error('Usage Limit.');
+  }
+}

src/lib/types.ts

@@ -0,0 +1,219 @@
+import { NextApiRequest } from 'next';
+import {
+  COLLECTION_TYPE,
+  DATA_TYPE,
+  EVENT_TYPE,
+  KAFKA_TOPIC,
+  PERMISSIONS,
+  REPORT_TYPES,
+  ROLES,
+} from './constants';
+import * as yup from 'yup';
+import { TIME_UNIT } from './date';
+
+type ObjectValues<T> = T[keyof T];
+
+export type TimeUnit = ObjectValues<typeof TIME_UNIT>;
+export type Permission = ObjectValues<typeof PERMISSIONS>;
+
+export type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;
+export type Role = ObjectValues<typeof ROLES>;
+export type EventType = ObjectValues<typeof EVENT_TYPE>;
+export type DynamicDataType = ObjectValues<typeof DATA_TYPE>;
+export type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;
+export type ReportType = ObjectValues<typeof REPORT_TYPES>;
+
+export interface WebsiteSearchFilter extends SearchFilter {
+  userId?: string;
+  teamId?: string;
+  includeTeams?: boolean;
+  onlyTeams?: boolean;
+}
+
+export interface UserSearchFilter extends SearchFilter {
+  teamId?: string;
+}
+
+export interface TeamSearchFilter extends SearchFilter {
+  userId?: string;
+}
+
+export interface ReportSearchFilter extends SearchFilter {
+  userId?: string;
+  websiteId?: string;
+  includeTeams?: boolean;
+}
+
+export interface SearchFilter {
+  query?: string;
+  page?: number;
+  pageSize?: number;
+  orderBy?: string;
+  sortDescending?: boolean;
+}
+
+export interface FilterResult<T> {
+  data: T[];
+  count: number;
+  page: number;
+  pageSize: number;
+  orderBy?: string;
+  sortDescending?: boolean;
+}
+
+export interface DynamicData {
+  [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];
+}
+
+export interface Auth {
+  user?: {
+    id: string;
+    username: string;
+    role: string;
+    isAdmin: boolean;
+  };
+  grant?: Permission[];
+  shareToken?: {
+    websiteId: string;
+  };
+}
+
+export interface YupRequest {
+  GET?: yup.ObjectSchema<any>;
+  POST?: yup.ObjectSchema<any>;
+  PUT?: yup.ObjectSchema<any>;
+  DELETE?: yup.ObjectSchema<any>;
+}
+
+export interface NextApiRequestQueryBody<TQuery = any, TBody = any> extends NextApiRequest {
+  auth?: Auth;
+  query: TQuery & { [key: string]: string | string[] };
+  body: TBody;
+  headers: any;
+  yup: YupRequest;
+}
+
+export interface NextApiRequestAuth extends NextApiRequest {
+  auth?: Auth;
+  headers: any;
+}
+
+export interface User {
+  id: string;
+  username: string;
+  password?: string;
+  role: string;
+  createdAt?: Date;
+}
+
+export interface Website {
+  id: string;
+  userId: string;
+  resetAt: Date;
+  name: string;
+  domain: string;
+  shareId: string;
+  createdAt: Date;
+}
+
+export interface Share {
+  id: string;
+  token: string;
+}
+
+export interface WebsiteActive {
+  x: number;
+}
+
+export interface WebsiteMetric {
+  x: string;
+  y: number;
+}
+
+export interface WebsiteEventMetric {
+  x: string;
+  t: string;
+  y: number;
+}
+
+export interface WebsiteEventData {
+  eventName?: string;
+  fieldName: string;
+  dataType: number;
+  fieldValue?: string;
+  total: number;
+}
+
+export interface WebsitePageviews {
+  pageviews: {
+    t: string;
+    y: number;
+  };
+  sessions: {
+    t: string;
+    y: number;
+  };
+}
+
+export interface WebsiteStats {
+  pageviews: { value: number; change: number };
+  uniques: { value: number; change: number };
+  bounces: { value: number; change: number };
+  totalTime: { value: number; change: number };
+}
+
+export interface RealtimeInit {
+  websites: Website[];
+  token: string;
+  data: RealtimeUpdate;
+}
+
+export interface RealtimeUpdate {
+  pageviews: any[];
+  sessions: any[];
+  events: any[];
+  timestamp: number;
+}
+
+export interface DateRange {
+  startDate: Date;
+  endDate: Date;
+  value: string;
+  unit?: TimeUnit;
+  selectedUnit?: { num: number; unit: TimeUnit };
+}
+
+export interface QueryFilters {
+  startDate?: Date;
+  endDate?: Date;
+  timezone?: string;
+  unit?: string;
+  eventType?: number;
+  url?: string;
+  referrer?: string;
+  title?: string;
+  query?: string;
+  os?: string;
+  browser?: string;
+  device?: string;
+  country?: string;
+  region?: string;
+  city?: string;
+  language?: string;
+  event?: string;
+}
+
+export interface QueryOptions {
+  joinSession?: boolean;
+  columns?: { [key: string]: string };
+  limit?: number;
+}
+
+export interface RealtimeData {
+  pageviews: any[];
+  sessions: any[];
+  events: any[];
+  timestamp: number;
+  countries?: any[];
+  visitors?: any[];
+}

src/lib/yup.ts

@@ -0,0 +1,18 @@
+import moment from 'moment-timezone';
+import * as yup from 'yup';
+import { UNIT_TYPES } from './constants';
+
+export const TimezoneTest = yup
+  .string()
+  .default('UTC')
+  .test(
+    'timezone',
+    () => `Invalid timezone`,
+    value => moment.tz.zone(value) !== null,
+  );
+
+export const UnitTypeTest = yup.string().test(
+  'unit',
+  () => `Invalid unit`,
+  value => UNIT_TYPES.includes(value),
+);