Merge pull request #1803 from umami-software/feat/um-171-cloud-mode-env-variable

Cities, Subdivisions, Page Title

db/clickhouse/schema.sql

@@ -15,9 +15,13 @@ CREATE TABLE event
     screen LowCardinality(String),
     language LowCardinality(String),
     country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    subdivision2 LowCardinality(String),
+    city String,
     --pageview
     url String,
     referrer String,
+    page_title String,
     --event
     event_type UInt32,
     event_name String,
@@ -33,8 +37,7 @@ CREATE TABLE event_queue (
     session_id UUID,
     event_id Nullable(UUID),
     rev_id UInt32,
-    url String,
+    --session
-    referrer String,
     hostname LowCardinality(String),
     browser LowCardinality(String),
     os LowCardinality(String),
@@ -42,6 +45,14 @@ CREATE TABLE event_queue (
     screen LowCardinality(String),
     language LowCardinality(String),
     country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    subdivision2 LowCardinality(String),
+    city String,
+    --pageview
+    url String,
+    referrer String,
+    page_title String,
+    --event
     event_type UInt32,
     event_name String,
     event_data String,
@@ -60,8 +71,6 @@ SELECT website_id,
     session_id,
     event_id,
     rev_id,
-    url,
-    referrer,
     hostname,
     browser,
     os,
@@ -69,8 +78,14 @@ SELECT website_id,
     screen,
     language,
     country,
+    subdivision1,
+    subdivision2,
+    city,
+    url,
+    referrer,
+    page_title,
     event_type,
     event_name,
-    if((empty(event_data) = 0) AND startsWith(event_data, '"'), concat('{', event_data, ': true}'), event_data) AS event_data,
+    event_data,
     created_at
 FROM event_queue;

db/mysql/migrations/01_init/migration.sql

@@ -24,6 +24,9 @@ CREATE TABLE `session` (
     `screen` VARCHAR(11) NULL,
     `language` VARCHAR(35) NULL,
     `country` CHAR(2) NULL,
+    `subdivision1` CHAR(3) NULL,
+    `subdivision2` VARCHAR(50) NULL,
+    `city` VARCHAR(50) NULL,
     `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
 
     UNIQUE INDEX `session_session_id_key`(`session_id`),
@@ -60,6 +63,7 @@ CREATE TABLE `website_event` (
     `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
     `url` VARCHAR(500) NOT NULL,
     `referrer` VARCHAR(500) NULL,
+    `page_title` VARCHAR(500) NULL,
     `event_type` INTEGER UNSIGNED NOT NULL DEFAULT 1,
     `event_name` VARCHAR(50) NULL,
     `event_data` JSON NULL,
@@ -117,3 +121,6 @@ CREATE TABLE `team_website` (
     INDEX `team_website_website_id_idx`(`website_id`),
     PRIMARY KEY (`team_website_id`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+-- AddSystemUser
+INSERT INTO "user" (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');

db/mysql/schema.prisma

@@ -25,16 +25,19 @@ model User {
 }
 
 model Session {
-  id        String    @id @unique @map("session_id") @db.VarChar(36)
+  id           String  @id @unique @map("session_id") @db.VarChar(36)
-  websiteId String    @map("website_id") @db.VarChar(36)
+  websiteId    String  @map("website_id") @db.VarChar(36)
-  hostname  String?   @db.VarChar(100)
+  hostname     String? @db.VarChar(100)
-  browser   String?   @db.VarChar(20)
+  browser      String? @db.VarChar(20)
-  os        String?   @db.VarChar(20)
+  os           String? @db.VarChar(20)
-  device    String?   @db.VarChar(20)
+  device       String? @db.VarChar(20)
-  screen    String?   @db.VarChar(11)
+  screen       String? @db.VarChar(11)
-  language  String?   @db.VarChar(35)
+  language     String? @db.VarChar(35)
-  country   String?   @db.Char(2)
+  country      String? @db.Char(2)
-  createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
+  subdivision1 String? @db.Char(3)
+  subdivision2 String? @db.VarChar(50)
+  city         String? @db.VarChar(50)
+  createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
 
   @@index([createdAt])
   @@index([websiteId])
@@ -68,6 +71,7 @@ model WebsiteEvent {
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
   url       String    @db.VarChar(500)
   referrer  String?   @db.VarChar(500)
+  pageTitle String?   @map("page_title") @db.VarChar(500)
   eventType Int       @default(1) @map("event_type") @db.UnsignedInt
   eventName String?   @map("event_name") @db.VarChar(50)
   eventData Json?     @map("event_data")

db/postgresql/migrations/05_add_page_title_subdivision_city/migration.sql

@@ -0,0 +1,7 @@
+-- AlterTable
+ALTER TABLE "session" ADD COLUMN     "city" VARCHAR(50),
+ADD COLUMN     "subdivision1" CHAR(3),
+ADD COLUMN     "subdivision2" VARCHAR(50);
+
+-- AlterTable
+ALTER TABLE "website_event" ADD COLUMN     "page_title" VARCHAR(500);

db/postgresql/schema.prisma

@@ -25,16 +25,19 @@ model User {
 }
 
 model Session {
-  id        String    @id @unique @map("session_id") @db.Uuid
+  id           String    @id @unique @map("session_id") @db.Uuid
-  websiteId String    @map("website_id") @db.Uuid
+  websiteId    String    @map("website_id") @db.Uuid
-  hostname  String?   @db.VarChar(100)
+  hostname     String?   @db.VarChar(100)
-  browser   String?   @db.VarChar(20)
+  browser      String?   @db.VarChar(20)
-  os        String?   @db.VarChar(20)
+  os           String?   @db.VarChar(20)
-  device    String?   @db.VarChar(20)
+  device       String?   @db.VarChar(20)
-  screen    String?   @db.VarChar(11)
+  screen       String?   @db.VarChar(11)
-  language  String?   @db.VarChar(35)
+  language     String?   @db.VarChar(35)
-  country   String?   @db.Char(2)
+  country      String?   @db.Char(2)
-  createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
+  subdivision1 String?   @db.Char(3)
+  subdivision2 String?   @db.VarChar(50)
+  city         String?   @db.VarChar(50)
+  createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
 
   @@index([createdAt])
   @@index([websiteId])
@@ -68,6 +71,7 @@ model WebsiteEvent {
   createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
   url       String    @db.VarChar(500)
   referrer  String?   @db.VarChar(500)
+  pageTitle String?   @map("page_title") @db.VarChar(500)
   eventType Int       @default(1) @map("event_type") @db.Integer
   eventName String?   @map("event_name") @db.VarChar(50)
   eventData Json?     @map("event_data")

lib/clickhouse.ts

@@ -112,9 +112,13 @@ function getFilterQuery(filters = {}, params = {}) {
 
     switch (key) {
       case 'url':
+      case 'pageTitle':
       case 'os':
       case 'browser':
       case 'device':
+      case 'subdivision1':
+      case 'subdivision2':
+      case 'city':
       case 'country':
         arr.push(`and ${key} = {${key}:String}`);
         params[key] = filter;
@@ -147,11 +151,25 @@ function getFilterQuery(filters = {}, params = {}) {
 }
 
 function parseFilters(filters: any = {}, params: any = {}) {
-  const { domain, url, eventUrl, referrer, os, browser, device, country, eventName, query } =
+  const {
-    filters;
+    domain,
+    url,
+    eventUrl,
+    referrer,
+    pageTitle,
+    os,
+    browser,
+    device,
+    country,
+    subdivision1,
+    subdivision2,
+    city,
+    eventName,
+    query,
+  } = filters;
 
-  const pageviewFilters = { domain, url, referrer, query };
+  const pageviewFilters = { domain, url, referrer, query, pageTitle };
-  const sessionFilters = { os, browser, device, country };
+  const sessionFilters = { os, browser, device, country, subdivision1, subdivision2, city };
   const eventFilters = { url: eventUrl, eventName };
 
   return {

lib/detect.js

@@ -27,7 +27,7 @@ export function getIpAddress(req) {
   return requestIp.getClientIp(req);
 }
 
-export function getDevice(screen, browser, os) {
+export function getDevice(screen, os) {
   if (!screen) return;
 
   const [width] = screen.split('x');
@@ -55,12 +55,7 @@ export function getDevice(screen, browser, os) {
   }
 }
 
-export async function getCountry(req, ip) {
+export async function getLocation(ip) {
-  // Cloudflare
-  if (req.headers['cf-ipcountry']) {
-    return req.headers['cf-ipcountry'];
-  }
-
   // Ignore local ips
   if (await isLocalhost(ip)) {
     return;
@@ -68,23 +63,31 @@ export async function getCountry(req, ip) {
 
   // Database lookup
   if (!lookup) {
-    lookup = await maxmind.open(path.resolve('node_modules/.geo/GeoLite2-Country.mmdb'));
+    lookup = await maxmind.open(path.resolve('node_modules/.geo/GeoLite2-City.mmdb'));
   }
 
   const result = lookup.get(ip);
+  const country = result?.country?.iso_code ?? result?.registered_country?.iso_code;
+  const subdivision1 = result?.subdivisions[0]?.iso_code;
+  const subdivision2 = result?.subdivisions[1]?.names?.en;
+  const city = result?.city?.names?.en;
 
-  return result?.country?.iso_code;
+  return { country, subdivision1, subdivision2, city };
 }
 
 export async function getClientInfo(req, { screen }) {
   const userAgent = req.headers['user-agent'];
   const ip = getIpAddress(req);
-  const country = await getCountry(req, ip);
+  const location = await getLocation(ip);
+  const country = location.country;
+  const subdivision1 = location.subdivision1;
+  const subdivision2 = location.subdivision2;
+  const city = location.city;
   const browser = browserName(userAgent);
   const os = detectOS(userAgent);
   const device = getDevice(screen, browser, os);
 
-  return { userAgent, browser, os, ip, country, device };
+  return { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device };
 }
 
 export function getJsonBody(req) {

lib/prisma.ts

@@ -133,8 +133,12 @@ function getFilterQuery(filters = {}, params = []): string {
     switch (key) {
       case 'url':
       case 'os':
+      case 'pageTitle':
       case 'browser':
       case 'device':
+      case 'subdivision1':
+      case 'subdivision2':
+      case 'city':
       case 'country':
         arr.push(`and ${key}=$${params.length + 1}`);
         params.push(decodeURIComponent(filter));
@@ -171,11 +175,25 @@ function parseFilters(
   params = [],
   sessionKey = 'session_id',
 ) {
-  const { domain, url, eventUrl, referrer, os, browser, device, country, eventName, query } =
+  const {
-    filters;
+    domain,
+    url,
+    eventUrl,
+    referrer,
+    pageTitle,
+    os,
+    browser,
+    device,
+    country,
+    subdivision1,
+    subdivision2,
+    city,
+    eventName,
+    query,
+  } = filters;
 
-  const pageviewFilters = { domain, url, referrer, query };
+  const pageviewFilters = { domain, url, referrer, query, pageTitle };
-  const sessionFilters = { os, browser, device, country };
+  const sessionFilters = { os, browser, device, country, subdivision1, subdivision2, city };
   const eventFilters = { url: eventUrl, eventName };
 
   return {
@@ -184,7 +202,7 @@ function parseFilters(
     eventFilters,
     event: { eventName },
     joinSession:
-      os || browser || device || country
+      os || browser || device || country || subdivision1 || subdivision2 || city
         ? `inner join session on website_event.${sessionKey} = session.${sessionKey}`
         : '',
     filterQuery: getFilterQuery(filters, params),

lib/session.js

@@ -44,7 +44,8 @@ export async function findSession(req) {
     throw new Error(`Website not found: ${websiteId}`);
   }
 
-  const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);
+  const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
+    await getClientInfo(req, payload);
   const sessionId = uuid(websiteId, hostname, ip, userAgent);
 
   // Clickhouse does not require session lookup
@@ -59,6 +60,9 @@ export async function findSession(req) {
       screen,
       language,
       country,
+      subdivision1,
+      subdivision2,
+      city,
     };
   }
 
@@ -84,6 +88,9 @@ export async function findSession(req) {
         screen,
         language,
         country,
+        subdivision1,
+        subdivision2,
+        city,
       });
     } catch (e) {
       if (!e.message.toLowerCase().includes('unique constraint')) {

pages/api/collect.ts

@@ -19,6 +19,9 @@ export interface NextApiRequestCollect extends NextApiRequest {
     screen: string;
     language: string;
     country: string;
+    subdivision1: string;
+    subdivision2: string;
+    city: string;
   };
 }
 
@@ -31,7 +34,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
 
   const { type, payload } = getJsonBody(req);
 
-  const { referrer, eventName, eventData } = payload;
+  const { referrer, eventName, eventData, pageTitle } = payload;
   let { url } = payload;
 
   // Validate eventData is JSON
@@ -95,12 +98,13 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
   }
 
   if (type === 'pageview') {
-    await savePageView({ ...session, url, referrer });
+    await savePageView({ ...session, url, referrer, pageTitle });
   } else if (type === 'event') {
     await saveEvent({
       ...session,
       url,
       referrer,
+      pageTitle,
       eventName,
       eventData,
     });

pages/api/config.ts

@@ -18,7 +18,7 @@ export default async (req: NextApiRequest, res: NextApiResponse<ConfigResponse>)
       updatesDisabled: !!process.env.DISABLE_UPDATES,
       telemetryDisabled: !!process.env.DISABLE_TELEMETRY,
       adminDisabled: !!process.env.DISABLE_ADMIN,
-      cloudMode: true, //!!process.env.CLOUD_MODE,
+      cloudMode: process.env.CLOUD_MODE,
     });
   }
 

pages/api/users/index.ts

@@ -2,10 +2,10 @@ import { canCreateUser, canViewUsers } from 'lib/auth';
 import { ROLES } from 'lib/constants';
 import { uuid } from 'lib/crypto';
 import { useAuth } from 'lib/middleware';
-import { NextApiRequestQueryBody } from 'lib/types';
+import { NextApiRequestQueryBody, User } from 'lib/types';
 import { NextApiResponse } from 'next';
 import { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics';
-import { createUser, getUser, getUsers, User } from 'queries';
+import { createUser, getUser, getUsers } from 'queries';
 
 export interface UsersRequestBody {
   username: string;

pages/api/websites/[id]/metrics.ts

@@ -7,7 +7,7 @@ import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';
 import { getPageviewMetrics, getSessionMetrics, getWebsite } from 'queries';
 
 const sessionColumns = ['browser', 'os', 'device', 'screen', 'country', 'language'];
-const pageviewColumns = ['url', 'referrer', 'query'];
+const pageviewColumns = ['url', 'referrer', 'query', 'pageTitle'];
 
 function getTable(type) {
   if (type === 'event') {
@@ -42,10 +42,14 @@ export interface WebsiteMetricsRequestQuery {
   endAt: number;
   url: string;
   referrer: string;
+  pageTitle: string;
   os: string;
   browser: string;
   device: string;
   country: string;
+  subdivision1: string;
+  subdivision2: string;
+  city: string;
 }
 
 export default async (
@@ -62,10 +66,14 @@ export default async (
     endAt,
     url,
     referrer,
+    pageTitle,
     os,
     browser,
     device,
     country,
+    subdivision1,
+    subdivision2,
+    city,
   } = req.query;
 
   if (req.method === 'GET') {
@@ -86,6 +94,9 @@ export default async (
           browser,
           device,
           country,
+          subdivision1,
+          subdivision2,
+          city,
         },
       });
 
@@ -127,10 +138,14 @@ export default async (
         domain,
         url: type !== 'url' && table !== 'event' ? url : undefined,
         referrer: type !== 'referrer' && table !== 'event' ? referrer : FILTER_IGNORED,
+        pageTitle: type !== 'pageTitle' && table !== 'event' ? pageTitle : undefined,
         os: type !== 'os' ? os : undefined,
         browser: type !== 'browser' ? browser : undefined,
         device: type !== 'device' ? device : undefined,
         country: type !== 'country' ? country : undefined,
+        subdivision1: type !== 'subdivision1' ? subdivision1 : undefined,
+        subdivision2: type !== 'subdivision2' ? subdivision2 : undefined,
+        city: type !== 'city' ? city : undefined,
         eventUrl: type !== 'url' && table === 'event' ? url : undefined,
         query: type === 'query' && table !== 'event' ? true : undefined,
       };

pages/api/websites/[id]/pageviews.ts

@@ -17,10 +17,14 @@ export interface WebsitePageviewRequestQuery {
   timezone: string;
   url?: string;
   referrer?: string;
+  pageTitle?: string;
   os?: string;
   browser?: string;
   device?: string;
   country?: string;
+  subdivision1?: string;
+  subdivision2?: string;
+  city?: string;
 }
 
 export default async (
@@ -38,10 +42,14 @@ export default async (
     timezone,
     url,
     referrer,
+    pageTitle,
     os,
     browser,
     device,
     country,
+    subdivision1,
+    subdivision2,
+    city,
   } = req.query;
 
   if (req.method === 'GET') {
@@ -66,10 +74,14 @@ export default async (
         filters: {
           url,
           referrer,
+          pageTitle,
           os,
           browser,
           device,
           country,
+          subdivision1,
+          subdivision2,
+          city,
         },
       }),
       getPageviewStats(websiteId, {
@@ -80,10 +92,14 @@ export default async (
         count: 'distinct website_event.',
         filters: {
           url,
+          pageTitle,
           os,
           browser,
           device,
           country,
+          subdivision1,
+          subdivision2,
+          city,
         },
       }),
     ]);

pages/api/websites/[id]/stats.ts

@@ -13,10 +13,14 @@ export interface WebsiteStatsRequestQuery {
   endAt: number;
   url: string;
   referrer: string;
+  pageTitle: string;
   os: string;
   browser: string;
   device: string;
   country: string;
+  subdivision1: string;
+  subdivision2: string;
+  city: string;
 }
 
 export default async (
@@ -26,7 +30,21 @@ export default async (
   await useCors(req, res);
   await useAuth(req, res);
 
-  const { id: websiteId, startAt, endAt, url, referrer, os, browser, device, country } = req.query;
+  const {
+    id: websiteId,
+    startAt,
+    endAt,
+    url,
+    referrer,
+    pageTitle,
+    os,
+    browser,
+    device,
+    country,
+    subdivision1,
+    subdivision2,
+    city,
+  } = req.query;
 
   if (req.method === 'GET') {
     if (!(await canViewWebsite(req.auth, websiteId))) {
@@ -46,10 +64,14 @@ export default async (
       filters: {
         url,
         referrer,
+        pageTitle,
         os,
         browser,
         device,
         country,
+        subdivision1,
+        subdivision2,
+        city,
       },
     });
     const prevPeriod = await getWebsiteStats(websiteId, {
@@ -58,10 +80,14 @@ export default async (
       filters: {
         url,
         referrer,
+        pageTitle,
         os,
         browser,
         device,
         country,
+        subdivision1,
+        subdivision2,
+        city,
       },
     });
 

queries/analytics/event/saveEvent.ts

@@ -10,6 +10,7 @@ export async function saveEvent(args: {
   websiteId: string;
   url: string;
   referrer?: string;
+  pageTitle?: string;
   eventName?: string;
   eventData?: any;
   hostname?: string;
@@ -19,6 +20,9 @@ export async function saveEvent(args: {
   screen?: string;
   language?: string;
   country?: string;
+  subdivision1?: string;
+  subdivision2?: string;
+  city?: string;
 }) {
   return runQuery({
     [PRISMA]: () => relationalQuery(args),
@@ -31,43 +35,41 @@ async function relationalQuery(data: {
   websiteId: string;
   url: string;
   referrer?: string;
+  pageTitle?: string;
   eventName?: string;
   eventData?: any;
 }) {
-  const { websiteId, id: sessionId, url, eventName, eventData, referrer } = data;
+  const { websiteId, id: sessionId, url, eventName, eventData, referrer, pageTitle } = data;
-
-  const params = {
-    id: uuid(),
-    websiteId,
-    sessionId,
-    url: url?.substring(0, URL_LENGTH),
-    referrer: referrer?.substring(0, URL_LENGTH),
-    eventType: EVENT_TYPE.customEvent,
-    eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
-    eventData,
-  };
 
   return prisma.client.websiteEvent.create({
-    data: params,
+    data: {
+      id: uuid(),
+      websiteId,
+      sessionId,
+      url: url?.substring(0, URL_LENGTH),
+      referrer: referrer?.substring(0, URL_LENGTH),
+      pageTitle: pageTitle,
+      eventType: EVENT_TYPE.customEvent,
+      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+      eventData,
+    },
   });
 }
 
-async function clickhouseQuery(data: {
+async function clickhouseQuery(data) {
-  id: string;
+  const {
-  websiteId: string;
+    websiteId,
-  url: string;
+    id: sessionId,
-  referrer?: string;
+    url,
-  eventName?: string;
+    pageTitle,
-  eventData?: any;
+    eventName,
-  hostname?: string;
+    eventData,
-  browser?: string;
+    country,
-  os?: string;
+    subdivision1,
-  device?: string;
+    subdivision2,
-  screen?: string;
+    city,
-  language?: string;
+    ...args
-  country?: string;
+  } = data;
-}) {
-  const { websiteId, id: sessionId, url, eventName, eventData, country, ...args } = data;
   const { getDateFormat, sendMessage } = kafka;
   const website = await cache.fetchWebsite(websiteId);
 
@@ -75,13 +77,17 @@ async function clickhouseQuery(data: {
     website_id: websiteId,
     session_id: sessionId,
     event_id: uuid(),
+    rev_id: website?.revId || 0,
+    country: country ? country : null,
+    subdivision1: subdivision1 ? subdivision1 : null,
+    subdivision2: subdivision2 ? subdivision2 : null,
+    city: city ? city : null,
     url: url?.substring(0, URL_LENGTH),
+    page_title: pageTitle,
     event_type: EVENT_TYPE.customEvent,
     event_name: eventName?.substring(0, EVENT_NAME_LENGTH),
     event_data: eventData ? JSON.stringify(eventData) : null,
-    rev_id: website?.revId || 0,
     created_at: getDateFormat(new Date()),
-    country: country ? country : null,
     ...args,
   };
 

queries/analytics/pageview/savePageView.ts

@@ -10,6 +10,7 @@ export async function savePageView(args: {
   websiteId: string;
   url: string;
   referrer?: string;
+  pageTitle?: string;
   hostname?: string;
   browser?: string;
   os?: string;
@@ -17,6 +18,9 @@ export async function savePageView(args: {
   screen?: string;
   language?: string;
   country?: string;
+  subdivision1?: string;
+  subdivision2?: string;
+  city?: string;
 }) {
   return runQuery({
     [PRISMA]: () => relationalQuery(args),
@@ -29,8 +33,9 @@ async function relationalQuery(data: {
   websiteId: string;
   url: string;
   referrer?: string;
+  pageTitle?: string;
 }) {
-  const { websiteId, id: sessionId, url, referrer } = data;
+  const { websiteId, id: sessionId, url, referrer, pageTitle } = data;
 
   return prisma.client.websiteEvent.create({
     data: {
@@ -39,25 +44,41 @@ async function relationalQuery(data: {
       sessionId,
       url: url?.substring(0, URL_LENGTH),
       referrer: referrer?.substring(0, URL_LENGTH),
+      pageTitle: pageTitle,
       eventType: EVENT_TYPE.pageView,
     },
   });
 }
 
 async function clickhouseQuery(data) {
-  const { websiteId, id: sessionId, url, referrer, country, ...args } = data;
+  const {
-  const website = await cache.fetchWebsite(websiteId);
+    websiteId,
+    id: sessionId,
+    url,
+    referrer,
+    pageTitle,
+    country,
+    subdivision1,
+    subdivision2,
+    city,
+    ...args
+  } = data;
   const { getDateFormat, sendMessage } = kafka;
+  const website = await cache.fetchWebsite(websiteId);
 
   const message = {
-    session_id: sessionId,
     website_id: websiteId,
+    session_id: sessionId,
+    rev_id: website?.revId || 0,
+    country: country ? country : null,
+    subdivision1: subdivision1 ? subdivision1 : null,
+    subdivision2: subdivision2 ? subdivision2 : null,
+    city: city ? city : null,
     url: url?.substring(0, URL_LENGTH),
     referrer: referrer?.substring(0, URL_LENGTH),
-    rev_id: website?.revId || 0,
+    page_title: pageTitle,
-    created_at: getDateFormat(new Date()),
-    country: country ? country : null,
     event_type: EVENT_TYPE.pageView,
+    created_at: getDateFormat(new Date()),
     ...args,
   };
 

queries/analytics/session/createSession.ts

@@ -31,8 +31,24 @@ async function clickhouseQuery(data: {
   screen?: string;
   language?: string;
   country?: string;
+  subdivision1?: string;
+  subdivision2?: string;
+  city?: string;
 }) {
-  const { id, websiteId, hostname, browser, os, device, screen, language, country } = data;
+  const {
+    id,
+    websiteId,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    subdivision2,
+    city,
+  } = data;
   const { getDateFormat, sendMessage } = kafka;
   const website = await cache.fetchWebsite(websiteId);
 
@@ -46,6 +62,9 @@ async function clickhouseQuery(data: {
     screen,
     language,
     country,
+    subdivision1,
+    subdivision2,
+    city,
     rev_id: website?.revId || 0,
     created_at: getDateFormat(new Date()),
   };

queries/analytics/session/getSession.ts

@@ -31,7 +31,10 @@ async function clickhouseQuery({ id: sessionId }: { id: string }) {
       device, 
       screen,
       language, 
-      country 
+      country,
+      subdivision1,
+      subdivision2,
+      city
     from event
       where session_id = {sessionId:UUID}
     limit 1`,

queries/analytics/session/getSessions.ts

@@ -35,7 +35,10 @@ async function clickhouseQuery(websiteId: string, startAt: Date) {
       device,
       screen,
       language,
-      country
+      country,
+      subdivision1,
+      subdivision2,
+      city
     from event
     where website_id = {websiteId:UUID}
       and created_at >= {startAt:DateTime('UTC')}`,

scripts/build-geo.js

@@ -7,12 +7,12 @@ const zlib = require('zlib');
 const tar = require('tar');
 
 let url =
-  'https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/GeoLite2-Country.tar.gz';
+  'https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/GeoLite2-City.tar.gz';
 
 if (process.env.MAXMIND_LICENSE_KEY) {
   url =
     `https://download.maxmind.com/app/geoip_download` +
-    `?edition_id=GeoLite2-Country&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`;
+    `?edition_id=GeoLite2-City&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`;
 }
 
 const dest = path.resolve(__dirname, '../node_modules/.geo');

tracker/index.js

@@ -47,6 +47,7 @@
     (dnt && doNotTrack()) ||
     (domain && !domains.includes(hostname));
 
+  const tracker_delay_duration = 300;
   const _data = 'data-';
   const _false = 'false';
   const attr = currentScript.getAttribute.bind(currentScript);
@@ -68,6 +69,7 @@
   let listeners = {};
   let currentUrl = `${pathname}${search}`;
   let currentRef = document.referrer;
+  let currentPageTitle = document.title;
   let cache;
 
   /* Collect metrics */
@@ -92,22 +94,35 @@
       .then(text => (cache = text));
   };
 
-  const trackView = (url = currentUrl, referrer = currentRef, websiteId = website) =>
+  const trackView = (
+    url = currentUrl,
+    referrer = currentRef,
+    websiteId = website,
+    pageTitle = currentPageTitle,
+  ) =>
     collect(
       'pageview',
       assign(getPayload(), {
         website: websiteId,
         url,
         referrer,
+        pageTitle,
       }),
     );
 
-  const trackEvent = (eventName, eventData, url = currentUrl, websiteId = website) =>
+  const trackEvent = (
+    eventName,
+    eventData,
+    url = currentUrl,
+    websiteId = website,
+    pageTitle = currentPageTitle,
+  ) =>
     collect(
       'event',
       assign(getPayload(), {
         website: websiteId,
         url,
+        pageTitle,
         eventName: eventName,
         eventData: eventData,
       }),
@@ -162,6 +177,7 @@
   const handlePush = (state, title, url) => {
     if (!url) return;
 
+    observeTitle();
     currentRef = currentUrl;
     const newUrl = url.toString();
 
@@ -172,7 +188,7 @@
     }
 
     if (currentUrl !== currentRef) {
-      trackView();
+      setTimeout(() => trackView(), tracker_delay_duration);
     }
   };
 
@@ -189,6 +205,19 @@
     observer.observe(document, { childList: true, subtree: true });
   };
 
+  const observeTitle = () => {
+    const monitorMutate = mutations => {
+      currentPageTitle = mutations[0].target.text;
+    };
+
+    const observer = new MutationObserver(monitorMutate);
+    observer.observe(document.querySelector('title'), {
+      subtree: true,
+      characterData: true,
+      childList: true,
+    });
+  };
+
   /* Global */
 
   if (!window.umami) {