m/umami
1
0
Fork 0
mirror of https://github.com/kremalicious/umami.git synced 2024-11-15 09:45:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/mikecao/umami into dev

Browse Source
This commit is contained in:
Mike Cao 2022-08-23 00:48:57 -07:00
commit 700158f5f6
25 changed files with 1089 additions and 509 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -76,12 +76,12 @@ docker-compose up
Alternatively, to pull just the Umami Docker image with PostgreSQL support:
```bash
docker pull docker.umami.is/umami-software/umami:postgresql-latest
docker pull docker.umami.dev/umami-software/umami:postgresql-latest
```
Or with MySQL support:
```bash
docker pull docker.umami.is/umami-software/umami:mysql-latest
docker pull docker.umami.dev/umami-software/umami:mysql-latest
```
## Getting updates

129
db/clickhouse/migrations/01_init/migration.sql Normal file
View File

@ -0,0 +1,129 @@
-- Create Pageview
CREATE TABLE pageview
(
website_id UInt32,
session_uuid UUID,
created_at DateTime('UTC'),
url String,
referrer String
)
engine = MergeTree PRIMARY KEY (session_uuid, created_at)
ORDER BY (session_uuid, created_at)
SETTINGS index_granularity = 8192;
CREATE TABLE pageview_queue (
website_id UInt32,
session_uuid UUID,
created_at DateTime('UTC'),
url String,
referrer String
)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'localhost:9092,localhost:9093,localhost:9094', -- input broker list
kafka_topic_list = 'pageview',
kafka_group_name = 'pageview_consumer_group',
kafka_format = 'JSONEachRow',
kafka_max_block_size = 1048576,
kafka_skip_broken_messages = 1;
CREATE MATERIALIZED VIEW pageview_queue_mv TO pageview AS
SELECT website_id,
session_uuid,
created_at,
url,
referrer
FROM pageview_queue;
-- Create Session
CREATE TABLE session
(
session_uuid UUID,
website_id UInt32,
created_at DateTime('UTC'),
hostname LowCardinality(String),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String)
)
engine = MergeTree PRIMARY KEY (session_uuid, created_at)
ORDER BY (session_uuid, created_at)
SETTINGS index_granularity = 8192;
CREATE TABLE session_queue (
session_uuid UUID,
website_id UInt32,
created_at DateTime('UTC'),
hostname LowCardinality(String),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String)
)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'localhost:9092,localhost:9093,localhost:9094', -- input broker list
kafka_topic_list = 'session',
kafka_group_name = 'session_consumer_group',
kafka_format = 'JSONEachRow',
kafka_max_block_size = 1048576,
kafka_skip_broken_messages = 1;
CREATE MATERIALIZED VIEW session_queue_mv TO session AS
SELECT session_uuid,
website_id,
created_at,
hostname,
browser,
os,
device,
screen,
language,
country
FROM session_queue;
-- Create event
CREATE TABLE event
(
event_uuid UUID,
website_id UInt32,
session_uuid UUID,
created_at DateTime('UTC'),
url String,
event_name String,
event_data String
)
engine = MergeTree PRIMARY KEY (event_uuid, created_at)
ORDER BY (event_uuid, created_at)
SETTINGS index_granularity = 8192;
CREATE TABLE event_queue (
event_uuid UUID,
website_id UInt32,
session_uuid UUID,
created_at DateTime('UTC'),
url String,
event_name String,
event_data String
)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'localhost:9092,localhost:9093,localhost:9094', -- input broker list
kafka_topic_list = 'event',
kafka_group_name = 'event_consumer_group',
kafka_format = 'JSONEachRow',
kafka_max_block_size = 1048576,
kafka_skip_broken_messages = 1;
CREATE MATERIALIZED VIEW event_queue_mv TO event AS
SELECT event_uuid,
website_id,
session_uuid,
created_at,
url,
event_name,
event_data
FROM event_queue;

44
db/mysql/migrations/02_add_event_data/migration.sql
View File

@ -49,14 +49,42 @@ CREATE TABLE `event_data` (
-- AddForeignKey
ALTER TABLE `event_data` ADD CONSTRAINT `event_data_event_id_fkey` FOREIGN KEY (`event_id`) REFERENCES `event`(`event_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- RenameIndex
ALTER TABLE `account` RENAME INDEX `username` TO `account_username_key`;
-- CreateProcedureRenameIndex
CREATE PROCEDURE `UmamiRenameIndexIfExists`(
IN i_table_name VARCHAR(128),
IN i_current_index_name VARCHAR(128),
IN i_new_index_name VARCHAR(128)
)
BEGIN
SET @tableName = i_table_name;
SET @currentIndexName = i_current_index_name;
SET @newIndexName = i_new_index_name;
SET @indexExists = 0;
SELECT
1
INTO @indexExists FROM
INFORMATION_SCHEMA.STATISTICS
WHERE
TABLE_NAME = @tableName
AND INDEX_NAME = @currentIndexName;
SET @query = CONCAT(
'ALTER TABLE `', @tableName, '` RENAME INDEX `', @currentIndexName, '` TO `', @newIndexName, '`;'
);
IF @indexExists THEN
PREPARE stmt FROM @query;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
END IF;
END;
-- RenameIndex
ALTER TABLE `session` RENAME INDEX `session_uuid` TO `session_session_uuid_key`;
CALL UmamiRenameIndexIfExists('account', 'username', 'account_username_key');
CALL UmamiRenameIndexIfExists('session', 'session_uuid', 'session_session_uuid_key');
CALL UmamiRenameIndexIfExists('website', 'share_id', 'website_share_id_key');
CALL UmamiRenameIndexIfExists('website', 'website_uuid', 'website_website_uuid_key');
-- RenameIndex
ALTER TABLE `website` RENAME INDEX `share_id` TO `website_share_id_key`;
-- RenameIndex
ALTER TABLE `website` RENAME INDEX `website_uuid` TO `website_website_uuid_key`;
-- Drop CreateProcedureRenameIndex
drop procedure `UmamiRenameIndexIfExists`;

106
db/mysql/migrations/03_big_int/migration.sql Normal file
View File

@ -0,0 +1,106 @@
/*
Warnings:
- The primary key for the `account` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to alter the column `user_id` on the `account` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- The primary key for the `event` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to alter the column `event_id` on the `event` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to alter the column `website_id` on the `event` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to alter the column `session_id` on the `event` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to alter the column `event_data_id` on the `event_data` table. The data in that column could be lost. The data in that column will be cast from `Int` to `UnsignedBigInt`.
- You are about to alter the column `event_id` on the `event_data` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- The primary key for the `pageview` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to alter the column `view_id` on the `pageview` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to alter the column `website_id` on the `pageview` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to alter the column `session_id` on the `pageview` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- The primary key for the `session` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to alter the column `session_id` on the `session` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to alter the column `website_id` on the `session` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- The primary key for the `website` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to alter the column `website_id` on the `website` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to alter the column `user_id` on the `website` table. The data in that column could be lost. The data in that column will be cast from `UnsignedInt` to `UnsignedBigInt`.
- You are about to drop the `_event_old` table. If the table is not empty, all the data it contains will be lost.
*/
-- DropForeignKey
ALTER TABLE `event` DROP FOREIGN KEY `event_ibfk_1`;
-- DropForeignKey
ALTER TABLE `event` DROP FOREIGN KEY `event_ibfk_2`;
-- DropForeignKey
ALTER TABLE `event_data` DROP FOREIGN KEY `event_data_event_id_fkey`;
-- DropForeignKey
ALTER TABLE `pageview` DROP FOREIGN KEY `pageview_ibfk_1`;
-- DropForeignKey
ALTER TABLE `pageview` DROP FOREIGN KEY `pageview_ibfk_2`;
-- DropForeignKey
ALTER TABLE `session` DROP FOREIGN KEY `session_ibfk_1`;
-- DropForeignKey
ALTER TABLE `website` DROP FOREIGN KEY `website_ibfk_1`;
-- AlterTable
ALTER TABLE `account` DROP PRIMARY KEY,
MODIFY `user_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
ADD PRIMARY KEY (`user_id`);
-- AlterTable
ALTER TABLE `event` DROP PRIMARY KEY,
MODIFY `event_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
MODIFY `website_id` BIGINT UNSIGNED NOT NULL,
MODIFY `session_id` BIGINT UNSIGNED NOT NULL,
ADD PRIMARY KEY (`event_id`);
-- AlterTable
ALTER TABLE `event_data` DROP PRIMARY KEY,
MODIFY `event_data_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
MODIFY `event_id` BIGINT UNSIGNED NOT NULL,
ADD PRIMARY KEY (`event_data_id`);
-- AlterTable
ALTER TABLE `pageview` DROP PRIMARY KEY,
MODIFY `view_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
MODIFY `website_id` BIGINT UNSIGNED NOT NULL,
MODIFY `session_id` BIGINT UNSIGNED NOT NULL,
ADD PRIMARY KEY (`view_id`);
-- AlterTable
ALTER TABLE `session` DROP PRIMARY KEY,
MODIFY `session_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
MODIFY `website_id` BIGINT UNSIGNED NOT NULL,
ADD PRIMARY KEY (`session_id`);
-- AlterTable
ALTER TABLE `website` DROP PRIMARY KEY,
MODIFY `website_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
MODIFY `user_id` BIGINT UNSIGNED NOT NULL,
ADD PRIMARY KEY (`website_id`);
-- DropTable
DROP TABLE `_event_old`;
-- AddForeignKey
ALTER TABLE `event` ADD CONSTRAINT `event_ibfk_2` FOREIGN KEY (`session_id`) REFERENCES `session`(`session_id`) ON DELETE CASCADE ON UPDATE NO ACTION;
-- AddForeignKey
ALTER TABLE `event` ADD CONSTRAINT `event_ibfk_1` FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE CASCADE ON UPDATE NO ACTION;
-- AddForeignKey
ALTER TABLE `event_data` ADD CONSTRAINT `event_data_event_id_fkey` FOREIGN KEY (`event_id`) REFERENCES `event`(`event_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `pageview` ADD CONSTRAINT `pageview_ibfk_2` FOREIGN KEY (`session_id`) REFERENCES `session`(`session_id`) ON DELETE CASCADE ON UPDATE NO ACTION;
-- AddForeignKey
ALTER TABLE `pageview` ADD CONSTRAINT `pageview_ibfk_1` FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE CASCADE ON UPDATE NO ACTION;
-- AddForeignKey
ALTER TABLE `session` ADD CONSTRAINT `session_ibfk_1` FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE CASCADE ON UPDATE NO ACTION;
-- AddForeignKey
ALTER TABLE `website` ADD CONSTRAINT `website_ibfk_1` FOREIGN KEY (`user_id`) REFERENCES `account`(`user_id`) ON DELETE CASCADE ON UPDATE NO ACTION;

35
db/mysql/migrations/04_remove_cascade_delete/migration.sql Normal file
View File

@ -0,0 +1,35 @@
-- DropForeignKey
ALTER TABLE `event` DROP FOREIGN KEY `event_ibfk_2`;
-- DropForeignKey
ALTER TABLE `event` DROP FOREIGN KEY `event_ibfk_1`;
-- DropForeignKey
ALTER TABLE `pageview` DROP FOREIGN KEY `pageview_ibfk_2`;
-- DropForeignKey
ALTER TABLE `pageview` DROP FOREIGN KEY `pageview_ibfk_1`;
-- DropForeignKey
ALTER TABLE `session` DROP FOREIGN KEY `session_ibfk_1`;
-- DropForeignKey
ALTER TABLE `website` DROP FOREIGN KEY `website_ibfk_1`;
-- AddForeignKey
ALTER TABLE `event` ADD CONSTRAINT `event_session_id_fkey` FOREIGN KEY (`session_id`) REFERENCES `session`(`session_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `event` ADD CONSTRAINT `event_website_id_fkey` FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `pageview` ADD CONSTRAINT `pageview_session_id_fkey` FOREIGN KEY (`session_id`) REFERENCES `session`(`session_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `pageview` ADD CONSTRAINT `pageview_website_id_fkey` FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `session` ADD CONSTRAINT `session_website_id_fkey` FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `website` ADD CONSTRAINT `website_user_id_fkey` FOREIGN KEY (`user_id`) REFERENCES `account`(`user_id`) ON DELETE RESTRICT ON UPDATE CASCADE;

40
db/mysql/schema.prisma
View File

@ -8,7 +8,7 @@ datasource db {
}
model account {
user_id Int @id @default(autoincrement()) @db.UnsignedInt
user_id BigInt @id @default(autoincrement()) @db.UnsignedBigInt
username String @unique() @db.VarChar(255)
password String @db.VarChar(60)
is_admin Boolean @default(false)
@ -18,14 +18,14 @@ model account {
}
model event {
event_id Int @id @default(autoincrement()) @db.UnsignedInt
website_id Int @db.UnsignedInt
session_id Int @db.UnsignedInt
event_id BigInt @id @default(autoincrement()) @db.UnsignedBigInt
website_id BigInt @db.UnsignedBigInt
session_id BigInt @db.UnsignedBigInt
created_at DateTime? @default(now()) @db.Timestamp(0)
url String @db.VarChar(500)
event_name String @db.VarChar(50)
session session @relation(fields: [session_id], references: [session_id], onDelete: Cascade, onUpdate: NoAction, map: "event_ibfk_2")
website website @relation(fields: [website_id], references: [website_id], onDelete: Cascade, onUpdate: NoAction, map: "event_ibfk_1")
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
event_data event_data?
@@index([created_at])
@ -34,21 +34,21 @@ model event {
}
model event_data {
event_data_id Int @id @default(autoincrement())
event_id Int @unique @db.UnsignedInt
event_data_id BigInt @id @default(autoincrement()) @db.UnsignedBigInt
event_id BigInt @unique @db.UnsignedBigInt
event_data Json
event event @relation(fields: [event_id], references: [event_id])
event event @relation(fields: [event_id], references: [event_id])
}
model pageview {
view_id Int @id @default(autoincrement()) @db.UnsignedInt
website_id Int @db.UnsignedInt
session_id Int @db.UnsignedInt
view_id BigInt @id @default(autoincrement()) @db.UnsignedBigInt
website_id BigInt @db.UnsignedBigInt
session_id BigInt @db.UnsignedBigInt
created_at DateTime? @default(now()) @db.Timestamp(0)
url String @db.VarChar(500)
referrer String? @db.VarChar(500)
session session @relation(fields: [session_id], references: [session_id], onDelete: Cascade, onUpdate: NoAction, map: "pageview_ibfk_2")
website website @relation(fields: [website_id], references: [website_id], onDelete: Cascade, onUpdate: NoAction, map: "pageview_ibfk_1")
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
@@index([created_at])
@@index([session_id])
@ -58,9 +58,9 @@ model pageview {
}
model session {
session_id Int @id @default(autoincrement()) @db.UnsignedInt
session_id BigInt @id @default(autoincrement()) @db.UnsignedBigInt
session_uuid String @unique() @db.VarChar(36)
website_id Int @db.UnsignedInt
website_id BigInt @db.UnsignedBigInt
created_at DateTime? @default(now()) @db.Timestamp(0)
hostname String? @db.VarChar(100)
browser String? @db.VarChar(20)
@ -69,7 +69,7 @@ model session {
screen String? @db.VarChar(11)
language String? @db.VarChar(35)
country String? @db.Char(2)
website website @relation(fields: [website_id], references: [website_id], onDelete: Cascade, onUpdate: NoAction, map: "session_ibfk_1")
website website @relation(fields: [website_id], references: [website_id])
event event[]
pageview pageview[]
@ -78,14 +78,14 @@ model session {
}
model website {
website_id Int @id @default(autoincrement()) @db.UnsignedInt
website_id BigInt @id @default(autoincrement()) @db.UnsignedBigInt
website_uuid String @unique() @db.VarChar(36)
user_id Int @db.UnsignedInt
user_id BigInt @db.UnsignedBigInt
name String @db.VarChar(100)
domain String? @db.VarChar(500)
share_id String? @unique() @db.VarChar(64)
created_at DateTime? @default(now()) @db.Timestamp(0)
account account @relation(fields: [user_id], references: [user_id], onDelete: Cascade, onUpdate: NoAction, map: "website_ibfk_1")
account account @relation(fields: [user_id], references: [user_id])
event event[]
pageview pageview[]
session session[]

8
db/postgresql/migrations/02_add_event_data/migration.sql
View File

@ -54,13 +54,13 @@ CREATE UNIQUE INDEX "event_data_event_id_key" ON "event_data"("event_id");
ALTER TABLE "event_data" ADD CONSTRAINT "event_data_event_id_fkey" FOREIGN KEY ("event_id") REFERENCES "event"("event_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- RenameIndex
ALTER INDEX "account.username_unique" RENAME TO "account_username_key";
ALTER INDEX IF EXISTS "account.username_unique" RENAME TO "account_username_key";
-- RenameIndex
ALTER INDEX "session.session_uuid_unique" RENAME TO "session_session_uuid_key";
ALTER INDEX IF EXISTS "session.session_uuid_unique" RENAME TO "session_session_uuid_key";
-- RenameIndex
ALTER INDEX "website.share_id_unique" RENAME TO "website_share_id_key";
ALTER INDEX IF EXISTS "website.share_id_unique" RENAME TO "website_share_id_key";
-- RenameIndex
ALTER INDEX "website.website_uuid_unique" RENAME TO "website_website_uuid_key";
ALTER INDEX IF EXISTS "website.website_uuid_unique" RENAME TO "website_website_uuid_key";

101
db/postgresql/migrations/03_big_int/migration.sql Normal file
View File

@ -0,0 +1,101 @@
/*
Warnings:
- The primary key for the `account` table will be changed. If it partially fails, the table could be left without primary key constraint.
- The primary key for the `event` table will be changed. If it partially fails, the table could be left without primary key constraint.
- The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint.
- The primary key for the `pageview` table will be changed. If it partially fails, the table could be left without primary key constraint.
- The primary key for the `session` table will be changed. If it partially fails, the table could be left without primary key constraint.
- The primary key for the `website` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to drop the `_event_old` table. If the table is not empty, all the data it contains will be lost.
*/
-- DropForeignKey
ALTER TABLE "event" DROP CONSTRAINT "event_session_id_fkey";
-- DropForeignKey
ALTER TABLE "event" DROP CONSTRAINT "event_website_id_fkey";
-- DropForeignKey
ALTER TABLE "event_data" DROP CONSTRAINT "event_data_event_id_fkey";
-- DropForeignKey
ALTER TABLE "pageview" DROP CONSTRAINT "pageview_session_id_fkey";
-- DropForeignKey
ALTER TABLE "pageview" DROP CONSTRAINT "pageview_website_id_fkey";
-- DropForeignKey
ALTER TABLE "session" DROP CONSTRAINT "session_website_id_fkey";
-- DropForeignKey
ALTER TABLE "website" DROP CONSTRAINT "website_user_id_fkey";
-- AlterTable
ALTER TABLE "account" DROP CONSTRAINT "account_pkey",
ALTER COLUMN "user_id" SET DATA TYPE BIGINT,
ADD CONSTRAINT "account_pkey" PRIMARY KEY ("user_id");
-- AlterTable
ALTER TABLE "event" DROP CONSTRAINT "event_pkey",
ALTER COLUMN "event_id" SET DATA TYPE BIGINT,
ALTER COLUMN "website_id" SET DATA TYPE BIGINT,
ALTER COLUMN "session_id" SET DATA TYPE BIGINT,
ADD CONSTRAINT "event_pkey" PRIMARY KEY ("event_id");
-- AlterTable
ALTER TABLE "event_data" DROP CONSTRAINT "event_data_pkey",
ALTER COLUMN "event_data_id" SET DATA TYPE BIGINT,
ALTER COLUMN "event_id" SET DATA TYPE BIGINT,
ADD CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_data_id");
-- AlterTable
ALTER TABLE "pageview" DROP CONSTRAINT "pageview_pkey",
ALTER COLUMN "view_id" SET DATA TYPE BIGINT,
ALTER COLUMN "website_id" SET DATA TYPE BIGINT,
ALTER COLUMN "session_id" SET DATA TYPE BIGINT,
ADD CONSTRAINT "pageview_pkey" PRIMARY KEY ("view_id");
-- AlterTable
ALTER TABLE "session" DROP CONSTRAINT "session_pkey",
ALTER COLUMN "session_id" SET DATA TYPE BIGINT,
ALTER COLUMN "website_id" SET DATA TYPE BIGINT,
ADD CONSTRAINT "session_pkey" PRIMARY KEY ("session_id");
-- AlterTable
ALTER TABLE "website" DROP CONSTRAINT "website_pkey",
ALTER COLUMN "website_id" SET DATA TYPE BIGINT,
ALTER COLUMN "user_id" SET DATA TYPE BIGINT,
ADD CONSTRAINT "website_pkey" PRIMARY KEY ("website_id");
-- Dev: Alter Sequence
ALTER SEQUENCE account_user_id_seq AS BIGINT;
ALTER SEQUENCE event_data_event_data_id_seq AS BIGINT;
ALTER SEQUENCE event_event_id_seq AS BIGINT;
ALTER SEQUENCE pageview_view_id_seq AS BIGINT;
ALTER SEQUENCE session_session_id_seq AS BIGINT;
ALTER SEQUENCE website_website_id_seq AS BIGINT;
-- DropTable
DROP TABLE "_event_old";
-- AddForeignKey
ALTER TABLE "event" ADD CONSTRAINT "event_session_id_fkey" FOREIGN KEY ("session_id") REFERENCES "session"("session_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "event" ADD CONSTRAINT "event_website_id_fkey" FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "event_data" ADD CONSTRAINT "event_data_event_id_fkey" FOREIGN KEY ("event_id") REFERENCES "event"("event_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "pageview" ADD CONSTRAINT "pageview_session_id_fkey" FOREIGN KEY ("session_id") REFERENCES "session"("session_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "pageview" ADD CONSTRAINT "pageview_website_id_fkey" FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "session" ADD CONSTRAINT "session_website_id_fkey" FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "website" ADD CONSTRAINT "website_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "account"("user_id") ON DELETE CASCADE ON UPDATE CASCADE;

35
db/postgresql/migrations/04_remove_casade_delete/migration.sql Normal file
View File

@ -0,0 +1,35 @@
-- DropForeignKey
ALTER TABLE "event" DROP CONSTRAINT IF EXISTS "event_session_id_fkey";
-- DropForeignKey
ALTER TABLE "event" DROP CONSTRAINT IF EXISTS "event_website_id_fkey";
-- DropForeignKey
ALTER TABLE "pageview" DROP CONSTRAINT IF EXISTS "pageview_session_id_fkey";
-- DropForeignKey
ALTER TABLE "pageview" DROP CONSTRAINT IF EXISTS "pageview_website_id_fkey";
-- DropForeignKey
ALTER TABLE "session" DROP CONSTRAINT IF EXISTS "session_website_id_fkey";
-- DropForeignKey
ALTER TABLE "website" DROP CONSTRAINT IF EXISTS "website_user_id_fkey";
-- AddForeignKey
ALTER TABLE "event" ADD CONSTRAINT EXISTS "event_session_id_fkey" FOREIGN KEY ("session_id") REFERENCES "session"("session_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "event" ADD CONSTRAINT "event_website_id_fkey" FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "pageview" ADD CONSTRAINT "pageview_session_id_fkey" FOREIGN KEY ("session_id") REFERENCES "session"("session_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "pageview" ADD CONSTRAINT "pageview_website_id_fkey" FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "session" ADD CONSTRAINT "session_website_id_fkey" FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "website" ADD CONSTRAINT "website_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "account"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;

40
db/postgresql/schema.prisma
View File

@ -8,7 +8,7 @@ datasource db {
}
model account {
user_id Int @id @default(autoincrement())
user_id BigInt @id @default(autoincrement())
username String @unique @db.VarChar(255)
password String @db.VarChar(60)
is_admin Boolean @default(false)
@ -18,14 +18,14 @@ model account {
}
model event {
event_id Int @id @default(autoincrement())
website_id Int
session_id Int
event_id BigInt @id @default(autoincrement())
website_id BigInt
session_id BigInt
created_at DateTime? @default(now()) @db.Timestamptz(6)
url String @db.VarChar(500)
event_name String @db.VarChar(50)
session session @relation(fields: [session_id], references: [session_id], onDelete: Cascade)
website website @relation(fields: [website_id], references: [website_id], onDelete: Cascade)
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
event_data event_data?
@@index([created_at])
@ -34,21 +34,21 @@ model event {
}
model event_data {
event_data_id Int @id @default(autoincrement())
event_id Int @unique
event_data_id BigInt @id @default(autoincrement())
event_id BigInt @unique
event_data Json
event event @relation(fields: [event_id], references: [event_id])
event event @relation(fields: [event_id], references: [event_id])
}
model pageview {
view_id Int @id @default(autoincrement())
website_id Int
session_id Int
view_id BigInt @id @default(autoincrement())
website_id BigInt
session_id BigInt
created_at DateTime? @default(now()) @db.Timestamptz(6)
url String @db.VarChar(500)
referrer String? @db.VarChar(500)
session session @relation(fields: [session_id], references: [session_id], onDelete: Cascade)
website website @relation(fields: [website_id], references: [website_id], onDelete: Cascade)
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
@@index([created_at])
@@index([session_id])
@ -58,9 +58,9 @@ model pageview {
}
model session {
session_id Int @id @default(autoincrement())
session_id BigInt @id @default(autoincrement())
session_uuid String @unique @db.Uuid
website_id Int
website_id BigInt
created_at DateTime? @default(now()) @db.Timestamptz(6)
hostname String? @db.VarChar(100)
browser String? @db.VarChar(20)
@ -69,7 +69,7 @@ model session {
screen String? @db.VarChar(11)
language String? @db.VarChar(35)
country String? @db.Char(2)
website website @relation(fields: [website_id], references: [website_id], onDelete: Cascade)
website website @relation(fields: [website_id], references: [website_id])
pageview pageview[]
event event[]
@ -78,14 +78,14 @@ model session {
}
model website {
website_id Int @id @default(autoincrement())
website_id BigInt @id @default(autoincrement())
website_uuid String @unique @db.Uuid
user_id Int
user_id BigInt
name String @db.VarChar(100)
domain String? @db.VarChar(500)
share_id String? @unique @db.VarChar(64)
created_at DateTime? @default(now()) @db.Timestamptz(6)
account account @relation(fields: [user_id], references: [user_id], onDelete: Cascade)
account account @relation(fields: [user_id], references: [user_id])
pageview pageview[]
session session[]
event event[]

4
lang/de-DE.json
View File

@ -67,7 +67,7 @@
"message.confirm-reset": "Sind Sie sicher, dass Sie die Statistiken von {target} zurücksetzen wollen?",
"message.copied": "In Zwischenablage kopiert!",
"message.delete-warning": "Alle zugehörigen Daten werden ebenfalls gelöscht.",
"message.edit-dashboard": "Edit dashboard",
"message.edit-dashboard": "Dashboard bearbeiten",
"message.failure": "Es ist ein Fehler aufgetreten.",
"message.get-share-url": "Freigabe-URL abrufen",
"message.get-tracking-code": "Erstelle Tracking Kennung",
@ -103,7 +103,7 @@
"metrics.operating-systems": "Betriebssysteme",
"metrics.page-views": "Seitenaufrufe",
"metrics.pages": "Seiten",
"metrics.query-parameters": "Query parameters",
"metrics.query-parameters": "Abfrageparameter",
"metrics.referrers": "Referrer",
"metrics.screens": "Bildschirmauflösungen",
"metrics.unique-visitors": "Eindeutige Besucher",

4
lang/fr-FR.json
View File

@ -27,7 +27,7 @@
"label.enable-share-url": "Activer l'URL de partage",
"label.invalid": "Invalide",
"label.invalid-domain": "Domaine invalide",
"label.language": "Langage",
"label.language": "Langue",
"label.last-days": "{x} derniers jours",
"label.last-hours": "{x} dernières heures",
"label.logged-in-as": "Connecté en tant que {username}",
@ -67,7 +67,7 @@
"message.confirm-reset": "Êtes-vous sûr de vouloir réinistialiser les statistiques de {target} ?",
"message.copied": "Copié !",
"message.delete-warning": "Toutes les données associées seront également supprimées.",
"message.edit-dashboard": "Edit dashboard",
"message.edit-dashboard": "Modifier l'ordre des sites",
"message.failure": "Un problème est survenu.",
"message.get-share-url": "Obtenir l'URL de partage",
"message.get-tracking-code": "Obtenir le code de suivi",

4
lang/zh-CN.json
View File

@ -67,7 +67,7 @@
"message.confirm-reset": "您确定要重置 {target} 的数据吗?",
"message.copied": "复制成功！",
"message.delete-warning": "所有相关数据将会被删除。",
"message.edit-dashboard": "Edit dashboard",
"message.edit-dashboard": "编辑仪表板",
"message.failure": "出现错误。",
"message.get-share-url": "获取共享链接",
"message.get-tracking-code": "获取跟踪代码",
@ -103,7 +103,7 @@
"metrics.operating-systems": "操作系统",
"metrics.page-views": "页面浏览量",
"metrics.pages": "网页",
"metrics.query-parameters": "Query parameters",
"metrics.query-parameters": "查询参数",
"metrics.referrers": "来源域名",
"metrics.screens": "屏幕尺寸",
"metrics.unique-visitors": "独立访客",

1
lib/constants.js
View File

@ -71,6 +71,7 @@ export const RELATIONAL = 'relational';
export const POSTGRESQL = 'postgresql';
export const MYSQL = 'mysql';
export const CLICKHOUSE = 'clickhouse';
export const KAFKA = 'kafka';
export const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',

17
lib/db.js
View File

@ -1,6 +1,8 @@
import { PrismaClient } from '@prisma/client';
import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
import chalk from 'chalk';
import { getKafkaService } from './kafka';
import {
MYSQL,
MYSQL_DATE_FORMATS,
@ -9,6 +11,7 @@ import {
CLICKHOUSE,
RELATIONAL,
FILTER_IGNORED,
KAFKA,
} from 'lib/constants';
import moment from 'moment-timezone';
import { CLICKHOUSE_DATE_FORMATS } from './constants';
@ -87,9 +90,7 @@ export function getDatabase() {
}
export function getAnalyticsDatabase() {
const type =
process.env.ANALYTICS_TYPE ||
(process.env.ANALYTICS_URL && process.env.ANALYTICS_URL.split(':')[0]);
const type = process.env.ANALYTICS_URL && process.env.ANALYTICS_URL.split(':')[0];
if (type === 'postgres') {
return POSTGRESQL;
@ -129,13 +130,13 @@ export function getDateQuery(field, unit, timezone) {
export function getDateQueryClickhouse(field, unit, timezone) {
if (timezone) {
return `date_trunc('${unit}', ${field},'${timezone}')`;
return `date_trunc('${unit}', ${field}, '${timezone}')`;
}
return `date_trunc('${unit}', ${field})`;
}
export function getDateFormatClickhouse(date) {
return `parseDateTimeBestEffort('${date.toUTCString()}')`;
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
}
export function getBetweenDatesClickhouse(field, start_at, end_at) {
@ -219,8 +220,6 @@ export function parseFilters(table, column, filters = {}, params = [], sessionKe
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
filters;
console.log({ table, column, filters, params });
const pageviewFilters = { domain, url, referrer, query };
const sessionFilters = { os, browser, device, country };
const eventFilters = { url: event_url, event_name };
@ -300,6 +299,10 @@ export async function runAnalyticsQuery(queries) {
}
if (db === CLICKHOUSE) {
const kafka = getKafkaService();
if (kafka === KAFKA && queries[KAFKA]) {
return queries[KAFKA]();
}
return queries[CLICKHOUSE]();
}
}

64
lib/kafka.js Normal file
View File

@ -0,0 +1,64 @@
import { Kafka } from 'kafkajs';
import dateFormat from 'dateformat';
export function getKafkaClient() {
if (!process.env.KAFKA_URL) {
return null;
}
const url = new URL(process.env.KAFKA_URL);
const brokers = process.env.KAFKA_BROKER.split(',');
if (url.username.length === 0 && url.password.length === 0) {
return new Kafka({
clientId: 'umami',
brokers: brokers,
connectionTimeout: 3000,
});
} else {
return new Kafka({
clientId: 'umami',
brokers: brokers,
connectionTimeout: 3000,
ssl: true,
sasl: {
mechanism: 'plain',
username: url.username,
password: url.password,
},
});
}
}
const kafka = global.kafka || getKafkaClient();
if (process.env.NODE_ENV !== 'production') {
global.kafka = kafka;
}
export { kafka };
export async function kafkaProducer(params, topic) {
const producer = kafka.producer();
await producer.connect();
await producer.send({
topic,
messages: [
{
key: 'key',
value: JSON.stringify(params),
},
],
});
}
export function getDateFormatKafka(date) {
return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
}
export function getKafkaService() {
const type = process.env.KAFKA_URL && process.env.KAFKA_URL.split(':')[0];
return type;
}

2
package.json
View File

@ -67,6 +67,7 @@
"cross-spawn": "^7.0.3",
"date-fns": "^2.23.0",
"date-fns-tz": "^1.1.4",
"dateformat": "^5.0.3",
"del": "^6.0.0",
"detect-browser": "^5.2.0",
"dotenv": "^10.0.0",
@ -79,6 +80,7 @@
"is-localhost-ip": "^1.4.0",
"isbot": "^3.4.5",
"jose": "2.0.5",
"kafkajs": "^2.1.0",
"maxmind": "^4.3.6",
"moment-timezone": "^0.5.33",
"next": "^12.2.4",

14
pages/api/collect.js
View File

@ -7,6 +7,7 @@ import { getJsonBody, getIpAddress } from 'lib/request';
import { ok, send, badRequest, forbidden } from 'lib/response';
import { createToken } from 'lib/crypto';
import { removeTrailingSlash } from 'lib/url';
import { uuid } from 'lib/crypto';
export default async (req, res) => {
await useCors(req, res);
@ -71,15 +72,24 @@ export default async (req, res) => {
url = removeTrailingSlash(url);
}
const event_uuid = uuid();
if (type === 'pageview') {
await savePageView(website_id, { session_id, session_uuid, url, referrer });
} else if (type === 'event') {
await saveEvent(website_id, { session_id, session_uuid, url, event_name, event_data });
await saveEvent(website_id, {
event_uuid,
session_id,
session_uuid,
url,
event_name,
event_data,
});
} else {
return badRequest(res);
}
const token = await createToken({ website_id, session_id });
const token = await createToken({ website_id, session_id, session_uuid });
return send(res, token);
};

27
queries/admin/account/deleteAccount.js
View File

@ -2,10 +2,27 @@ import { prisma, runQuery } from 'lib/db';
export async function deleteAccount(user_id) {
return runQuery(
prisma.account.delete({
where: {
user_id,
},
}),
prisma.$transaction([
prisma.pageview.deleteMany({
where: { session: { website: { user_id } } },
}),
prisma.event_data.deleteMany({
where: { event: { session: { website: { user_id } } } },
}),
prisma.event.deleteMany({
where: { session: { website: { user_id } } },
}),
prisma.session.deleteMany({
where: { website: { user_id } },
}),
prisma.website.deleteMany({
where: { user_id },
}),
prisma.account.delete({
where: {
user_id,
},
}),
]),
);
}

22
queries/admin/website/deleteWebsite.js
View File

@ -2,10 +2,22 @@ import { prisma, runQuery } from 'lib/db';
export async function deleteWebsite(website_id) {
return runQuery(
prisma.website.delete({
where: {
website_id,
},
}),
prisma.$transaction([
prisma.pageview.deleteMany({
where: { session: { website: { website_id } } },
}),
prisma.event_data.deleteMany({
where: { event: { session: { website: { website_id } } } },
}),
prisma.event.deleteMany({
where: { session: { website: { website_id } } },
}),
prisma.session.deleteMany({
where: { website: { website_id } },
}),
prisma.website.delete({
where: { website_id },
}),
]),
);
}

2
queries/analytics/event/getEventMetrics.js
View File

@ -56,7 +56,7 @@ async function clickhouseQuery(
return rawQueryClickhouse(
`
select
event_value x,
event_name x,
${getDateQueryClickhouse('created_at', unit, timezone)} t,
count(*) y
from event

29
queries/analytics/event/saveEvent.js
View File

@ -1,4 +1,4 @@
import { CLICKHOUSE, RELATIONAL, URL_LENGTH } from 'lib/constants';
import { CLICKHOUSE, RELATIONAL, KAFKA, URL_LENGTH } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
@ -6,11 +6,13 @@ import {
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';
export async function saveEvent(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
@ -37,13 +39,32 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_
);
}
async function clickhouseQuery(website_id, { session_uuid, url, event_name }) {
const params = [website_id, session_uuid, url?.substr(0, URL_LENGTH), event_name?.substr(0, 50)];
async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, event_name }) {
const params = [
website_id,
event_uuid,
session_uuid,
url?.substr(0, URL_LENGTH),
event_name?.substr(0, 50),
];
return rawQueryClickhouse(
`
insert into umami_dev.event (created_at, website_id, session_uuid, url, event_name)
insert into umami.event (created_at, website_id, session_uuid, url, event_name)
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_name }) {
const params = {
event_uuid: event_uuid,
website_id: website_id,
session_uuid: session_uuid,
created_at: getDateFormatKafka(new Date()),
url: url?.substr(0, URL_LENGTH),
event_name: event_name?.substr(0, 50),
};
await kafkaProducer(params, 'event');
}

18
queries/analytics/pageview/savePageView.js
View File

@ -1,4 +1,4 @@
import { CLICKHOUSE, RELATIONAL, URL_LENGTH } from 'lib/constants';
import { CLICKHOUSE, RELATIONAL, KAFKA, URL_LENGTH } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
@ -6,11 +6,13 @@ import {
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';
export async function savePageView(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
@ -37,8 +39,20 @@ async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {
return rawQueryClickhouse(
`
insert into umami_dev.pageview (created_at, website_id, session_uuid, url, referrer)
insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { session_uuid, url, referrer }) {
const params = {
website_id: website_id,
session_uuid: session_uuid,
created_at: getDateFormatKafka(new Date()),
url: url?.substr(0, URL_LENGTH),
referrer: referrer?.substr(0, URL_LENGTH),
};
await kafkaProducer(params, 'pageview');
}

28
queries/analytics/session/createSession.js
View File

@ -1,4 +1,4 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { CLICKHOUSE, RELATIONAL, KAFKA } from 'lib/constants';
import {
getDateFormatClickhouse,
prisma,
@ -6,12 +6,14 @@ import {
runAnalyticsQuery,
runQuery,
} from 'lib/db';
import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';
import { getSessionByUuid } from 'queries';
export async function createSession(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
@ -46,10 +48,32 @@ async function clickhouseQuery(
];
await rawQueryClickhouse(
`insert into umami_dev.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)
`insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)
values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
params,
);
return getSessionByUuid(session_uuid);
}
async function kafkaQuery(
website_id,
{ session_uuid, hostname, browser, os, screen, language, country, device },
) {
const params = {
session_uuid: session_uuid,
website_id: website_id,
created_at: getDateFormatKafka(new Date()),
hostname: hostname,
browser: browser,
os: os,
device: device,
screen: screen,
language: language,
country: country ? country : null,
};
await kafkaProducer(params, 'session');
return getSessionByUuid(session_uuid);
}

820
yarn.lock
View File

File diff suppressed because it is too large Load Diff