Merge pull request #1219 from mikecao/migration

Migration scripts

.gitignore

@@ -11,7 +11,7 @@
 # next.js
 /.next/
 /out/
-/prisma/schema.prisma
+/prisma/
 
 # production
 /build

Dockerfile

@@ -32,10 +32,14 @@ ENV NEXT_TELEMETRY_DISABLED 1
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
+RUN yarn global add prisma
+
 # You only need to copy next.config.js if you are NOT using the default configuration
 COPY --from=builder /app/next.config.js ./
 COPY --from=builder /app/public ./public
 COPY --from=builder /app/package.json ./package.json
+COPY --from=builder /app/prisma/schema.prisma ./prisma/schema.prisma
+COPY --from=builder /app/prisma/migrations ./prisma/migrations
 
 # Automatically leverage output traces to reduce image size
 # https://nextjs.org/docs/advanced-features/output-file-tracing
@@ -48,4 +52,4 @@ EXPOSE 3000
 
 ENV PORT 3000
 
-CMD ["node", "server.js"]
+CMD ["yarn", "start-docker"]

README.md

@@ -16,10 +16,10 @@ See [Running on Railway](https://umami.is/docs/running-on-railway) to get starte
 
 ### Requirements
 
-- A server with Node.js 12 or newer
-- A database (MySQL or Postgresql)
+- A server with Node.js version 12 or newer
+- A database. Umami supports [MySQL](https://www.mysql.com/) and [Postgresql](https://www.postgresql.org/) databases.
 
-### Install Yarn (if needed)
+### Install Yarn
 
 ```
 npm install -g yarn
@@ -33,25 +33,6 @@ cd umami
 yarn install
 ```
 
-### Create database tables
-
-Umami supports [MySQL](https://www.mysql.com/) and [Postgresql](https://www.postgresql.org/).
-Create a database for your Umami installation and install the tables with the included scripts.
-
-For MySQL:
-
-```
-mysql -u username -p databasename < sql/schema.mysql.sql
-```
-
-For Postgresql:
-
-```
-psql -h hostname -U username -d databasename -f sql/schema.postgresql.sql
-```
-
-This will also create a login account with username **admin** and password **umami**.
-
 ### Configure umami
 
 Create an `.env` file with the following
@@ -70,6 +51,14 @@ mysql://username:mypassword@localhost:3306/mydb
 
 The `HASH_SALT` is used to generate unique values for your installation.
 
+This will also create a login account with username **admin** and password **umami**.
+
+### Create database tables
+
+```bash
+yarn update-db
+```
+
 ### Build the application
 
 ```bash
@@ -82,7 +71,7 @@ yarn build
 yarn start
 ```
 
-By default this will launch the application on `http://localhost:3000`. You will need to either 
+By default this will launch the application on `http://localhost:3000`. You will need to either
 [proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) requests from your web server
 or change the [port](https://nextjs.org/docs/api-reference/cli#production) to serve the application directly.
 
@@ -112,6 +101,7 @@ To get the latest features, simply do a pull, install any new dependencies, and
 git pull
 yarn install
 yarn build
+yarn update-db
 ```
 
 To update the Docker image, simply pull the new images and rebuild:

app.json

@@ -1,26 +1,16 @@
 {
-    "name": "Umami",
-    "description": "Umami is a simple, fast, website analytics alternative to Google Analytics.",
-    "keywords": [
-      "analytics",
-      "charts",
-      "statistics",
-      "web-analytics"
-    ],
-    "website": "https://umami.is",
-    "repository": "https://github.com/mikecao/umami",
-    "addons": [
-      "heroku-postgresql"
-    ],
-    "env": {
-      "HASH_SALT": {
-        "description": "Used to generate unique values for your installation",
-        "required": true,
-        "generator": "secret"
-      }
-    },
-    "scripts": {
-      "postdeploy": "psql $DATABASE_URL -f sql/schema.postgresql.sql"
-    },
-    "success_url": "/"
+  "name": "Umami",
+  "description": "Umami is a simple, fast, website analytics alternative to Google Analytics.",
+  "keywords": ["analytics", "charts", "statistics", "web-analytics"],
+  "website": "https://umami.is",
+  "repository": "https://github.com/mikecao/umami",
+  "addons": ["heroku-postgresql"],
+  "env": {
+    "HASH_SALT": {
+      "description": "Used to generate unique values for your installation",
+      "required": true,
+      "generator": "secret"
+    }
+  },
+  "success_url": "/"
 }

db/mysql/migrations/20210320112658_init/migration.sql

@@ -97,3 +97,6 @@ ALTER TABLE `session` ADD FOREIGN KEY (`website_id`) REFERENCES `website`(`websi
 
 -- AddForeignKey
 ALTER TABLE `website` ADD FOREIGN KEY (`user_id`) REFERENCES `account`(`user_id`) ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- CreateAdminUser
+INSERT INTO account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);

db/postgresql/migrations/20210320112717_init/migration.sql

@@ -127,3 +127,6 @@ ALTER TABLE "session" ADD FOREIGN KEY ("website_id") REFERENCES "website"("websi
 
 -- AddForeignKey
 ALTER TABLE "website" ADD FOREIGN KEY ("user_id") REFERENCES "account"("user_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- CreateAdminUser
+INSERT INTO account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);

lib/db.js

@@ -11,23 +11,23 @@ const options = {
 };
 
 function logQuery(e) {
-  if (process.env.LOG_QUERY) {
-    console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
-  }
+  console.log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
 }
 
 let prisma;
 
 if (process.env.NODE_ENV === 'production') {
   prisma = new PrismaClient(options);
-  prisma.$on('query', logQuery);
 } else {
   if (!global.prisma) {
     global.prisma = new PrismaClient(options);
-    global.prisma.$on('query', logQuery);
   }
 
   prisma = global.prisma;
 }
 
+if (process.env.LOG_QUERY) {
+  prisma.$on('query', logQuery);
+}
+
 export default prisma;

next.config.js

@@ -3,7 +3,8 @@ const pkg = require('./package.json');
 
 module.exports = {
   env: {
-    VERSION: pkg.version,
+    version: pkg.version,
+    loginDisabled: process.env.DISABLE_LOGIN,
   },
   basePath: process.env.BASE_PATH,
   experimental: {

package.json

@@ -12,20 +12,21 @@
   "scripts": {
     "dev": "next dev",
     "build": "npm-run-all build-tracker build-geo build-db build-app",
-    "start": "next start",
+    "start": "npm-run-all check-db start-next",
+    "start-docker": "npm-run-all check-db start-server",
     "start-env": "node -r dotenv/config scripts/start-env.js",
+    "start-server": "node server.js",
+    "start-next": "next start",
     "build-app": "next build",
     "build-tracker": "rollup -c rollup.tracker.config.js",
-    "build-db": "npm-run-all copy-db-schema build-db-client",
+    "build-db": "npm-run-all copy-db-files build-db-client",
     "build-lang": "npm-run-all format-lang compile-lang",
     "build-geo": "node scripts/build-geo.js",
-    "build-db-schema": "dotenv prisma introspect",
-    "build-db-client": "dotenv prisma generate",
-    "build-mysql-schema": "dotenv prisma db pull -- --schema=./prisma/schema.mysql.prisma",
-    "build-mysql-client": "dotenv prisma generate -- --schema=./prisma/schema.mysql.prisma",
-    "build-postgresql-schema": "dotenv prisma db pull -- --schema=./prisma/schema.postgresql.prisma",
-    "build-postgresql-client": "dotenv prisma generate -- --schema=./prisma/schema.postgresql.prisma",
-    "copy-db-schema": "node scripts/copy-db-schema.js",
+    "build-db-schema": "prisma db pull",
+    "build-db-client": "prisma generate",
+    "update-db": "prisma migrate deploy",
+    "check-db": "node scripts/check-db.js",
+    "copy-db-files": "node scripts/copy-db-files.js",
     "generate-lang": "npm-run-all extract-lang merge-lang",
     "extract-lang": "formatjs extract \"{pages,components}/**/*.js\" --out-file build/messages.json",
     "merge-lang": "node scripts/merge-lang.js",
@@ -61,6 +62,7 @@
     "classnames": "^2.3.1",
     "colord": "^2.9.2",
     "cors": "^2.8.5",
+    "cross-spawn": "^7.0.3",
     "date-fns": "^2.23.0",
     "date-fns-tz": "^1.1.4",
     "del": "^6.0.0",

pages/_middleware.js

@@ -15,12 +15,6 @@ function customScriptName(req) {
   }
 }
 
-function disableLogin(req) {
-  if (process.env.DISABLE_LOGIN && req.nextUrl.pathname.endsWith('/login')) {
-    return new Response('Login is disabled', { status: 403 });
-  }
-}
-
 function forceSSL(req, res) {
   if (process.env.FORCE_SSL && req.nextUrl.protocol === 'http:') {
     res.headers.set('Strict-Transport-Security', 'max-age=31536000; includeSubDomains');
@@ -30,7 +24,7 @@ function forceSSL(req, res) {
 }
 
 export function middleware(req) {
-  const fns = [customScriptName, disableLogin];
+  const fns = [customScriptName];
 
   for (const fn of fns) {
     const res = fn(req);

pages/login.js

@@ -3,7 +3,7 @@ import Layout from 'components/layout/Layout';
 import LoginForm from 'components/forms/LoginForm';
 
 export default function LoginPage() {
-  if (process.env.DISABLE_LOGIN) {
+  if (process.env.loginDisabled) {
     return null;
   }
 

prisma/mysql/schema.mysql.prisma

@@ -1 +0,0 @@
-../schema.mysql.prisma

prisma/postgresql/schema.postgresql.prisma

@@ -1 +0,0 @@
-../schema.postgresql.prisma

prisma/seed.js

@@ -1,29 +0,0 @@
-const bcrypt = require('bcryptjs');
-const { PrismaClient } = require('@prisma/client');
-const prisma = new PrismaClient();
-const SALT_ROUNDS = 10;
-
-const hashPassword = password => {
-  return bcrypt.hashSync(password, SALT_ROUNDS);
-};
-
-async function main() {
-  await prisma.account.upsert({
-    where: { username: 'admin' },
-    update: {},
-    create: {
-      username: 'admin',
-      password: hashPassword(process.env.ADMIN_PASSWORD || 'umami'),
-      is_admin: true,
-    },
-  });
-}
-
-main()
-  .catch(e => {
-    console.error(e);
-    process.exit(1);
-  })
-  .finally(async () => {
-    await prisma.$disconnect();
-  });

scripts/check-db.js

@@ -0,0 +1,94 @@
+require('dotenv').config();
+const { PrismaClient } = require('@prisma/client');
+const prisma = new PrismaClient();
+const chalk = require('chalk');
+const spawn = require('cross-spawn');
+
+let message = '';
+
+function success(msg) {
+  console.log(chalk.greenBright(`✓ ${msg}`));
+}
+
+async function checkEnv() {
+  if (!process.env.DATABASE_URL) {
+    throw new Error('DATABASE_URL is not defined.');
+  } else {
+    success('DATABASE_URL is defined.');
+  }
+}
+
+async function checkConnection() {
+  try {
+    await prisma.$connect();
+
+    success('Database connection successful.');
+  } catch (e) {
+    throw new Error('Unable to connect to the database.');
+  }
+}
+
+async function checkTables() {
+  try {
+    await prisma.account.findFirst();
+
+    success('Database tables found.');
+  } catch (e) {
+    message = `To update your database, you need to run:\n${chalk.bold.whiteBright(
+      'yarn update-db',
+    )}`;
+    throw new Error('Database tables not found.');
+  }
+}
+
+async function run(cmd, args) {
+  const buffer = [];
+  const proc = spawn(cmd, args);
+
+  return new Promise((resolve, reject) => {
+    proc.stdout.on('data', data => buffer.push(data));
+
+    proc.on('error', () => {
+      reject(new Error('Failed to run Prisma.'));
+    });
+
+    proc.on('exit', () => resolve(buffer.join('')));
+  });
+}
+
+async function checkMigrations() {
+  const output = await run('prisma', ['migrate', 'status']);
+
+  const missingMigrations = output.includes('Following migration have not yet been applied');
+  const notManaged = output.includes('The current database is not managed');
+
+  if (notManaged) {
+    const cmd = output.match(/yarn prisma migrate resolve --applied ".*"/g);
+
+    message = `You need to update your database by running:\n${chalk.bold.whiteBright(cmd[0])}`;
+    throw new Error('Database is out of date.');
+  } else if (missingMigrations) {
+    message = output;
+    throw new Error('Database is out of date.');
+  }
+
+  success('Database is up to date.');
+}
+
+(async () => {
+  let err = false;
+  for (let fn of [checkEnv, checkConnection, checkTables, checkMigrations]) {
+    try {
+      await fn();
+    } catch (e) {
+      console.log(chalk.red(`✗ ${e.message}`));
+      err = true;
+    } finally {
+      await prisma.$disconnect();
+      if (err) {
+        console.log(message);
+        process.exit(1);
+      }
+    }
+  }
+})();

scripts/copy-db-files.js

@@ -1,8 +1,8 @@
 require('dotenv').config();
-const fs = require('fs');
+const fse = require('fs-extra');
 const path = require('path');
 
-function getDatabase() {
+function getDatabaseType() {
   const type =
     process.env.DATABASE_TYPE ||
     (process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]);
@@ -14,7 +14,7 @@ function getDatabase() {
   return type;
 }
 
-const databaseType = getDatabase();
+const databaseType = getDatabaseType();
 
 if (!databaseType || !['mysql', 'postgresql'].includes(databaseType)) {
   throw new Error('Missing or invalid database');
@@ -22,9 +22,9 @@ if (!databaseType || !['mysql', 'postgresql'].includes(databaseType)) {
 
 console.log(`Database type detected: ${databaseType}`);
 
-const src = path.resolve(__dirname, `../prisma/schema.${databaseType}.prisma`);
-const dest = path.resolve(__dirname, '../prisma/schema.prisma');
+const src = path.resolve(__dirname, `../db/${databaseType}`);
+const dest = path.resolve(__dirname, '../prisma');
 
-fs.copyFileSync(src, dest);
+fse.copySync(src, dest);
 
 console.log(`Copied ${src} to ${dest}`);

sql/schema.mysql.sql

@@ -1,80 +0,0 @@
-drop table if exists event;
-drop table if exists pageview;
-drop table if exists session;
-drop table if exists website;
-drop table if exists account;
-
-create table account (
-    user_id int unsigned not null auto_increment primary key,
-    username varchar(255) unique not null,
-    password varchar(60) not null,
-    is_admin bool not null default false,
-    created_at timestamp default current_timestamp,
-    updated_at timestamp default current_timestamp
-) ENGINE=InnoDB COLLATE=utf8_general_ci;
-
-create table website (
-    website_id int unsigned not null auto_increment primary key,
-    website_uuid varchar(36) unique not null,
-    user_id int unsigned not null,
-    name varchar(100) not null,
-    domain varchar(500),
-    share_id varchar(64) unique,
-    created_at timestamp default current_timestamp,
-    foreign key (user_id) references account(user_id) on delete cascade
-) ENGINE=InnoDB COLLATE=utf8_general_ci;
-
-create table session (
-    session_id int unsigned not null auto_increment primary key,
-    session_uuid varchar(36) unique not null,
-    website_id int unsigned not null references website(website_id) on delete cascade,
-    created_at timestamp default current_timestamp,
-    hostname varchar(100),
-    browser varchar(20),
-    os varchar(20),
-    device varchar(20),
-    screen varchar(11),
-    language varchar(35),
-    country char(2),
-    foreign key (website_id) references website(website_id) on delete cascade
-) ENGINE=InnoDB COLLATE=utf8_general_ci;
-
-create table pageview (
-    view_id int unsigned not null auto_increment primary key,
-    website_id int unsigned not null,
-    session_id int unsigned not null,
-    created_at timestamp default current_timestamp,
-    url varchar(500) not null,
-    referrer varchar(500),
-    foreign key (website_id) references website(website_id) on delete cascade,
-    foreign key (session_id) references session(session_id) on delete cascade
-) ENGINE=InnoDB COLLATE=utf8_general_ci;
-
-create table event (
-    event_id int unsigned not null auto_increment primary key,
-    website_id int unsigned not null,
-    session_id int unsigned not null,
-    created_at timestamp default current_timestamp,
-    url varchar(500) not null,
-    event_type varchar(50) not null,
-    event_value varchar(50) not null,
-    foreign key (website_id) references website(website_id) on delete cascade,
-    foreign key (session_id) references session(session_id) on delete cascade
-) ENGINE=InnoDB COLLATE=utf8_general_ci;
-
-create index website_user_id_idx on website(user_id);
-
-create index session_created_at_idx on session(created_at);
-create index session_website_id_idx on session(website_id);
-
-create index pageview_created_at_idx on pageview(created_at);
-create index pageview_website_id_idx on pageview(website_id);
-create index pageview_session_id_idx on pageview(session_id);
-create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
-create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
-
-create index event_created_at_idx on event(created_at);
-create index event_website_id_idx on event(website_id);
-create index event_session_id_idx on event(session_id);
-
-insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);

sql/schema.postgresql.sql

@@ -1,74 +0,0 @@
-drop table if exists event;
-drop table if exists pageview;
-drop table if exists session;
-drop table if exists website;
-drop table if exists account;
-
-create table account (
-    user_id serial primary key,
-    username varchar(255) unique not null,
-    password varchar(60) not null,
-    is_admin bool not null default false,
-    created_at timestamp with time zone default current_timestamp,
-    updated_at timestamp with time zone default current_timestamp
-);
-
-create table website (
-    website_id serial primary key,
-    website_uuid uuid unique not null,
-    user_id int not null references account(user_id) on delete cascade,
-    name varchar(100) not null,
-    domain varchar(500),
-    share_id varchar(64) unique,
-    created_at timestamp with time zone default current_timestamp
-);
-
-create table session (
-    session_id serial primary key,
-    session_uuid uuid unique not null,
-    website_id int not null references website(website_id) on delete cascade,
-    created_at timestamp with time zone default current_timestamp,
-    hostname varchar(100),
-    browser varchar(20),
-    os varchar(20),
-    device varchar(20),
-    screen varchar(11),
-    language varchar(35),
-    country char(2)
-);
-
-create table pageview (
-    view_id serial primary key,
-    website_id int not null references website(website_id) on delete cascade,
-    session_id int not null references session(session_id) on delete cascade,
-    created_at timestamp with time zone default current_timestamp,
-    url varchar(500) not null,
-    referrer varchar(500)
-);
-
-create table event (
-    event_id serial primary key,
-    website_id int not null references website(website_id) on delete cascade,
-    session_id int not null references session(session_id) on delete cascade,
-    created_at timestamp with time zone default current_timestamp,
-    url varchar(500) not null,
-    event_type varchar(50) not null,
-    event_value varchar(50) not null
-);
-
-create index website_user_id_idx on website(user_id);
-
-create index session_created_at_idx on session(created_at);
-create index session_website_id_idx on session(website_id);
-
-create index pageview_created_at_idx on pageview(created_at);
-create index pageview_website_id_idx on pageview(website_id);
-create index pageview_session_id_idx on pageview(session_id);
-create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
-create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
-
-create index event_created_at_idx on event(created_at);
-create index event_website_id_idx on event(website_id);
-create index event_session_id_idx on event(session_id);
-
-insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);

store/version.js

@@ -7,7 +7,7 @@ import { getItem } from 'lib/web';
 const REPO_URL = 'https://api.umami.is/v1/updates';
 
 const initialState = {
-  current: process.env.VERSION,
+  current: process.env.version,
   latest: null,
   hasUpdate: false,
 };

yarn.lock

@@ -2246,25 +2246,10 @@ camelcase@^6.2.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
-caniuse-lite@^1.0.30001283:
-  version "1.0.30001314"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001314.tgz#65c7f9fb7e4594fca0a333bec1d8939662377596"
-  integrity sha512-0zaSO+TnCHtHJIbpLroX7nsD+vYuOVjl3uzFbJO1wMVbuveJA0RK2WcQA9ZUIOiO0/ArMiMgHJLxfEZhQiC0kw==
-
-caniuse-lite@^1.0.30001313:
-  version "1.0.30001320"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001320.tgz#8397391bec389b8ccce328636499b7284ee13285"
-  integrity sha512-MWPzG54AGdo3nWx7zHZTefseM5Y1ccM7hlQKHRqJkPozUaw3hNbBTMmLn16GG2FUzjR13Cr3NPfhIieX5PzXDA==
-
-caniuse-lite@^1.0.30001317, caniuse-lite@^1.0.30001332, caniuse-lite@^1.0.30001335:
-  version "1.0.30001344"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001344.tgz#8a1e7fdc4db9c2ec79a05e9fd68eb93a761888bb"
-  integrity sha512-0ZFjnlCaXNOAYcV7i+TtdKBp0L/3XEU2MF/x6Du1lrh+SRX4IfzIVL4HNJg5pB2PmFb8rszIGyOvsZnqqRoc2g==
-
-caniuse-lite@^1.0.30001349:
-  version "1.0.30001352"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001352.tgz#cc6f5da3f983979ad1e2cdbae0505dccaa7c6a12"
-  integrity sha512-GUgH8w6YergqPQDGWhJGt8GDRnY0L/iJVQcU3eJ46GYf52R8tk0Wxp0PymuFVZboJYXGiCqwozAYZNRjVj6IcA==
+caniuse-lite@^1.0.30001283, caniuse-lite@^1.0.30001313, caniuse-lite@^1.0.30001317, caniuse-lite@^1.0.30001332, caniuse-lite@^1.0.30001335, caniuse-lite@^1.0.30001349:
+  version "1.0.30001356"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001356.tgz"
+  integrity sha512-/30854bktMLhxtjieIxsrJBfs2gTM1pel6MXKF3K+RdIVJZcsn2A2QdhsuR4/p9+R204fZw0zCBBhktX8xWuyQ==
 
 chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2:
   version "2.4.2"