From c379ae7b7a8619189365f3cb96b801173eb513c5 Mon Sep 17 00:00:00 2001 From: poma Date: Fri, 13 Dec 2019 19:32:09 +0700 Subject: [PATCH] fix timeouts under high load in nginx and express, fix redis persistence --- .dockerignore | 3 +- docker-compose.yml | 58 +++++++--- package-lock.json | 258 ++++++++++++++++++++++++++++++++++++++++++++- src/index.js | 3 +- 4 files changed, 302 insertions(+), 20 deletions(-) diff --git a/.dockerignore b/.dockerignore index 1dcef2d..ded222f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ node_modules -.env \ No newline at end of file +.env +.git \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 10a7dc6..c19febf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,5 @@ version: '2' -volumes: - redis: - services: kovan: image: tornadocash/relayer @@ -13,32 +10,61 @@ services: NET_ID: 42 RPC_URL: https://kovan.infura.io # without 0x prefix - PRIVATE_KEY: + PRIVATE_KEY: # 2.5 means 2.5% RELAYER_FEE: 2.5 - REDIS_URL: redis://redis:6379 + REDIS_URL: redis + nginx_proxy_read_timeout: 600 depends_on: - redis + + redis: + image: redis + restart: always + command: [redis-server, --appendonly, 'yes'] + volumes: + - redis:/data + nginx: - image: jwilder/nginx-proxy + image: nginx:alpine + container_name: nginx restart: always ports: - 80:80 - 443:443 volumes: - - /etc/nginx/certs - - /etc/nginx/vhost.d - - /usr/share/nginx/html - - /var/run/docker.sock:/tmp/docker.sock:ro - letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion + - conf:/etc/nginx/conf.d + - vhost:/etc/nginx/vhost.d + - html:/usr/share/nginx/html + - certs:/etc/nginx/certs + labels: + co.elastic.logs/module: nginx + co.elastic.logs/fileset.stdout: access + co.elastic.logs/fileset.stderr: error + + dockergen: + image: poma/docker-gen + container_name: dockergen restart: always + command: -notify-sighup nginx -watch /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf volumes_from: - nginx volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - redis: - image: redis + + letsencrypt: + image: jrcs/letsencrypt-nginx-proxy-companion + container_name: letsencrypt restart: always - volumes: - - redis:/data \ No newline at end of file + environment: + NGINX_DOCKER_GEN_CONTAINER: dockergen + volumes_from: + - nginx + - dockergen + +volumes: + conf: + vhost: + html: + certs: + redis: \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c84cd3c..1c151bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -374,6 +374,33 @@ "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=" }, + "bull": { + "version": "3.12.1", + "resolved": "https://registry.npmjs.org/bull/-/bull-3.12.1.tgz", + "integrity": "sha512-X3bSP7gTqPXLYVSyUtQuTOqZuU0GwVbV304Et84Z8bxYP60R1VD3FUOLsESVRA9LIUEOWVH3hE8MFqlszmO0Gw==", + "requires": { + "cron-parser": "^2.13.0", + "debuglog": "^1.0.0", + "get-port": "^5.0.0", + "ioredis": "^4.14.1", + "lodash": "^4.17.15", + "p-timeout": "^3.1.0", + "promise.prototype.finally": "^3.1.1", + "semver": "^6.3.0", + "util.promisify": "^1.0.0", + "uuid": "^3.3.3" + }, + "dependencies": { + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "requires": { + "p-finally": "^1.0.0" + } + } + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -473,6 +500,16 @@ "mimic-response": "^1.0.0" } }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, + "coingecko-api": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/coingecko-api/-/coingecko-api-1.0.8.tgz", + "integrity": "sha512-sKnv8TVFDMOfkUtbR4m77wg4AxQ/k/y7TOJKYPd4waF8b83Mb2WsroAQoMgFGJfCbgZPJ2PWYuF74cxiITMuZg==" + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -586,6 +623,15 @@ "sha.js": "^2.4.8" } }, + "cron-parser": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.13.0.tgz", + "integrity": "sha512-UWeIpnRb0eyoWPVk+pD3TDpNx3KCFQeezO224oJIkktBrcW6RoAPOx5zIKprZGfk6vcYSmA8yQXItejSaDBhbQ==", + "requires": { + "is-nan": "^1.2.1", + "moment-timezone": "^0.5.25" + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -650,6 +696,11 @@ "ms": "2.0.0" } }, + "debuglog": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", + "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=" + }, "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", @@ -768,6 +819,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1484,6 +1540,21 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "get-port": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.0.0.tgz", + "integrity": "sha512-imzMU0FjsZqNa6BqOjbbW6w5BivHIuQKopjpPqcnx0AVHJQKCxK1O+Ab3OrVXhrekqfVMjwA9ZYu062R+KcIsQ==", + "requires": { + "type-fest": "^0.3.0" + }, + "dependencies": { + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==" + } + } + }, "get-stream": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", @@ -1758,6 +1829,37 @@ "through": "^2.3.6" } }, + "ioredis": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.14.1.tgz", + "integrity": "sha512-94W+X//GHM+1GJvDk6JPc+8qlM7Dul+9K+lg3/aHixPN7ZGkW6qlvX0DG6At9hWtH2v3B32myfZqWoANUJYGJA==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "redis-commands": "1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ipaddr.js": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", @@ -1804,6 +1906,14 @@ "resolved": "https://registry.npmjs.org/is-hex-prefixed/-/is-hex-prefixed-1.0.0.tgz", "integrity": "sha1-fY035q135dEnFIkTxXPggtd39VQ=" }, + "is-nan": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.2.1.tgz", + "integrity": "sha1-n69ltvttskt/XAYoR16nH5iEAeI=", + "requires": { + "define-properties": "^1.1.1" + } + }, "is-natural-number": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/is-natural-number/-/is-natural-number-4.0.1.tgz", @@ -1993,8 +2103,17 @@ "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, "lowercase-keys": { "version": "1.0.1", @@ -2149,6 +2268,19 @@ "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-4.10.3.tgz", "integrity": "sha512-bcukePBvuA3qovmq0Qtqu9+1APCIGkFHnsozrPIVromt5XFGGgkQSfaN0H6RI8gStHkO/hRgimvS3tooNes4pQ==" }, + "moment": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" + }, + "moment-timezone": { + "version": "0.5.27", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.27.tgz", + "integrity": "sha512-EIKQs7h5sAsjhPCqN6ggx6cEbs94GK050254TIJySD1bzoM5JTYDwAU1IoVOeTOL6Gm27kYJ51/uuvq1kIlrbw==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -2238,6 +2370,56 @@ "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "requires": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" + } + }, + "object.getownpropertydescriptors": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz", + "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + }, + "dependencies": { + "es-abstract": { + "version": "1.17.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.0-next.1.tgz", + "integrity": "sha512-7MmGr03N7Rnuid6+wyhD9sHNE2n4tFSwExnU2lQl3lIo2ShXWGePY80zYaoMOmILWv57H0amMjZGHNzzGG70Rw==", + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.1.4", + "is-regex": "^1.0.4", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimleft": "^2.1.0", + "string.prototype.trimright": "^2.1.0" + } + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" + }, + "object-inspect": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", + "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==" + } + } + }, "oboe": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/oboe/-/oboe-2.1.4.tgz", @@ -2429,6 +2611,46 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "promise.prototype.finally": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz", + "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.0", + "function-bind": "^1.1.1" + }, + "dependencies": { + "es-abstract": { + "version": "1.17.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.0-next.1.tgz", + "integrity": "sha512-7MmGr03N7Rnuid6+wyhD9sHNE2n4tFSwExnU2lQl3lIo2ShXWGePY80zYaoMOmILWv57H0amMjZGHNzzGG70Rw==", + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.1.4", + "is-regex": "^1.0.4", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimleft": "^2.1.0", + "string.prototype.trimright": "^2.1.0" + } + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" + }, + "object-inspect": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", + "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==" + } + } + }, "proxy-addr": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", @@ -2532,6 +2754,24 @@ "util-deprecate": "~1.0.1" } }, + "redis-commands": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "regexpp": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", @@ -2857,6 +3097,11 @@ "tweetnacl": "~0.14.0" } }, + "standard-as-callback": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", + "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -3261,6 +3506,15 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "util.promisify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz", + "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==", + "requires": { + "define-properties": "^1.1.2", + "object.getownpropertydescriptors": "^2.0.3" + } + }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/src/index.js b/src/index.js index 4912476..22f47cb 100644 --- a/src/index.js +++ b/src/index.js @@ -36,7 +36,8 @@ app.get('/status', async function (req, res) { app.post('/relay', relayController) -app.listen(port || 8000) +let server = app.listen(port || 8000) +server.setTimeout(600000) console.log('Gas price oracle started.') fetcher.fetchGasPrice() fetcher.fetchPrices()