diff --git a/Dockerfile b/Dockerfile index 37e09bff..f92e0647 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,15 +8,17 @@ RUN apt-get -qq update \ && pip install --no-cache-dir . \ && apt-get autoremove \ && apt-get clean + VOLUME ["/data", "/certs"] -WORKDIR /data + +ENV PYTHONUNBUFFERED 0 ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0 ENV BIGCHAINDB_WSSERVER_SCHEME ws - -ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws ENV BIGCHAINDB_WSSERVER_ADVERTISED_PORT 9985 +ENV BIGCHAINDB_START_TENDERMINT 0 ENTRYPOINT ["bigchaindb"] CMD ["start"] diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index ed79b122..be7da0d6 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -27,13 +27,11 @@ spec: configMapKeyRef: name: vars key: mongodb-backend-port - - name: BIGCHAINDB_DATABASE_REPLICASET + - name: BIGCHAINDB_DATABASE_BACKEND valueFrom: configMapKeyRef: - name: vars - key: mongodb-replicaset-name - - name: BIGCHAINDB_DATABASE_BACKEND - value: mongodb + name: bdb-config + key: bdb-db-backend - name: BIGCHAINDB_DATABASE_NAME valueFrom: configMapKeyRef: @@ -69,13 +67,6 @@ spec: configMapKeyRef: name: vars key: bigchaindb-wsserver-advertised-scheme - - name: BIGCHAINDB_KEYPAIR_PUBLIC - valueFrom: - configMapKeyRef: - name: bdb-config - key: bdb-public-key - - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: "" - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY valueFrom: configMapKeyRef: @@ -111,12 +102,23 @@ spec: configMapKeyRef: name: bdb-config key: bdb-user - # The following env var is not required for the bootstrap/first node - #- name: BIGCHAINDB_KEYRING - # valueFrom: - # configMapKeyRef: - # name: bdb-config - # key: bdb-keyring + - name: BIGCHAINDB_START_TENDERMINT + value: "0" + - name: TENDERMINT_HOST + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-instance-name + - name: TENDERMINT_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-rpc-port + command: + - bash + - "-c" + - | + bigchaindb -l DEBUG start ports: - containerPort: "" protocol: TCP @@ -124,6 +126,9 @@ spec: - containerPort: "" protocol: TCP name: bdb-ws-port + - containerPort: "" + protocol: TCP + name: tm-abci-port volumeMounts: - name: bdb-certs mountPath: /etc/bigchaindb/ssl/ diff --git a/k8s/bigchaindb/bigchaindb-svc.yaml b/k8s/bigchaindb/bigchaindb-svc.yaml index a745e8d9..c5fef92d 100644 --- a/k8s/bigchaindb/bigchaindb-svc.yaml +++ b/k8s/bigchaindb/bigchaindb-svc.yaml @@ -17,5 +17,9 @@ spec: targetPort: "" name: bdb-ws-port protocol: TCP + - port: "" + targetPort: "" + name: tm-abci-port + protocol: TCP type: ClusterIP clusterIP: None diff --git a/k8s/configuration/config-map.yaml b/k8s/configuration/config-map.yaml index fafe365c..d823eb62 100644 --- a/k8s/configuration/config-map.yaml +++ b/k8s/configuration/config-map.yaml @@ -41,10 +41,6 @@ data: # in this cluster. mdb-mon-instance-name: "" - # mdb-bak-instance-name is the name of the MongoDB Backup Agent instance - # in this cluster. - mdb-bak-instance-name: "" - # ngx-mdb-instance-name is the FQDN of the MongoDB instance in this # Kubernetes cluster. ngx-mdb-instance-name: ".default.svc.cluster.local" @@ -57,11 +53,6 @@ data: # Kubernetes cluster. ngx-bdb-instance-name: ".default.svc.cluster.local" - # mongodb-frontend-port is the port number on which external clients can - # access MongoDB. This needs to be restricted to only other MongoDB instances - # by enabling an authentication mechanism on MongoDB. - mongodb-frontend-port: "27017" - # mongodb-backend-port is the port on which MongoDB is actually # available/listening for requests. mongodb-backend-port: "27017" @@ -88,9 +79,6 @@ data: bigchaindb-ws-port: "9985" bigchaindb-ws-interface: "0.0.0.0" - # mongodb-replicaset-name is the MongoDB replica set name - mongodb-replicaset-name: "bigchain-rs" - # bigchaindb-database-name is the database collection used by BigchainDB with # the MongoDB backend. bigchaindb-database-name: "bigchain" @@ -111,17 +99,9 @@ metadata: name: bdb-config namespace: default data: - # Colon-separated list of all *other* nodes' BigchainDB public keys. - bdb-keyring: "<':' separated list of public keys>" - # BigchainDB instance authentication user name bdb-user: "" - # BigchainDB public key of *this* node. - # Generated using bigchaindb_driver in the docs - # Example: "EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce" - bdb-public-key: "" - # bigchaindb-backlog-reassign-delay is the number of seconds a transaction # can remain in the backlog before being reassigned. bigchaindb-backlog-reassign-delay: "120" @@ -138,3 +118,51 @@ data: # bigchaindb-log-level is the log level used to log to the console. bigchaindb-log-level: "debug" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tendermint-config + namespace: default +data: + # tm-seeds is the list of all the peers in the network. + tm-seeds: "<',' separated list of all tendermint nodes in the network>" + + # tm-validators is the list of all validators in the network. + tm-validators: "<',' separated list of all validators in the network>" + + # tm-validator-power is the validators voting power, make sure the order and + # the number of nodes in tm-validator-power and tm-validators is the same. + tm-validator-power: "<',' separated list of validator power of each node in the network>" + + # tm-genesis-time is the official time of blockchain start. + # example: 0001-01-01T00:00:00Z + tm-genesis-time: "" + + # tm-chain-id is the ID of the blockchain. Must be unique for every blockchain. + # example: test-chain-KPI1Ud + tm-chain-id: "" + + # tendermint-instance-name is the name of the Tendermint instance + # in the cluster + tm-instance-name: "" + + # ngx-tm-instance-name is the FQDN of the tendermint instance in this cluster + ngx-tm-instance-name: ".default.svc.cluster.local" + + # tm-abci-port is used by Tendermint Core for ABCI traffic. BigchainDB nodes + # use that internally. + tm-abci-port: "46658" + + # tm-p2p-port is used by Tendermint Core to communicate with + # other peers in the network. This port is accessible publicly. + tm-p2p-port: "46656" + + # tm-rpc-port is used by Tendermint Core to rpc. BigchainDB nodes + # use this port internally. + tm-rpc-port: "46657" + + # tm-pub-key-access is the port number used to host/publish the + # public key of the tendemrint node in this cluster. + tm-pub-key-access: "9986" diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml index ad3ca7d7..b35e6548 100644 --- a/k8s/configuration/secret.yaml +++ b/k8s/configuration/secret.yaml @@ -22,17 +22,6 @@ data: --- apiVersion: v1 kind: Secret -metadata: - name: bdb-private-key - namespace: default -type: Opaque -data: - # Base64-encoded BigchainDB private key of *this* node - # Generated using bigchaindb_driver in the docs - private.key: "" ---- -apiVersion: v1 -kind: Secret metadata: name: mdb-certs namespace: default @@ -53,16 +42,6 @@ data: --- apiVersion: v1 kind: Secret -metadata: - name: mdb-bak-certs - namespace: default -type: Opaque -data: - # Base64-encoded, concatenated certificate and private key - mdb-bak-instance.pem: "" ---- -apiVersion: v1 -kind: Secret metadata: name: bdb-certs namespace: default diff --git a/k8s/mongodb-backup-agent/container/Dockerfile b/k8s/mongodb-backup-agent/container/Dockerfile deleted file mode 100644 index 7bd96dec..00000000 --- a/k8s/mongodb-backup-agent/container/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:xenial -LABEL maintainer "dev@bigchaindb.com" -ARG DEBIAN_FRONTEND=noninteractive -ARG DEB_FILE=mongodb-mms-backup-agent_latest_amd64.ubuntu1604.deb -ARG FILE_URL="https://cloud.mongodb.com/download/agent/backup/"$DEB_FILE -WORKDIR / -RUN apt update \ - && apt -y upgrade \ - && apt -y install --no-install-recommends \ - curl \ - ca-certificates \ - logrotate \ - libsasl2-2 \ - && curl -OL $FILE_URL \ - && dpkg -i $DEB_FILE \ - && rm -f $DEB_FILE \ - && apt -y purge curl \ - && apt -y autoremove \ - && apt clean -COPY mongodb_backup_agent_entrypoint.bash / -RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/ -VOLUME /etc/mongod/ssl /etc/mongod/ca -USER mongodb-mms-agent -ENTRYPOINT ["/mongodb_backup_agent_entrypoint.bash"] diff --git a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash deleted file mode 100755 index 91a0b22c..00000000 --- a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -docker build -t bigchaindb/mongodb-backup-agent:3.5 . - -docker push bigchaindb/mongodb-backup-agent:3.5 diff --git a/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash b/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash deleted file mode 100755 index 4ac4d75e..00000000 --- a/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -MONGODB_BACKUP_CONF_FILE=/etc/mongodb-mms/backup-agent.config - -mms_api_keyfile_path=`printenv MMS_API_KEYFILE_PATH` -mms_groupid_keyfile_path=`printenv MMS_GROUPID_KEYFILE_PATH` -ca_crt_path=`printenv CA_CRT_PATH` -backup_pem_path=`printenv BACKUP_PEM_PATH` - -if [[ -z "${mms_api_keyfile_path:?MMS_API_KEYFILE_PATH not specified. Exiting!}" || \ - -z "${ca_crt_path:?CA_CRT_PATH not specified. Exiting!}" || \ - -z "${backup_pem_path:?BACKUP_PEM_PATH not specified. Exiting!}" || \ - -z "${mms_groupid_keyfile_path:?MMS_GROUPID_KEYFILE_PATH not specified. Exiting!}" ]]; then - exit 1 -else - echo MMS_API_KEYFILE_PATH="$mms_api_keyfile_path" - echo MMS_GROUPID_KEYFILE_PATH="$mms_groupid_keyfile_path" - echo CA_CRT_PATH="$ca_crt_path" - echo BACKUP_PEM_PATH="$backup_pem_path" -fi - -sed -i '/mmsApiKey/d' ${MONGODB_BACKUP_CONF_FILE} -sed -i '/mmsGroupId/d' ${MONGODB_BACKUP_CONF_FILE} -sed -i '/mothership/d' ${MONGODB_BACKUP_CONF_FILE} - -# Get the api key from file -mms_api_key=`cat ${mms_api_keyfile_path}` -mms_groupid_key=`cat ${mms_groupid_keyfile_path}` - -echo "mmsApiKey="${mms_api_key} >> ${MONGODB_BACKUP_CONF_FILE} -echo "mmsGroupId="${mms_groupid_key} >> ${MONGODB_BACKUP_CONF_FILE} -echo "mothership=api-backup.eu-west-1.mongodb.com" >> ${MONGODB_BACKUP_CONF_FILE} - -# Append SSL settings to the config file -echo "useSslForAllConnections=true" >> ${MONGODB_BACKUP_CONF_FILE} -echo "sslRequireValidServerCertificates=true" >> ${MONGODB_BACKUP_CONF_FILE} -echo "sslTrustedServerCertificates="${ca_crt_path} >> ${MONGODB_BACKUP_CONF_FILE} -echo "sslClientCertificate="${backup_pem_path} >> ${MONGODB_BACKUP_CONF_FILE} -echo "#sslClientCertificatePassword=" >> ${MONGODB_BACKUP_CONF_FILE} - -echo "INFO: starting mdb backup..." -exec mongodb-mms-backup-agent -c $MONGODB_BACKUP_CONF_FILE diff --git a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml deleted file mode 100644 index 4aeb66bf..00000000 --- a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml +++ /dev/null @@ -1,65 +0,0 @@ -############################################################ -# This config file defines a k8s Deployment for the # -# bigchaindb/mongodb-backup-agent Docker image # -# # -# It connects to a MongoDB instance in a separate pod, # -# all remote MongoDB instances in the cluster, # -# and also to MongoDB Cloud Manager (an external service). # -# Notes: # -# MongoDB agents connect to Cloud Manager on port 443. # -############################################################ - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mdb-bak-instance-0-dep -spec: - replicas: 1 - template: - metadata: - name: mdb-bak-instance-0-dep - labels: - app: mdb-bak-instance-0-dep - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: mdb-backup - image: bigchaindb/mongodb-backup-agent:3.5 - imagePullPolicy: IfNotPresent - env: - - name: MMS_API_KEYFILE_PATH - value: /etc/mongod/cloud/agent-api-key - - name: MMS_GROUPID_KEYFILE_PATH - value: /etc/mongod/cloud/group-id - - name: CA_CRT_PATH - value: /etc/mongod/ca/ca.pem - - name: BACKUP_PEM_PATH - value: /etc/mongod/ssl/mdb-bak-instance.pem - resources: - limits: - cpu: 200m - memory: 768Mi - volumeMounts: - - name: mdb-bak-certs - mountPath: /etc/mongod/ssl/ - readOnly: true - - name: ca-auth - mountPath: /etc/mongod/ca/ - readOnly: true - - name: cloud-manager-credentials - mountPath: /etc/mongod/cloud/ - readOnly: true - restartPolicy: Always - volumes: - - name: mdb-bak-certs - secret: - secretName: mdb-bak-certs - defaultMode: 0404 - - name: ca-auth - secret: - secretName: ca-auth - defaultMode: 0404 - - name: cloud-manager-credentials - secret: - secretName: cloud-manager-credentials - defaultMode: 0404 diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile index cf4aad42..1da9dc26 100644 --- a/k8s/mongodb/container/Dockerfile +++ b/k8s/mongodb/container/Dockerfile @@ -5,8 +5,8 @@ RUN apt-get update \ && apt-get -y upgrade \ && apt-get autoremove \ && apt-get clean -COPY mongod.conf.template /etc/mongod.conf -COPY mongod_entrypoint.bash / +COPY mongod.conf.tmt.template /etc/mongod.conf +COPY mongod_tmt_entrypoint.bash / VOLUME /data/db /data/configdb /etc/mongod/ssl /etc/mongod/ca EXPOSE 27017 -ENTRYPOINT ["/mongod_entrypoint.bash"] +ENTRYPOINT ["/mongod_tmt_entrypoint.bash"] diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md index 4cec6250..a2b0d22e 100644 --- a/k8s/mongodb/container/README.md +++ b/k8s/mongodb/container/README.md @@ -1,14 +1,5 @@ ## Custom MongoDB container for BigchainDB Backend -### Need - -* MongoDB needs the hostname provided in the `rs.initiate()` command to be - resolvable through the hosts file locally. -* In the future, with the introduction of TLS for inter-cluster MongoDB - communications, we will need a way to specify detailed configuration. -* We also need a way to overwrite certain parameters to suit our use case. - - ### Step 1: Build and Push the Latest Container Use the `docker_build_and_push.bash` script to build the latest docker image and upload it to Docker Hub. @@ -27,84 +18,11 @@ docker run \ --volume=:/data/db \ --volume=:/data/configdb \ --volume=:/mongo-ssl:ro \ - bigchaindb/mongodb:3.0 \ + bigchaindb/mongodb: \ --mongodb-port \ --mongodb-key-file-path /mongo-ssl/.pem \ - --mongodb-key-file-password \ --mongodb-ca-file-path /mongo-ssl/.crt \ --mongodb-crl-file-path /mongo-ssl/.pem \ - --replica-set-name \ --mongodb-fqdn \ --mongodb-ip ``` - -#### Step 3: Initialize the Replica Set - -Login to one of the MongoDB containers, say mdb1: - -`docker exec -it mongodb bash` - -Since we need TLS certificates to use the mongo shell now, copy them using: - -``` -docker cp bdb-instance-0.pem mongodb:/ -docker cp ca.crt mongodb:/ -``` - -Start the `mongo` shell: - -``` -mongo --host mdb1-fqdn --port mdb1-port --verbose --ssl \ - --sslCAFile /ca.crt \ - --sslPEMKeyFile /bdb-instance-0.pem \ - --sslPEMKeyPassword password -``` - -Run the rs.initiate() command: -``` -rs.initiate({ - _id : ":" - } ] -}) -``` - -For example: - -``` -rs.initiate({ _id : "test-repl-set", members: [ { _id : 0, host : -"mdb-instance-0.westeurope.cloudapp.azure.com:27017" } ] }) -``` - -You should also see changes in the mongo shell prompt from `>` to -`test-repl-set:OTHER>` to `test-repl-set:SECONDARY>` to finally -`test-repl-set:PRIMARY>`. -If this instance is not the primary, you can use the `rs.status()` command to -find out who is the primary. - - -#### Step 4: Add members to the Replica Set - -We can only add members to a replica set from the PRIMARY instance. -Login to the PRIMARY and open a `mongo` shell. - -Run the rs.add() command with the ip and port number of the other -containers/instances: -``` -rs.add(":") -``` - -For example: - -Add mdb2 to replica set from mdb1: -``` -rs.add("bdb-cluster-1.northeurope.cloudapp.azure.com:27017") -``` - -Add mdb3 to replica set from mdb1: -``` -rs.add("bdb-cluster-2.northeurope.cloudapp.azure.com:27017") -``` - diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash index bb48dd2f..c0e7dcf6 100755 --- a/k8s/mongodb/container/docker_build_and_push.bash +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -1,5 +1,4 @@ #!/bin/bash -docker build -t bigchaindb/mongodb:3.3 . - -docker push bigchaindb/mongodb:3.3 +docker build -t bigchaindb/localmongodb:1.0 . +docker push bigchaindb/localmongodb:1.0 diff --git a/k8s/mongodb/container/mongod.conf.template b/k8s/mongodb/container/mongod.conf.template index d8ae1bce..c0a2d288 100644 --- a/k8s/mongodb/container/mongod.conf.template +++ b/k8s/mongodb/container/mongod.conf.template @@ -25,8 +25,6 @@ systemLog: verbosity: 0 query: verbosity: 0 - replication: - verbosity: 0 sharding: verbosity: 0 storage: @@ -95,7 +93,3 @@ storage: operationProfiling: mode: slowOp slowOpThresholdMs: 100 - -replication: - replSetName: REPLICA_SET_NAME - enableMajorityReadConcern: true diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index 6a6fe309..ad6aad43 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -3,10 +3,8 @@ set -euo pipefail MONGODB_PORT="" MONGODB_KEY_FILE_PATH="" -#MONGODB_KEY_FILE_PASSWORD="" MONGODB_CA_FILE_PATH="" MONGODB_CRL_FILE_PATH="" -REPLICA_SET_NAME="" MONGODB_FQDN="" MONGODB_IP="" @@ -21,11 +19,6 @@ while [[ $# -gt 1 ]]; do MONGODB_KEY_FILE_PATH="$2" shift ;; - --mongodb-key-file-password) - # TODO(Krish) move this to a mapped file later - MONGODB_KEY_FILE_PASSWORD="$2" - shift - ;; --mongodb-ca-file-path) MONGODB_CA_FILE_PATH="$2" shift @@ -34,10 +27,6 @@ while [[ $# -gt 1 ]]; do MONGODB_CRL_FILE_PATH="$2" shift ;; - --replica-set-name) - REPLICA_SET_NAME="$2" - shift - ;; --mongodb-fqdn) MONGODB_FQDN="$2" shift @@ -59,20 +48,15 @@ while [[ $# -gt 1 ]]; do done # sanity checks -if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ - -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \ +if [[ -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \ -z "${MONGODB_FQDN:?MONGODB_FQDN not specified. Exiting!}" || \ -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ - -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then - # Not handling the STORAGE_ENGINE_CACHE_SIZE because - # it is optional. If not specified the default cache - # size is: max((50% RAM - 1GB), 256MB) - echo "Missing required enviroment variable(s)." + -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ + -z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then exit 1 else - echo REPLICA_SET_NAME="$REPLICA_SET_NAME" echo MONGODB_PORT="$MONGODB_PORT" echo MONGODB_FQDN="$MONGODB_FQDN" echo MONGODB_IP="$MONGODB_IP" @@ -88,10 +72,8 @@ HOSTS_FILE_PATH=/etc/hosts # configure the mongod.conf file sed -i "s|MONGODB_PORT|${MONGODB_PORT}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_KEY_FILE_PATH|${MONGODB_KEY_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} -#sed -i "s|MONGODB_KEY_FILE_PASSWORD|${MONGODB_KEY_FILE_PASSWORD}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_CA_FILE_PATH|${MONGODB_CA_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_CRL_FILE_PATH|${MONGODB_CRL_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} -sed -i "s|REPLICA_SET_NAME|${REPLICA_SET_NAME}|g" ${MONGODB_CONF_FILE_PATH} if [ ! -z "$STORAGE_ENGINE_CACHE_SIZE" ]; then if [[ "$STORAGE_ENGINE_CACHE_SIZE" =~ ^[0-9]+(G|M|T)B$ ]]; then sed -i.bk "s|STORAGE_ENGINE_CACHE_SIZE|${STORAGE_ENGINE_CACHE_SIZE}|g" ${MONGODB_CONF_FILE_PATH} diff --git a/k8s/mongodb/mongo-sc.yaml b/k8s/mongodb/mongo-sc.yaml index 2b155d54..17140f15 100644 --- a/k8s/mongodb/mongo-sc.yaml +++ b/k8s/mongodb/mongo-sc.yaml @@ -1,5 +1,5 @@ #################################################################### -# This YAML section desribes a StorageClass for the mongodb dbPath # +# This YAML section desribes a StorageClass for the mongodb dbPath # #################################################################### kind: StorageClass apiVersion: storage.k8s.io/v1beta1 @@ -15,7 +15,7 @@ parameters: #kind: Managed --- ###################################################################### -# This YAML section desribes a StorageClass for the mongodb configDB # +# This YAML section desribes a StorageClass for the mongodb configDB # ###################################################################### kind: StorageClass apiVersion: storage.k8s.io/v1beta1 diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index 1243da26..e371a255 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -21,7 +21,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: mongodb - image: bigchaindb/mongodb:3.2 + image: bigchaindb/localmongodb:1.0 imagePullPolicy: IfNotPresent env: - name: MONGODB_FQDN @@ -33,11 +33,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: MONGODB_REPLICA_SET_NAME - valueFrom: - configMapKeyRef: - name: vars - key: mongodb-replicaset-name - name: MONGODB_PORT valueFrom: configMapKeyRef: @@ -57,8 +52,6 @@ spec: - /etc/mongod/ca/ca.pem - --mongodb-crl-file-path - /etc/mongod/ca/crl.pem - - --replica-set-name - - $(MONGODB_REPLICA_SET_NAME) - --mongodb-fqdn - $(MONGODB_FQDN) - --mongodb-ip diff --git a/k8s/nginx-http/container/Dockerfile b/k8s/nginx-http/container/Dockerfile index e35dd5e0..41b1bd26 100644 --- a/k8s/nginx-http/container/Dockerfile +++ b/k8s/nginx-http/container/Dockerfile @@ -6,6 +6,6 @@ RUN apt-get update \ && apt-get autoremove \ && apt-get clean COPY nginx.conf.template /etc/nginx/nginx.conf -COPY nginx_entrypoint.bash / -EXPOSE 80 27017 -ENTRYPOINT ["/nginx_entrypoint.bash"] +COPY nginx_tmt_entrypoint.bash / +EXPOSE 80 27017 9986 46656 +ENTRYPOINT ["/nginx_tmt_entrypoint.bash"] diff --git a/k8s/nginx-http/container/README.md b/k8s/nginx-http/container/README.md index 860ffa4e..9a680c29 100644 --- a/k8s/nginx-http/container/README.md +++ b/k8s/nginx-http/container/README.md @@ -9,7 +9,7 @@ reflect any changes made to the container. ### Note about testing Websocket connections: -You can test the WebSocket server by using +You can test the WebSocket server by using [wsc](https://www.npmjs.com/package/wsc) tool with a command like: `wsc -er ws://localhost:9985/api/v1/streams/valid_transactions`. diff --git a/k8s/nginx-http/container/docker_build_and_push.bash b/k8s/nginx-http/container/docker_build_and_push.bash index 5011eb2d..7c43267e 100755 --- a/k8s/nginx-http/container/docker_build_and_push.bash +++ b/k8s/nginx-http/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/nginx_http:1.1 . +docker build -t bigchaindb/nginx_http:unstable . -docker push bigchaindb/nginx_http:1.1 +docker push bigchaindb/nginx_http:unstable diff --git a/k8s/nginx-http/container/nginx.conf.template b/k8s/nginx-http/container/nginx.conf.template index bc8b8245..a0bb19bb 100644 --- a/k8s/nginx-http/container/nginx.conf.template +++ b/k8s/nginx-http/container/nginx.conf.template @@ -123,16 +123,15 @@ http { } } -# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP -# connection. +# NGINX stream block for TCP and UDP proxies. stream { - log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' '$proxy_protocol_addr $proxy_protocol_port ' '$protocol $status $session_time $bytes_sent ' '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' '"$upstream_bytes_received" "$upstream_connect_time" '; - access_log /dev/stdout mdb_log buffer=16k flush=5s; + access_log /dev/stdout bdb_log buffer=16k flush=5s; # Define a zone 'two' of size 10 megabytes to store the counters # that hold number of TCP connections from a specific IP address. @@ -149,16 +148,23 @@ stream { # The following map block enables lazy-binding to the backend at runtime, # rather than binding as soon as NGINX starts. - map $remote_addr $mdb_backend { - default MONGODB_BACKEND_HOST; + map $remote_addr $tm_backend { + default TM_BACKEND_HOST; } - # Frontend server to forward connections to MDB instance. + # Server to forward connection to nginx instance hosting + # tendermint node public key. server { - listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5; - preread_timeout 30s; + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; tcp_nodelay on; - proxy_pass $mdb_backend:MONGODB_BACKEND_PORT; + proxy_pass $tm_backend:TM_P2P_PORT; } } diff --git a/k8s/nginx-http/container/nginx_entrypoint.bash b/k8s/nginx-http/container/nginx_entrypoint.bash index 797e24c3..3918dbaa 100755 --- a/k8s/nginx-http/container/nginx_entrypoint.bash +++ b/k8s/nginx-http/container/nginx_entrypoint.bash @@ -31,7 +31,10 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting! -z "${bdb_api_port:?BIGCHAINDB_API_PORT not specified. Exiting!}" || \ -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \ -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ - -z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" ]]; then + -z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then exit 1 else echo CLUSTER_FRONTEND_PORT="$cluster_frontend_port" @@ -43,6 +46,9 @@ else echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host" echo BIGCHAINDB_API_PORT="$bdb_api_port" echo BIGCHAINDB_WS_PORT="$bdb_ws_port" + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" + echo TM_BACKEND_HOST="$tm_backend_host" + echo TM_P2P_PORT="$tm_p2p_port" fi NGINX_CONF_FILE=/etc/nginx/nginx.conf @@ -57,6 +63,9 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} # start nginx echo "INFO: starting nginx..." diff --git a/k8s/nginx-http/nginx-http-dep.yaml b/k8s/nginx-http/nginx-http-dep.yaml index 5a9359f5..aed29d56 100644 --- a/k8s/nginx-http/nginx-http-dep.yaml +++ b/k8s/nginx-http/nginx-http-dep.yaml @@ -30,11 +30,6 @@ spec: configMapKeyRef: name: vars key: cluster-dns-server-ip - - name: MONGODB_FRONTEND_PORT - valueFrom: - configMapKeyRef: - name: vars - key: mongodb-frontend-port - name: MONGODB_BACKEND_HOST valueFrom: configMapKeyRef: @@ -60,14 +55,33 @@ spec: configMapKeyRef: name: vars key: bigchaindb-ws-port + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-pub-key-access + - name: TM_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: tendermint-config + key: ngx-tm-instance-name + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-p2p-port ports: - - containerPort: "" - protocol: TCP - containerPort: "" protocol: TCP name: ngx-health - containerPort: "" protocol: TCP + - containerPort: "" + protocol: TCP + name: tm-pub-key + - containerPort: "" + protocol: TCP + name: tm-p2p-port livenessProbe: httpGet: path: /health diff --git a/k8s/nginx-http/nginx-http-svc.yaml b/k8s/nginx-http/nginx-http-svc.yaml index 76c603d2..c0fcab5b 100644 --- a/k8s/nginx-http/nginx-http-svc.yaml +++ b/k8s/nginx-http/nginx-http-svc.yaml @@ -17,4 +17,12 @@ spec: targetPort: "" name: public-cluster-port protocol: TCP + - port: "" + targetPort: "" + name: tm-pub-key-access + protocol: TCP + - port: "" + targetPort: "" + protocol: TCP + name: tm-p2p-port type: LoadBalancer diff --git a/k8s/nginx-https/container/Dockerfile b/k8s/nginx-https/container/Dockerfile index 3bd6b607..4d6d2f77 100644 --- a/k8s/nginx-https/container/Dockerfile +++ b/k8s/nginx-https/container/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update \ && apt-get -y upgrade \ && apt-get autoremove \ && apt-get clean -COPY nginx.conf.template /etc/nginx/nginx.conf -COPY nginx_entrypoint.bash / -EXPOSE 80 443 27017 -ENTRYPOINT ["/nginx_entrypoint.bash"] +COPY nginx.conf.tmt.template /etc/nginx/nginx.conf +COPY nginx_tmt_entrypoint.bash / +EXPOSE 80 443 27017 9986 46656 +ENTRYPOINT ["/nginx_tmt_entrypoint.bash"] diff --git a/k8s/nginx-https/container/README.md b/k8s/nginx-https/container/README.md index 1f649b10..77478f0e 100644 --- a/k8s/nginx-https/container/README.md +++ b/k8s/nginx-https/container/README.md @@ -9,7 +9,7 @@ reflect any changes made to the container. ### Note about testing Websocket connections: -You can test the WebSocket server by using +You can test the WebSocket server by using [wsc](https://www.npmjs.com/package/wsc) tool with a command like: `wsc -er wss://localhost:9985/api/v1/streams/valid_transactions`. diff --git a/k8s/nginx-https/container/docker_build_and_push.bash b/k8s/nginx-https/container/docker_build_and_push.bash index 76494bcb..abc556d8 100755 --- a/k8s/nginx-https/container/docker_build_and_push.bash +++ b/k8s/nginx-https/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/nginx_https:1.1 . +docker build -t bigchaindb/nginx_https:unstable . -docker push bigchaindb/nginx_https:1.1 +docker push bigchaindb/nginx_https:unstable diff --git a/k8s/nginx-https/container/nginx.conf.template b/k8s/nginx-https/container/nginx.conf.template index 3ffb4cce..c372fbe0 100644 --- a/k8s/nginx-https/container/nginx.conf.template +++ b/k8s/nginx-https/container/nginx.conf.template @@ -2,8 +2,7 @@ # 1. Acts as the HTTPS termination point. # 2. Forwards BDB HTTP requests to OpenResty backend. # 3. Forwards BDB WS requests to BDB backend. -# 4. Forwards MDB TCP connections to MDB backend. -# 5. Does health check with LB. +# 4. Does health check with LB. worker_processes 2; daemon off; @@ -152,16 +151,15 @@ http { } } -# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP -# connection. +# NGINX stream block for TCP and UDP proxies. stream { - log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' '$proxy_protocol_addr $proxy_protocol_port ' '$protocol $status $session_time $bytes_sent ' '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' '"$upstream_bytes_received" "$upstream_connect_time" '; - access_log /dev/stdout mdb_log buffer=16k flush=5s; + access_log /dev/stdout bdb_log buffer=16k flush=5s; # Define a zone 'two' of size 10 megabytes to store the counters # that hold number of TCP connections from a specific IP address. @@ -170,10 +168,6 @@ stream { # Enable logging when connections are being throttled. limit_conn_log_level notice; - # For a multi node BigchainDB deployment we need around 2^5 connections - # (for inter-node communication)per node via NGINX, we can bump this up in case - # there is a requirement to scale up. But we should not remove this - # for security reasons. # Allow 256 connections from the same IP address. limit_conn two 256; @@ -182,16 +176,23 @@ stream { # The following map block enables lazy-binding to the backend at runtime, # rather than binding as soon as NGINX starts. - map $remote_addr $mdb_backend { - default MONGODB_BACKEND_HOST; + map $remote_addr $tm_backend { + default TM_BACKEND_HOST; } - # Frontend server to forward connections to MDB instance. + # Server to forward connection to nginx instance hosting + # tendermint node public key. server { - listen MONGODB_FRONTEND_PORT so_keepalive=3m:1m:5; - preread_timeout 30s; + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; tcp_nodelay on; - proxy_pass $mdb_backend:MONGODB_BACKEND_PORT; + proxy_pass $tm_backend:TM_P2P_PORT; } } diff --git a/k8s/nginx-https/container/nginx_entrypoint.bash b/k8s/nginx-https/container/nginx_entrypoint.bash index 8b9a77cc..dfcc298f 100755 --- a/k8s/nginx-https/container/nginx_entrypoint.bash +++ b/k8s/nginx-https/container/nginx_entrypoint.bash @@ -27,6 +27,10 @@ bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` bdb_api_port=`printenv BIGCHAINDB_API_PORT` bdb_ws_port=`printenv BIGCHAINDB_WS_PORT` +# Tendermint vars +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` +tm_backend_host=`printenv TM_BACKEND_HOST` +tm_p2p_port=`printenv TM_P2P_PORT` # sanity check if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!}" || \ @@ -40,7 +44,11 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting! -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \ -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ -z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \ - -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" ]]; then + -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then + echo "Missing required environment variables. Exiting!" exit 1 else echo CLUSTER_FQDN="$cluster_fqdn" @@ -55,6 +63,9 @@ else echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host" echo BIGCHAINDB_API_PORT="$bdb_api_port" echo BIGCHAINDB_WS_PORT="$bdb_ws_port" + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" + echo TM_BACKEND_HOST="$tm_backend_host" + echo TM_P2P_PORT="$tm_p2p_port" fi NGINX_CONF_FILE=/etc/nginx/nginx.conf @@ -72,8 +83,10 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} # start nginx echo "INFO: starting nginx..." exec nginx -c /etc/nginx/nginx.conf - diff --git a/k8s/nginx-https/nginx-https-dep.yaml b/k8s/nginx-https/nginx-https-dep.yaml index 79dfe040..e26323b8 100644 --- a/k8s/nginx-https/nginx-https-dep.yaml +++ b/k8s/nginx-https/nginx-https-dep.yaml @@ -35,11 +35,6 @@ spec: configMapKeyRef: name: vars key: cluster-dns-server-ip - - name: MONGODB_FRONTEND_PORT - valueFrom: - configMapKeyRef: - name: vars - key: mongodb-frontend-port - name: MONGODB_BACKEND_HOST valueFrom: configMapKeyRef: @@ -75,18 +70,37 @@ spec: configMapKeyRef: name: vars key: bigchaindb-ws-port + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-pub-key-access + - name: TM_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: tendermint-config + key: ngx-tm-instance-name + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-p2p-port ports: # return a pretty error message on port 80, since we are expecting # HTTPS traffic. - containerPort: 80 protocol: TCP - - containerPort: "" - protocol: TCP - containerPort: "" protocol: TCP - containerPort: "" protocol: TCP name: ngx-port + - containerPort: "" + protocol: TCP + name: tm-pub-key + - containerPort: "" + protocol: TCP + name: tm-p2p-port livenessProbe: httpGet: path: /health diff --git a/k8s/nginx-https/nginx-https-svc.yaml b/k8s/nginx-https/nginx-https-svc.yaml index 1d817fe5..24b18cb1 100644 --- a/k8s/nginx-https/nginx-https-svc.yaml +++ b/k8s/nginx-https/nginx-https-svc.yaml @@ -21,6 +21,14 @@ spec: targetPort: "" name: public-mdb-port protocol: TCP + - port: "" + targetPort: "" + name: tm-pub-key-access + protocol: TCP + - port: "" + targetPort: "" + protocol: TCP + name: tm-p2p-port - port: 80 targetPort: 80 name: public-insecure-cluster-port diff --git a/k8s/tendermint/nginx_container/Dockerfile b/k8s/tendermint/nginx_container/Dockerfile new file mode 100644 index 00000000..9e3a1cf8 --- /dev/null +++ b/k8s/tendermint/nginx_container/Dockerfile @@ -0,0 +1,12 @@ +FROM nginx:stable +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /etc/nginx/conf.d/access_pub_key.conf +COPY nginx_entrypoint.bash / +VOLUME /usr/share/nginx +EXPOSE 9986 +ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/tendermint/nginx_container/README.md b/k8s/tendermint/nginx_container/README.md new file mode 100644 index 00000000..98e11971 --- /dev/null +++ b/k8s/tendermint/nginx_container/README.md @@ -0,0 +1,19 @@ +## Nginx container for hosting public key for a tendermint instance + + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + +### Step 2: Run the container + +``` +docker run \ + --name=tendermint_instance_pub_key \ + --env TENDERMINT_PUB_KEY_ACCESS_PORT='' + --publish=: \ + --volume=:/tendermint_node_data \ + bigchaindb/nginx_pub_key_access: +``` diff --git a/k8s/tendermint/nginx_container/docker_build_and_push.bash b/k8s/tendermint/nginx_container/docker_build_and_push.bash new file mode 100755 index 00000000..880df86d --- /dev/null +++ b/k8s/tendermint/nginx_container/docker_build_and_push.bash @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t bigchaindb/nginx_pub_key_access:unstable . + +docker push bigchaindb/nginx_pub_key_access:unstable diff --git a/k8s/tendermint/nginx_container/nginx.conf.template b/k8s/tendermint/nginx_container/nginx.conf.template new file mode 100644 index 00000000..90251ddf --- /dev/null +++ b/k8s/tendermint/nginx_container/nginx.conf.template @@ -0,0 +1,10 @@ +# Serve the public key for a tendermint instance + +server { + listen PUBLIC_KEY_ACCESS_PORT default_server; + listen [::]:PUBLIC_KEY_ACCESS_PORT default_server ipv6only=on; + location / { + root /usr/share/nginx/; + autoindex on; + } +} diff --git a/k8s/tendermint/nginx_container/nginx_entrypoint.bash b/k8s/tendermint/nginx_container/nginx_entrypoint.bash new file mode 100755 index 00000000..100b89a9 --- /dev/null +++ b/k8s/tendermint/nginx_container/nginx_entrypoint.bash @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +# Tendermint public key access port +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` + +if [[ -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting}" ]]; then + exit 1 +else + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" +fi + +NGINX_CONF_FILE=/etc/nginx/conf.d/access_pub_key.conf + +# configure the access_pub_key file with env variable(s) +sed -i "s|PUBLIC_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} + +cat /etc/nginx/conf.d/access_pub_key.conf +# start nginx +echo "INFO: starting nginx..." +exec nginx -g "daemon off;" diff --git a/k8s/tendermint/tendermint-ext-conn-svc.yaml b/k8s/tendermint/tendermint-ext-conn-svc.yaml new file mode 100644 index 00000000..9b009fa3 --- /dev/null +++ b/k8s/tendermint/tendermint-ext-conn-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + # Name of tendermint instance you are trying to connect to + # e.g. tm-instance-0 + name: "" + namespace: default +spec: + ports: +spec: + ports: + - port: "" + name: p2p + - port: "" + name: pubkey + # FQDN of remote cluster/NGINX instance + externalName: "" diff --git a/k8s/tendermint/tendermint-pv.yaml b/k8s/tendermint/tendermint-pv.yaml new file mode 100644 index 00000000..dedf5063 --- /dev/null +++ b/k8s/tendermint/tendermint-pv.yaml @@ -0,0 +1,41 @@ +######################################################### +# This YAML section desribes a k8s PV for tendermint db # +######################################################### +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-tm-db +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 20Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: tendermint-db +--- +############################################################## +# This YAML section desribes a k8s PV for Tendermint config # +############################################################## +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-tm-configdb +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: tendermint-config-db diff --git a/k8s/tendermint/tendermint-pvc.yaml b/k8s/tendermint/tendermint-pvc.yaml new file mode 100644 index 00000000..c7fa6a94 --- /dev/null +++ b/k8s/tendermint/tendermint-pvc.yaml @@ -0,0 +1,32 @@ +########################################################## +# This section file desribes a k8s pvc for tendermint db # +########################################################## +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: tendermint-db-claim + annotations: + volume.beta.kubernetes.io/storage-class: tendermint-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +################################################################ +# This YAML section desribes a k8s pvc for tendermint configdb # +################################################################ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: tendermint-config-db-claim + annotations: + volume.beta.kubernetes.io/storage-class: tendermint-config-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + diff --git a/k8s/tendermint/tendermint-sc.yaml b/k8s/tendermint/tendermint-sc.yaml new file mode 100644 index 00000000..897edc52 --- /dev/null +++ b/k8s/tendermint/tendermint-sc.yaml @@ -0,0 +1,31 @@ +################################################################### +# This YAML section desribes a StorageClass for the tendermint db # +################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: tendermint-db +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] + location: westeurope + # If you have created a different storage account e.g. for Premium Storage + #storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed +--- +######################################################################### +# This YAML section desribes a StorageClass for the tendermint configdb # +######################################################################### +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: tendermint-config-db +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] + location: westeurope + # If you have created a different storage account e.g. for Premium Storage + #storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed diff --git a/k8s/tendermint/tendermint-ss.yaml b/k8s/tendermint/tendermint-ss.yaml new file mode 100644 index 00000000..1e0a4af9 --- /dev/null +++ b/k8s/tendermint/tendermint-ss.yaml @@ -0,0 +1,115 @@ +################################################################################# +# This YAML file desribes a StatefulSet with a service for running and exposing # +# a Tendermint instance. It depends on the tendermint-config-db-claim # +# and tendermint-db-claim k8s pvc. # +################################################################################# + +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: tm-instance-0-ss + namespace: default +spec: + serviceName: tm-instance-0 + replicas: 1 + template: + metadata: + name: tm-instance-0-ss + labels: + app: tm-instance-0-ss + spec: + restartPolicy: Always + volumes: + - name: tm-data + persistentVolumeClaim: + claimName: tendermint-db-claim + - name: tm-config-data + persistentVolumeClaim: + claimName: tendermint-config-db-claim + containers: + # Treating nginx + tendermint as a POD because they should not + # exist without each other + # Nginx container for hosting public key of this ndoe + - name: nginx + imagePullPolicy: Always + image: bigchaindb/nginx_pub_key_access:unstable + env: + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-pub-key-access + ports: + - containerPort: "" + name: tm-pk-access + volumeMounts: + - name: tm-config-data + mountPath: /usr/share/nginx + readOnly: true + #Tendermint container + - name: tendermint + imagePullPolicy: Always + image: bigchaindb/tendermint:unstable + env: + - name: TM_SEEDS + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-seeds + - name: TM_VALIDATOR_POWER + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-validator-power + - name: TM_VALIDATORS + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-validators + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-pub-key-access + - name: TM_GENESIS_TIME + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-genesis-time + - name: TM_CHAIN_ID + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-chain-id + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-p2p-port + - name: TMHOME + value: /tendermint + - name: TM_PROXY_APP + valueFrom: + configMapKeyRef: + name: vars + key: bdb-instance-name + - name: TM_ABCI_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-abci-port + # Resource constraint on the pod, can be changed + resources: + limits: + cpu: 200m + memory: 5G + volumeMounts: + - name: tm-data + mountPath: /tendermint + - name: tm-config-data + mountPath: /tendermint_node_data + ports: + - containerPort: "" + name: p2p + - containerPort: "" + name: rpc diff --git a/k8s/tendermint/tendermint-svc.yaml b/k8s/tendermint/tendermint-svc.yaml new file mode 100644 index 00000000..d79a2335 --- /dev/null +++ b/k8s/tendermint/tendermint-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: tm-instance-1 + namespace: default + labels: + name: tm-instance-1 +spec: + selector: + app: tm-instance-1-ss + ports: + - port: "" + targetPort: "" + name: p2p + protocol: TCP + - port: "" + targetPort: "" + name: rpc + protocol: TCP + - port: "" + targetPort: "" + name: pub-key-access + protocol: TCP + clusterIP: None diff --git a/k8s/tendermint/tendermint_container/Dockerfile b/k8s/tendermint/tendermint_container/Dockerfile new file mode 100644 index 00000000..a9e100b3 --- /dev/null +++ b/k8s/tendermint/tendermint_container/Dockerfile @@ -0,0 +1,8 @@ +FROM tendermint/tendermint:develop +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +COPY genesis.json.template /etc/tendermint/genesis.json +COPY tendermint_entrypoint.bash / +VOLUME /tendermint /tendermint_node_data +EXPOSE 46656 46657 +ENTRYPOINT ["/tendermint_entrypoint.bash"] diff --git a/k8s/tendermint/tendermint_container/README.md b/k8s/tendermint/tendermint_container/README.md new file mode 100644 index 00000000..c5ad76ed --- /dev/null +++ b/k8s/tendermint/tendermint_container/README.md @@ -0,0 +1,29 @@ +## Tendermint container used for BFT replication and consensus + + +### Step 1: Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + +### Step 2: Run the container + +``` +docker run \ + --name=tendermint \ + --env TM_PUB_KEY_ACCESS_PORT= \ + --env TM_SEEDS= \ + --env TM_VALIDATOR_POWER= \ + --env TM_VALIDATORS= \ + --env TM_GENESIS_TIME= \ + --env TM_CHAIN_ID= \ + --env TM_P2P_PORT= \ + --env TMHOME= \ + --env TM_PROXY_APP= \ + --publish=: \ + --publish=: \ + --volume :/tendermint \ + --volume=:/tendermint_node_data \ + bigchaindb/tendermint: +``` diff --git a/k8s/tendermint/tendermint_container/docker_build_and_push.bash b/k8s/tendermint/tendermint_container/docker_build_and_push.bash new file mode 100755 index 00000000..6cd23cd4 --- /dev/null +++ b/k8s/tendermint/tendermint_container/docker_build_and_push.bash @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t bigchaindb/tendermint:unstable . + +docker push bigchaindb/tendermint:unstable diff --git a/k8s/tendermint/tendermint_container/genesis.json.template b/k8s/tendermint/tendermint_container/genesis.json.template new file mode 100644 index 00000000..7b726b63 --- /dev/null +++ b/k8s/tendermint/tendermint_container/genesis.json.template @@ -0,0 +1,6 @@ +{ + "genesis_time": TM_GENESIS_TIME, + "chain_id": TM_CHAIN_ID, + "validators": [], + "app_hash": "" +} diff --git a/k8s/tendermint/tendermint_container/tendermint_entrypoint.bash b/k8s/tendermint/tendermint_container/tendermint_entrypoint.bash new file mode 100755 index 00000000..f44fd9af --- /dev/null +++ b/k8s/tendermint/tendermint_container/tendermint_entrypoint.bash @@ -0,0 +1,98 @@ +#!/bin/bash +set -euo pipefail + +# Cluster vars +tm_seeds=`printenv TM_SEEDS` +tm_validators=`printenv TM_VALIDATORS` +tm_validator_power=`printenv TM_VALIDATOR_POWER` +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` +tm_genesis_time=`printenv TM_GENESIS_TIME` +tm_chain_id=`printenv TM_CHAIN_ID` +tm_p2p_port=`printenv TM_P2P_PORT` + + +# tendermint node vars +tmhome=`printenv TMHOME` +tm_proxy_app=`printenv TM_PROXY_APP` +tm_abci_port=`printenv TM_ABCI_PORT` + + +# sanity check +if [[ -z "${tm_seeds:?TM_SEEDS not specified. Exiting!}" || \ + -z "${tm_validators:?TM_VALIDATORS not specified. Exiting!}" || \ + -z "${tm_validator_power:?TM_VALIDATOR_POWER not specified. Exiting!}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_genesis_time:?TM_GENESIS_TIME not specified. Exiting!}" || \ + -z "${tm_chain_id:?TM_CHAIN_ID not specified. Exiting!}" || \ + -z "${tmhome:?TMHOME not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" || \ + -z "${tm_abci_port:?TM_ABCI_PORT not specified. Exiting! }" ]]; then + echo "Missing required enviroment variables." + exit 1 +else + echo tm_seeds="$TM_SEEDS" + echo tm_validators="$TM_VALIDATORS" + echo tm_validator_power="$TM_VALIDATOR_POWER" + echo tm_pub_key_access_port="$TM_PUB_KEY_ACCESS_PORT" + echo tm_genesis_time="$TM_GENESIS_TIME" + echo tm_chain_id="$TM_CHAIN_ID" + echo tmhome="$TMHOME" + echo tm_p2p_port="$TM_P2P_PORT" + echo tm_abci_port="$TM_ABCI_PORT" +fi + +# copy template +cp /etc/tendermint/genesis.json /tendermint/genesis.json + +TM_GENESIS_FILE=/tendermint/genesis.json +TM_PUB_KEY_DIR=/tendermint_node_data + +# configure the nginx.conf file with env variables +sed -i "s|TM_GENESIS_TIME|\"${tm_genesis_time}\"|g" ${TM_GENESIS_FILE} +sed -i "s|TM_CHAIN_ID|\"${tm_chain_id}\"|g" ${TM_GENESIS_FILE} + +if [ ! -f /tendermint/priv_validator.json ]; then + tendermint gen_validator > /tendermint/priv_validator.json + # pub_key.json will be served by the nginx container + cat /tendermint/priv_validator.json + cat /tendermint/priv_validator.json | jq ".pub_key" > "$TM_PUB_KEY_DIR"/pub_key.json +fi + +# fill genesis file with validators +IFS=',' read -ra VALS_ARR <<< "$TM_VALIDATORS" +IFS=',' read -ra VAL_POWERS_ARR <<< "$TM_VALIDATOR_POWER" +if [ ${#VALS_ARR[@]} -ne ${#VAL_POWERS_ARR[@]} ]; then + echo "Invalid configuration of Validator(s) and Validator Power(s)" + exit 1 +fi +for i in "${!VALS_ARR[@]}"; do + # wait until validator generates priv/pub key pair + set +e + echo Validator: "${VALS_ARR[$i]}" + echo Validator Power: "${VALS_POWERS_ARR[$i]}" + echo "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + sleep 5 + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + done + set -e + # add validator to genesis file along with its pub_key + curl -s "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" | jq ". as \$k | {pub_key: \$k, power: ${VAL_POWERS_ARR[$i]}, name: \"${VALS_ARR[$i]}\"}" > pub_validator.json + cat /tendermint/genesis.json | jq ".validators |= .+ [$(cat pub_validator.json)]" > tmpgenesis && mv tmpgenesis /tendermint/genesis.json + rm pub_validator.json + done + +# construct seeds +IFS=',' read -ra SEEDS_ARR <<< "$tm_seeds" +seeds=() +for s in "${SEEDS_ARR[@]}"; do + seeds+=("$s:$tm_p2p_port") +done +seeds=$(IFS=','; echo "${seeds[*]}") + +# start nginx +echo "INFO: starting tendermint..." +exec tendermint node --p2p.seeds="$seeds" --moniker="`hostname`" --proxy_app="tcp://$tm_proxy_app:$tm_abci_port"