From ca21c7b321c67b02ff6ade6cce5b7bbee5c154ae Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Mar 2017 16:14:39 +0100 Subject: [PATCH] Remove all stuff related to 'bigchaindb load' in AWS scripts --- deploy-cluster-aws/awsdeploy.sh | 100 ++++++++---------- deploy-cluster-aws/example_deploy_conf.py | 4 - deploy-cluster-aws/fabfile.py | 15 --- deploy-cluster-aws/launch_ec2_nodes.py | 10 +- .../clusters-feds/aws-testing-cluster.md | 1 - 5 files changed, 46 insertions(+), 84 deletions(-) diff --git a/deploy-cluster-aws/awsdeploy.sh b/deploy-cluster-aws/awsdeploy.sh index 00d1f431..b733ef2d 100755 --- a/deploy-cluster-aws/awsdeploy.sh +++ b/deploy-cluster-aws/awsdeploy.sh @@ -39,7 +39,6 @@ fi echo "NUM_NODES = "$NUM_NODES echo "BRANCH = "$BRANCH -echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY echo "SSH_KEY_NAME" = $SSH_KEY_NAME echo "USE_KEYPAIRS_FILE = "$USE_KEYPAIRS_FILE echo "IMAGE_ID = "$IMAGE_ID @@ -85,7 +84,7 @@ if [[ $CONFILES_COUNT != $NUM_NODES ]]; then fi # Auto-generate the tag to apply to all nodes in the cluster -TAG="BDB-"$WHAT_TO_DEPLOY"-"`date +%m-%d@%H:%M` +TAG="BDB-Server-"`date +%m-%d@%H:%M` echo "TAG = "$TAG # Change the file permissions on the SSH private key file @@ -121,25 +120,24 @@ fab install_base_software fab get_pip3 fab upgrade_setuptools -if [ "$WHAT_TO_DEPLOY" == "servers" ]; then - # (Re)create the RethinkDB configuration file conf/rethinkdb.conf - if [ "$ENABLE_WEB_ADMIN" == "True" ]; then - if [ "$BIND_HTTP_TO_LOCALHOST" == "True" ]; then - python create_rethinkdb_conf.py --enable-web-admin --bind-http-to-localhost - else - python create_rethinkdb_conf.py --enable-web-admin - fi +# (Re)create the RethinkDB configuration file conf/rethinkdb.conf +if [ "$ENABLE_WEB_ADMIN" == "True" ]; then + if [ "$BIND_HTTP_TO_LOCALHOST" == "True" ]; then + python create_rethinkdb_conf.py --enable-web-admin --bind-http-to-localhost else - python create_rethinkdb_conf.py + python create_rethinkdb_conf.py --enable-web-admin fi - # Rollout RethinkDB and start it - fab prep_rethinkdb_storage:$USING_EBS - fab install_rethinkdb - fab configure_rethinkdb - fab delete_rethinkdb_data - fab start_rethinkdb +else + python create_rethinkdb_conf.py fi +# Rollout RethinkDB and start it +fab prep_rethinkdb_storage:$USING_EBS +fab install_rethinkdb +fab configure_rethinkdb +fab delete_rethinkdb_data +fab start_rethinkdb + # Rollout BigchainDB (but don't start it yet) if [ "$BRANCH" == "pypi" ]; then fab install_bigchaindb_from_pypi @@ -156,48 +154,40 @@ fi # Configure BigchainDB on all nodes -if [ "$WHAT_TO_DEPLOY" == "servers" ]; then - # The idea is to send a bunch of locally-created configuration - # files out to each of the instances / nodes. +# The idea is to send a bunch of locally-created configuration +# files out to each of the instances / nodes. - # Assume a set of $NUM_NODES BigchaindB config files - # already exists in the confiles directory. - # One can create a set using a command like - # ./make_confiles.sh confiles $NUM_NODES - # (We can't do that here now because this virtual environment - # is a Python 2 environment that may not even have - # bigchaindb installed, so bigchaindb configure can't be called) +# Assume a set of $NUM_NODES BigchaindB config files +# already exists in the confiles directory. +# One can create a set using a command like +# ./make_confiles.sh confiles $NUM_NODES +# (We can't do that here now because this virtual environment +# is a Python 2 environment that may not even have +# bigchaindb installed, so bigchaindb configure can't be called) - # Transform the config files in the confiles directory - # to have proper keyrings etc. - if [ "$USE_KEYPAIRS_FILE" == "True" ]; then - python clusterize_confiles.py -k confiles $NUM_NODES - else - python clusterize_confiles.py confiles $NUM_NODES - fi - - # Send one of the config files to each instance - for (( HOST=0 ; HOST<$NUM_NODES ; HOST++ )); do - CONFILE="bcdb_conf"$HOST - echo "Sending "$CONFILE - fab set_host:$HOST send_confile:$CONFILE - done - - # Initialize BigchainDB (i.e. Create the RethinkDB database, - # the tables, the indexes, and genesis glock). Note that - # this will only be sent to one of the nodes, see the - # definition of init_bigchaindb() in fabfile.py to see why. - fab init_bigchaindb - fab set_shards:$NUM_NODES - echo "To set the replication factor to 3, do: fab set_replicas:3" - echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" +# Transform the config files in the confiles directory +# to have proper keyrings etc. +if [ "$USE_KEYPAIRS_FILE" == "True" ]; then + python clusterize_confiles.py -k confiles $NUM_NODES else - # Deploying clients - fab send_client_confile:client_confile - - # Start sending load from the clients to the servers - fab start_bigchaindb_load + python clusterize_confiles.py confiles $NUM_NODES fi +# Send one of the config files to each instance +for (( HOST=0 ; HOST<$NUM_NODES ; HOST++ )); do + CONFILE="bcdb_conf"$HOST + echo "Sending "$CONFILE + fab set_host:$HOST send_confile:$CONFILE +done + +# Initialize BigchainDB (i.e. Create the RethinkDB database, +# the tables, the indexes, and genesis glock). Note that +# this will only be sent to one of the nodes, see the +# definition of init_bigchaindb() in fabfile.py to see why. +fab init_bigchaindb +fab set_shards:$NUM_NODES +echo "To set the replication factor to 3, do: fab set_replicas:3" +echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" + # cleanup rm add2known_hosts.sh diff --git a/deploy-cluster-aws/example_deploy_conf.py b/deploy-cluster-aws/example_deploy_conf.py index 623151ef..6aab8f30 100644 --- a/deploy-cluster-aws/example_deploy_conf.py +++ b/deploy-cluster-aws/example_deploy_conf.py @@ -23,10 +23,6 @@ NUM_NODES=3 # It's where to get the BigchainDB code to be deployed on the nodes BRANCH="master" -# WHAT_TO_DEPLOY is either "servers" or "clients" -# What do you want to deploy? -WHAT_TO_DEPLOY="servers" - # SSH_KEY_NAME is the name of the SSH private key file # in $HOME/.ssh/ # It is used for SSH communications with AWS instances. diff --git a/deploy-cluster-aws/fabfile.py b/deploy-cluster-aws/fabfile.py index 9ef24edd..737109f9 100644 --- a/deploy-cluster-aws/fabfile.py +++ b/deploy-cluster-aws/fabfile.py @@ -237,15 +237,6 @@ def send_confile(confile): run('bigchaindb show-config') -@task -@parallel -def send_client_confile(confile): - put(confile, 'tempfile') - run('mv tempfile ~/.bigchaindb') - print('For this node, bigchaindb show-config says:') - run('bigchaindb show-config') - - # Initialize BigchainDB # i.e. create the database, the tables, # the indexes, and the genesis block. @@ -278,12 +269,6 @@ def start_bigchaindb(): sudo('screen -d -m bigchaindb -y start &', pty=False) -@task -@parallel -def start_bigchaindb_load(): - sudo('screen -d -m bigchaindb load &', pty=False) - - # Install and run New Relic @task @parallel diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py index e02b7b62..5418069f 100644 --- a/deploy-cluster-aws/launch_ec2_nodes.py +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -26,7 +26,7 @@ import boto3 from awscommon import get_naeips -SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'SSH_KEY_NAME', +SETTINGS = ['NUM_NODES', 'BRANCH', 'SSH_KEY_NAME', 'USE_KEYPAIRS_FILE', 'IMAGE_ID', 'INSTANCE_TYPE', 'SECURITY_GROUP', 'USING_EBS', 'EBS_VOLUME_SIZE', 'EBS_OPTIMIZED', 'ENABLE_WEB_ADMIN', 'BIND_HTTP_TO_LOCALHOST'] @@ -77,9 +77,6 @@ if not isinstance(NUM_NODES, int): if not isinstance(BRANCH, str): raise SettingsTypeError('BRANCH should be a string') -if not isinstance(WHAT_TO_DEPLOY, str): - raise SettingsTypeError('WHAT_TO_DEPLOY should be a string') - if not isinstance(SSH_KEY_NAME, str): raise SettingsTypeError('SSH_KEY_NAME should be a string') @@ -117,11 +114,6 @@ if NUM_NODES > 64: 'The AWS deployment configuration file sets it to {}'. format(NUM_NODES)) -if WHAT_TO_DEPLOY not in ['servers', 'clients']: - raise ValueError('WHAT_TO_DEPLOY should be either "servers" or "clients". ' - 'The AWS deployment configuration file sets it to {}'. - format(WHAT_TO_DEPLOY)) - if SSH_KEY_NAME in ['not-set-yet', '', None]: raise ValueError('SSH_KEY_NAME should be set. ' 'The AWS deployment configuration file sets it to {}'. diff --git a/docs/server/source/clusters-feds/aws-testing-cluster.md b/docs/server/source/clusters-feds/aws-testing-cluster.md index ac1deff1..d4b4c12e 100644 --- a/docs/server/source/clusters-feds/aws-testing-cluster.md +++ b/docs/server/source/clusters-feds/aws-testing-cluster.md @@ -86,7 +86,6 @@ Step 2 is to make an AWS deployment configuration file, if necessary. There's an ```text NUM_NODES=3 BRANCH="master" -WHAT_TO_DEPLOY="servers" SSH_KEY_NAME="not-set-yet" USE_KEYPAIRS_FILE=False IMAGE_ID="ami-8504fdea"