bigchaindb
/
bigchaindb
mirror of https://github.com/bigchaindb/bigchaindb.git
1
0
Fork 0
Code Releases Activity

Merge branch 'feature/improve-docker-configuration'

This commit is contained in:
vrde 2016-04-28 16:42:45 +02:00
parent ec0662bdc7 c9773b3f2c
commit ed1415161d
No known key found for this signature in database
GPG Key ID: 6581C7C39B3D397D
12 changed files with 321 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Dockerfile
View File

@ -1,12 +1,26 @@
FROM python:3.5
FROM rethinkdb:2.3
RUN apt-get update
RUN apt-get -y install python3 python3-pip
RUN pip3 install --upgrade pip
RUN pip3 install --upgrade setuptools
RUN mkdir -p /usr/src/app
WORKDIR /usr/src/app
RUN pip install --upgrade pip
COPY . /usr/src/app/
RUN pip install --no-cache-dir -e .[dev]
WORKDIR /usr/src/app
RUN pip3 install --no-cache-dir -e .
WORKDIR /data
ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb
ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1
ENTRYPOINT ["bigchaindb", "--experimental-start-rethinkdb"]
CMD ["start"]
EXPOSE 8080 9984 28015 29015

117
bigchaindb/commands/bigchain.py
View File

@ -9,12 +9,20 @@ import logging
import argparse
import copy
import json
import builtins
import logstats
import bigchaindb
import bigchaindb.config_utils
from bigchaindb.util import ProcessGroup
from bigchaindb.client import temp_client
from bigchaindb import db
from bigchaindb.exceptions import DatabaseAlreadyExists, KeypairNotFoundException
from bigchaindb.commands.utils import base_parser, start
from bigchaindb.exceptions import (StartupError,
DatabaseAlreadyExists,
KeypairNotFoundException)
from bigchaindb.commands import utils
from bigchaindb.processes import Processes
from bigchaindb import crypto
@ -23,6 +31,14 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# We need this because `input` always prints on stdout, while it should print
# to stderr. It's a very old bug, check it out here:
# - https://bugs.python.org/issue1927
def input(prompt):
print(prompt, end='', file=sys.stderr)
return builtins.input()
def run_show_config(args):
"""Show the current configuration"""
# TODO Proposal: remove the "hidden" configuration. Only show config. If
@ -43,7 +59,11 @@ def run_configure(args, skip_if_exists=False):
skip_if_exists (bool): skip the function if a config file already exists
"""
config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH
config_file_exists = os.path.exists(config_path)
config_file_exists = False
# if the config path is `-` then it's stdout
if config_path != '-':
config_file_exists = os.path.exists(config_path)
if config_file_exists and skip_if_exists:
return
@ -54,10 +74,15 @@ def run_configure(args, skip_if_exists=False):
if want != 'y':
return
# Patch the default configuration with the new values
conf = copy.deepcopy(bigchaindb._config)
conf = copy.deepcopy(bigchaindb.config)
print('Generating keypair')
# Patch the default configuration with the new values
conf = bigchaindb.config_utils.update(
conf,
bigchaindb.config_utils.env_config(bigchaindb.config))
print('Generating keypair', file=sys.stderr)
conf['keypair']['private'], conf['keypair']['public'] = \
crypto.generate_key_pair()
@ -80,9 +105,12 @@ def run_configure(args, skip_if_exists=False):
input('Statsd {}? (default `{}`): '.format(key, val)) \
or val
bigchaindb.config_utils.write_config(conf, config_path)
print('Configuration written to {}'.format(config_path))
print('Ready to go!')
if config_path != '-':
bigchaindb.config_utils.write_config(conf, config_path)
else:
print(json.dumps(conf, indent=4, sort_keys=True))
print('Configuration written to {}'.format(config_path), file=sys.stderr)
print('Ready to go!', file=sys.stderr)
def run_export_my_pubkey(args):
@ -110,8 +138,8 @@ def run_init(args):
try:
db.init()
except DatabaseAlreadyExists:
print('The database already exists.')
print('If you wish to re-initialize it, first drop it.')
print('The database already exists.', file=sys.stderr)
print('If you wish to re-initialize it, first drop it.', file=sys.stderr)
def run_drop(args):
@ -122,8 +150,15 @@ def run_drop(args):
def run_start(args):
"""Start the processes to run the node"""
# run_configure(args, skip_if_exists=True)
bigchaindb.config_utils.autoconfigure(filename=args.config, force=True)
if args.start_rethinkdb:
try:
proc = utils.start_rethinkdb()
except StartupError as e:
sys.exit('Error starting RethinkDB, reason is: {}'.format(e))
logger.info('RethinkDB started with PID %s' % proc.pid)
try:
db.init()
except DatabaseAlreadyExists:
@ -137,10 +172,46 @@ def run_start(args):
processes.start()
def _run_load(tx_left, stats):
logstats.thread.start(stats)
client = temp_client()
while True:
tx = client.create()
stats['transactions'] += 1
if tx_left is not None:
tx_left -= 1
if tx_left == 0:
break
def run_load(args):
bigchaindb.config_utils.autoconfigure(filename=args.config, force=True)
logger.info('Starting %s processes', args.multiprocess)
stats = logstats.Logstats()
logstats.thread.start(stats)
tx_left = None
if args.count > 0:
tx_left = int(args.count / args.multiprocess)
workers = ProcessGroup(concurrency=args.multiprocess,
target=_run_load,
args=(tx_left, stats.get_child()))
workers.start()
def main():
parser = argparse.ArgumentParser(
description='Control your BigchainDB node.',
parents=[base_parser])
parents=[utils.base_parser])
parser.add_argument('--experimental-start-rethinkdb',
dest='start_rethinkdb',
action='store_true',
help='Run RethinkDB on start')
# all the commands are contained in the subparsers object,
# the command selected by the user will be stored in `args.command`
@ -172,7 +243,25 @@ def main():
subparsers.add_parser('start',
help='Start BigchainDB')
start(parser, globals())
load_parser = subparsers.add_parser('load',
help='Write transactions to the backlog')
load_parser.add_argument('-m', '--multiprocess',
nargs='?',
type=int,
default=False,
help='Spawn multiple processes to run the command, '
'if no value is provided, the number of processes '
'is equal to the number of cores of the host machine')
load_parser.add_argument('-c', '--count',
default=0,
type=int,
help='Number of transactions to push. If the parameter -m '
'is set, the count is distributed equally to all the '
'processes')
utils.start(parser, globals())
if __name__ == '__main__':

84
bigchaindb/commands/bigchain_benchmark.py
View File

@ -1,84 +0,0 @@
"""Command line interface for the `bigchaindb-benchmark` command."""
import logging
import argparse
import logstats
import bigchaindb
import bigchaindb.config_utils
from bigchaindb.util import ProcessGroup
from bigchaindb.client import temp_client
from bigchaindb.commands.utils import base_parser, start
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def _run_load(tx_left, stats):
logstats.thread.start(stats)
client = temp_client()
# b = bigchaindb.Bigchain()
while True:
tx = client.create()
stats['transactions'] += 1
if tx_left is not None:
tx_left -= 1
if tx_left == 0:
break
def run_load(args):
bigchaindb.config_utils.autoconfigure(filename=args.config, force=True)
logger.info('Starting %s processes', args.multiprocess)
stats = logstats.Logstats()
logstats.thread.start(stats)
tx_left = None
if args.count > 0:
tx_left = int(args.count / args.multiprocess)
workers = ProcessGroup(concurrency=args.multiprocess,
target=_run_load,
args=(tx_left, stats.get_child()))
workers.start()
def main():
parser = argparse.ArgumentParser(description='Benchmark your bigchain federation.',
parents=[base_parser])
# all the commands are contained in the subparsers object,
# the command selected by the user will be stored in `args.command`
# that is used by the `main` function to select which other
# function to call.
subparsers = parser.add_subparsers(title='Commands',
dest='command')
# parser for database level commands
load_parser = subparsers.add_parser('load',
help='Write transactions to the backlog')
load_parser.add_argument('-m', '--multiprocess',
nargs='?',
type=int,
default=False,
help='Spawn multiple processes to run the command, '
'if no value is provided, the number of processes '
'is equal to the number of cores of the host machine')
load_parser.add_argument('-c', '--count',
default=0,
type=int,
help='Number of transactions to push. If the parameter -m '
'is set, the count is distributed equally to all the '
'processes')
start(parser, globals())
if __name__ == '__main__':
main()

50
bigchaindb/commands/utils.py
View File

@ -4,10 +4,57 @@ for ``argparse.ArgumentParser``.
import argparse
import multiprocessing as mp
import subprocess
import rethinkdb as r
import bigchaindb
from bigchaindb.exceptions import StartupError
from bigchaindb import db
from bigchaindb.version import __version__
def start_rethinkdb():
"""Start RethinkDB as a child process and wait for it to be
available.
Raises:
``bigchaindb.exceptions.StartupError`` if RethinkDB cannot
be started.
"""
proc = subprocess.Popen(['rethinkdb', '--bind', 'all'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
dbname = bigchaindb.config['database']['name']
line = ''
for line in proc.stdout:
if line.startswith('Server ready'):
# FIXME: seems like tables are not ready when the server is ready,
# that's why we need to query RethinkDB to know the state
# of the database. This code assumes the tables are ready
# when the database is ready. This seems a valid assumption.
try:
conn = db.get_conn()
# Before checking if the db is ready, we need to query
# the server to check if it contains that db
if r.db_list().contains(dbname).run(conn):
r.db(dbname).wait().run(conn)
except (r.ReqlOpFailedError, r.ReqlDriverError) as exc:
raise StartupError('Error waiting for the database `{}` '
'to be ready'.format(dbname)) from exc
return proc
# We are here when we exhaust the stdout of the process.
# The last `line` contains info about the error.
raise StartupError(line)
def start(parser, scope):
"""Utility function to execute a subcommand.
@ -51,7 +98,8 @@ def start(parser, scope):
base_parser = argparse.ArgumentParser(add_help=False, prog='bigchaindb')
base_parser.add_argument('-c', '--config',
help='Specify the location of the configuration file')
help='Specify the location of the configuration file '
'(use "-" for stdout)')
base_parser.add_argument('-y', '--yes', '--yes-please',
action='store_true',

3
bigchaindb/config_utils.py
View File

@ -91,7 +91,8 @@ def file_config(filename=None):
file at CONFIG_DEFAULT_PATH, if filename == None)
"""
logger.debug('On entry into file_config(), filename = {}'.format(filename))
if not filename:
if filename is None:
filename = CONFIG_DEFAULT_PATH
logger.debug('file_config() will try to open `{}`'.format(filename))

3
bigchaindb/exceptions.py
View File

@ -28,4 +28,5 @@ class DatabaseDoesNotExist(Exception):
class KeypairNotFoundException(Exception):
"""Raised if operation cannot proceed because the keypair was not given"""
class StartupError(Exception):
"""Raised when there is an error starting up the system"""

9
docs/source/bigchaindb-cli.md
View File

@ -1,6 +1,6 @@
# The BigchainDB Command Line Interface (CLI)
There are some command-line commands for working with BigchainDB: `bigchaindb` and `bigchaindb-benchmark`. This section provides an overview of those commands.
The command to interact with BigchainDB is `bigchaindb`. This section provides an overview of the command.
## bigchaindb
@ -37,10 +37,9 @@ This command drops (erases) the RethinkDB database. You will be prompted to make
This command starts BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`.
## bigchaindb-benchmark
### bigchaindb load
The `bigchaindb-benchmark` command is used to run benchmarking tests. You can learn more about it using:
This command is used to run benchmarking tests. You can learn more about it using:
```text
$ bigchaindb-benchmark -h
$ bigchaindb-benchmark load -h
$ bigchaindb load -h
```

143
docs/source/installing-server.md
View File

@ -111,9 +111,110 @@ If it's the first time you've run `bigchaindb start`, then it creates the databa
**NOT for Production Use**
For those who like using Docker and wish to experiment with BigchainDB in non-production environments, we currently maintain a `dockerfile` that can be used to build an image for `bigchaindb`, along with a `docker-compose.yml` file to manage a "standalone node", consisting mainly of two containers: one for RethinkDB, and another for BigchainDB.
For those who like using Docker and wish to experiment with BigchainDB in
non-production environments, we currently maintain a Docker image and a
`Dockerfile` that can be used to build an image for `bigchaindb`.
Assuming you have `docker` and `docker-compose` installed, you would proceed as follows.
### Pull and Run the Image from Docker Hub
Assuming you have Docker installed, you would proceed as follows.
In a terminal shell, pull the latest version of the BigchainDB Docker image using:
```text
docker pull bigchaindb/bigchaindb:latest
```
then do a one-time configuration step to create the config file; we will use
the `-y` option to accept all the default values. The configuration file will
be stored in a file on your host machine at `~/bigchaindb_docker/.bigchaindb`:
```text
$ docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \
bigchaindb/bigchaindb:latest -y configure
Generating keypair
Configuration written to /data/.bigchaindb
Ready to go!
```
Let's analyze that command:
* `docker run` tells Docker to run some image
* `--rm` remove the container once we are done
* `-v "$HOME/bigchaindb_docker:/data"` map the host directory
`$HOME/bigchaindb_docker` to the container directory `/data`;
this allows us to have the data persisted on the host machine,
you can read more in the [official Docker
documentation](https://docs.docker.com/engine/userguide/containers/dockervolumes/#mount-a-host-directory-as-a-data-volume)
* `-t` allocate a pseudo-TTY
* `-i` keep STDIN open even if not attached
* `bigchaindb/bigchaindb:latest` the image to use
* `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values
After configuring the system, you can run BigchainDB with the following command:
```text
$ docker run -v "$HOME/bigchaindb_docker:/data" -d \
--name bigchaindb \
-p "58080:8080" -p "59984:9984" \
bigchaindb/bigchaindb:latest start
```
The command is slightly different from the previous one, the differences are:
* `-d` run the container in the background
* `--name bigchaindb` give a nice name to the container so it's easier to
refer to it later
* `-p "58080:8080"` map the host port `58080` to the container port `8080`
(the RethinkDB admin interface)
* `-p "59984:9984"` map the host port `59984` to the container port `9984`
(the BigchainDB API server)
* `start` start the BigchainDB service
Another way to publish the ports exposed by the container is to use the `-P` (or
`--publish-all`) option. This will publish all exposed ports to random ports. You can
always run `docker ps` to check the random mapping.
You can also access the RethinkDB dashboard at
[http://localhost:58080/](http://localhost:58080/)
If that doesn't work, then replace `localhost` with the IP or hostname of the
machine running the Docker engine. If you are running docker-machine (e.g. on
Mac OS X) this will be the IP of the Docker machine (`docker-machine ip
machine_name`).
#### Load Testing with Docker
Now that we have BigchainDB running in the Docker container named `bigchaindb`, we can
start another BigchainDB container to generate a load test for it.
First, make sure the container named `bigchaindb` is still running. You can check that using:
```text
docker ps
```
You should see a container named `bigchaindb` in the list.
You can load test the BigchainDB running in that container by running the `bigchaindb load` command in a second container:
```text
$ docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \
--link bigchaindb \
bigchaindb/bigchaindb:latest load
```
Note the `--link` option to link to the first container (named `bigchaindb`).
Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](bigchaindb-cli.html).
If you look at the RethinkDB dashboard (in your web browser), you should see the effects of the load test. You can also see some effects in the Docker logs using:
```text
$ docker logs -f bigchaindb
```
### Building Your Own Image
Assuming you have Docker installed, you would proceed as follows.
In a terminal shell:
```text
@ -122,41 +223,7 @@ $ git clone git@github.com:bigchaindb/bigchaindb.git
Build the Docker image:
```text
$ docker-compose build
$ docker build --tag local-bigchaindb .
```
then do a one-time configuration step to create the config file; it will be
stored on your host machine under ` ~/.bigchaindb_docker/config`:
```text
$ docker-compose run --rm bigchaindb bigchaindb configure
Starting bigchaindb_rethinkdb-data_1
Generating keypair
API Server bind? (default `localhost:9984`):
Database host? (default `localhost`): rethinkdb
Database port? (default `28015`):
Database name? (default `bigchain`):
Statsd host? (default `localhost`):
Statsd port? (default `8125`):
Statsd rate? (default `0.01`):
Ready to go!
```
As shown above, make sure that you set the database and statsd hosts to their
corresponding service names (`rethinkdb`, `statsd`), defined in`docker-compose.yml`
and `docker-compose-monitor.yml`.
You can then start it up (in the background, as a daemon) using:
```text
$ docker-compose up -d
```
then you can load test transactions via:
```text
$ docker exec -it docker-bigchaindb bigchaindb-benchmark load -m
```
If you're on Linux, you can probably view the RethinkDB dashboard at:
[http://localhost:58080/](http://localhost:58080/)
If that doesn't work, then replace `localhost` with the IP or hostname of the machine running the Docker engine. If you are running docker-machine (e.g.: on Mac OS X) this will be the IP of the Docker machine (`docker-machine ip machine_name`).
Now you can use your own image to run BigchainDB containers.

4
docs/source/monitoring.md
View File

@ -22,11 +22,11 @@ then point a browser tab to:
The login and password are `admin` by default. If BigchainDB is running and processing transactions, you should see analytics—if not, [start BigchainDB](installing-server.html#run-bigchaindb) and load some test transactions:
```text
$ bigchaindb-benchmark load
$ bigchaindb load
```
then refresh the page after a few seconds.
If you're not interested in monitoring, don't worry: BigchainDB will function just fine without any monitoring setup.
Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking!
Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking!

13
docs/source/running-unit-tests.md
View File

@ -18,16 +18,3 @@ $ python setup.py test
(Aside: How does the above command work? The documentation for [pytest-runner](https://pypi.python.org/pypi/pytest-runner) explains. We use [pytest](http://pytest.org/latest/) to write all unit tests.)
### Using docker-compose to Run the Tests
You can also use `docker-compose` to run the unit tests. (You don't have to start RethinkDB first: `docker-compose` does that on its own, when it reads the `docker-compose.yml` file.)
First, build the images (~once), using:
```text
$ docker-compose build
```
then run the unit tests using:
```text
$ docker-compose run --rm bigchaindb py.test -v
```

5
setup.py
View File

@ -65,15 +65,14 @@ setup(
entry_points={
'console_scripts': [
'bigchaindb=bigchaindb.commands.bigchain:main',
'bigchaindb-benchmark=bigchaindb.commands.bigchain_benchmark:main'
'bigchaindb=bigchaindb.commands.bigchain:main'
],
'bigchaindb.consensus': [
'default=bigchaindb.consensus:BaseConsensusRules'
]
},
install_requires=[
'rethinkdb==2.2.0.post4',
'rethinkdb==2.3.0',
'pysha3==0.3',
'pytz==2015.7',
'cryptoconditions==0.1.6',

36
tests/test_commands.py
View File

@ -1,4 +1,5 @@
import json
from unittest.mock import Mock, patch
from argparse import Namespace
from pprint import pprint
import copy
@ -62,10 +63,22 @@ def mock_bigchaindb_backup_config(monkeypatch):
def test_bigchain_run_start(mock_run_configure, mock_processes_start, mock_db_init_with_existing_db):
from bigchaindb.commands.bigchain import run_start
args = Namespace(config=None, yes=True)
args = Namespace(start_rethinkdb=False, config=None, yes=True)
run_start(args)
@patch('bigchaindb.commands.utils.start_rethinkdb')
def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb,
mock_run_configure,
mock_processes_start,
mock_db_init_with_existing_db):
from bigchaindb.commands.bigchain import run_start
args = Namespace(start_rethinkdb=True, config=None, yes=True)
run_start(args)
mock_start_rethinkdb.assert_called_with()
@pytest.mark.skipif(reason="BigchainDB doesn't support the automatic creation of a config file anymore")
def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_processes_start,
mock_generate_key_pair, mock_db_init_with_existing_db):
@ -173,7 +186,7 @@ def test_run_configure_when_config_does_not_exist(monkeypatch,
mock_bigchaindb_backup_config):
from bigchaindb.commands.bigchain import run_configure
monkeypatch.setattr('os.path.exists', lambda path: False)
monkeypatch.setattr('builtins.input', lambda question: '\n')
monkeypatch.setattr('builtins.input', lambda: '\n')
args = Namespace(config='foo', yes=True)
return_value = run_configure(args)
assert return_value is None
@ -189,9 +202,26 @@ def test_run_configure_when_config_does_exist(monkeypatch,
from bigchaindb.commands.bigchain import run_configure
monkeypatch.setattr('os.path.exists', lambda path: True)
monkeypatch.setattr('builtins.input', lambda question: '\n')
monkeypatch.setattr('builtins.input', lambda: '\n')
monkeypatch.setattr('bigchaindb.config_utils.write_config', mock_write_config)
args = Namespace(config='foo', yes=None)
run_configure(args)
assert value == {}
@patch('subprocess.Popen')
def test_start_rethinkdb_returns_a_process_when_successful(mock_popen):
from bigchaindb.commands import utils
mock_popen.return_value = Mock(stdout=['Server ready'])
assert utils.start_rethinkdb() is mock_popen.return_value
@patch('subprocess.Popen')
def test_start_rethinkdb_exits_when_cannot_start(mock_popen):
from bigchaindb import exceptions
from bigchaindb.commands import utils
mock_popen.return_value = Mock(stdout=['Nopety nope'])
with pytest.raises(exceptions.StartupError):
utils.start_rethinkdb()