bigchaindb/bigchaindb
1
0
Fork 0
mirror of https://github.com/bigchaindb/bigchaindb.git synced 2024-06-17 18:13:22 +02:00
Code Releases Activity

Merge pull request #1082 from bigchaindb/feat/1057/mongodb-configure

Browse Source 
Update configure command to support multiple backends
This commit is contained in:
Rodolphe Marques 2017-01-31 16:21:35 +01:00 committed by GitHub
commit 426b02a3f1
9 changed files with 147 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
bigchaindb/__init__.py
View File

@ -5,6 +5,25 @@ import os
# PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16
# basically, the port number is 9984
_database_rethinkdb = {
'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'),
'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'),
'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)),
'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'),
}
_database_mongodb = {
'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'),
'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'),
'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)),
'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'),
'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'),
}
_database_map = {
'mongodb': _database_mongodb,
'rethinkdb': _database_rethinkdb
}
config = {
'server': {
@ -14,13 +33,9 @@ config = {
'workers': None, # if none, the value will be cpu_count * 2 + 1
'threads': None, # if none, the value will be cpu_count * 2 + 1
},
'database': {
'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'),
'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'),
'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)),
'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'),
'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'),
},
'database': _database_map[
os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb')
],
'keypair': {
'public': None,
'private': None,

8
bigchaindb/backend/connection.py
View File

@ -40,6 +40,12 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None):
host = host or bigchaindb.config['database']['host']
port = port or bigchaindb.config['database']['port']
dbname = name or bigchaindb.config['database']['name']
# Not sure how to handle this here. This setting is only relevant for
# mongodb.
# I added **kwargs for both RethinkDBConnection and MongoDBConnection
# to handle these these additional args. In case of RethinkDBConnection
# it just does not do anything with it.
replicaset = replicaset or bigchaindb.config['database'].get('replicaset')
try:
module_name, _, class_name = BACKENDS[backend].rpartition('.')
@ -51,7 +57,7 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None):
raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc
logger.debug('Connection: {}'.format(Class))
return Class(host, port, dbname)
return Class(host, port, dbname, replicaset=replicaset)
class Connection:

2
bigchaindb/backend/mongodb/connection.py
View File

@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
class MongoDBConnection(Connection):
def __init__(self, host=None, port=None, dbname=None, max_tries=3,
replicaset=None):
replicaset=None, **kwargs):
"""Create a new Connection instance.
Args:

2
bigchaindb/backend/rethinkdb/connection.py
View File

@ -17,7 +17,7 @@ class RethinkDBConnection(Connection):
more times to run the query or open a connection.
"""
def __init__(self, host, port, dbname, max_tries=3):
def __init__(self, host, port, dbname, max_tries=3, **kwargs):
"""Create a new :class:`~.RethinkDBConnection` instance.
See :meth:`.Connection.__init__` for

15
bigchaindb/commands/bigchain.py
View File

@ -86,6 +86,11 @@ def run_configure(args, skip_if_exists=False):
conf['keypair']['private'], conf['keypair']['public'] = \
crypto.generate_key_pair()
# select the correct config defaults based on the backend
print('Generating default configuration for backend {}'
.format(args.backend))
conf['database'] = bigchaindb._database_map[args.backend]
if not args.yes:
for key in ('bind', ):
val = conf['server'][key]
@ -282,9 +287,13 @@ def create_parser():
dest='command')
# parser for writing a config file
subparsers.add_parser('configure',
help='Prepare the config file '
'and create the node keypair')
config_parser = subparsers.add_parser('configure',
help='Prepare the config file '
'and create the node keypair')
config_parser.add_argument('backend',
choices=['rethinkdb', 'mongodb'],
help='The backend to use. It can be either '
'rethinkdb or mongodb.')
# parsers for showing/exporting config values
subparsers.add_parser('show-config',

26
tests/backend/test_generics.py
View File

@ -1,6 +1,3 @@
from importlib import import_module
from unittest.mock import patch
from pytest import mark, raises
@ -69,29 +66,6 @@ def test_changefeed_class(changefeed_class_func_name, args_qty):
changefeed_class_func(None, *range(args_qty))
@mark.parametrize('db,conn_cls', (
('mongodb', 'MongoDBConnection'),
('rethinkdb', 'RethinkDBConnection'),
))
@patch('bigchaindb.backend.schema.create_indexes',
autospec=True, return_value=None)
@patch('bigchaindb.backend.schema.create_tables',
autospec=True, return_value=None)
@patch('bigchaindb.backend.schema.create_database',
autospec=True, return_value=None)
def test_init_database(mock_create_database, mock_create_tables,
mock_create_indexes, db, conn_cls):
from bigchaindb.backend.schema import init_database
conn = getattr(
import_module('bigchaindb.backend.{}.connection'.format(db)),
conn_cls,
)('host', 'port', 'dbname')
init_database(connection=conn, dbname='mickeymouse')
mock_create_database.assert_called_once_with(conn, 'mickeymouse')
mock_create_tables.assert_called_once_with(conn, 'mickeymouse')
mock_create_indexes.assert_called_once_with(conn, 'mickeymouse')
@mark.parametrize('admin_func_name,kwargs', (
('get_config', {'table': None}),
('reconfigure', {'table': None, 'shards': None, 'replicas': None}),

39
tests/commands/test_commands.py
View File

@ -12,7 +12,8 @@ def test_make_sure_we_dont_remove_any_command():
parser = create_parser()
assert parser.parse_args(['configure']).command
assert parser.parse_args(['configure', 'rethinkdb']).command
assert parser.parse_args(['configure', 'mongodb']).command
assert parser.parse_args(['show-config']).command
assert parser.parse_args(['export-my-pubkey']).command
assert parser.parse_args(['init']).command
@ -31,8 +32,8 @@ def test_start_raises_if_command_not_implemented():
with pytest.raises(NotImplementedError):
# Will raise because `scope`, the third parameter,
# doesn't contain the function `run_configure`
utils.start(parser, ['configure'], {})
# doesn't contain the function `run_start`
utils.start(parser, ['start'], {})
def test_start_raises_if_no_arguments_given():
@ -204,7 +205,7 @@ def test_run_configure_when_config_does_not_exist(monkeypatch,
from bigchaindb.commands.bigchain import run_configure
monkeypatch.setattr('os.path.exists', lambda path: False)
monkeypatch.setattr('builtins.input', lambda: '\n')
args = Namespace(config='foo', yes=True)
args = Namespace(config='foo', backend='rethinkdb', yes=True)
return_value = run_configure(args)
assert return_value is None
@ -228,6 +229,36 @@ def test_run_configure_when_config_does_exist(monkeypatch,
assert value == {}
@pytest.mark.parametrize('backend', (
'rethinkdb',
'mongodb',
))
def test_run_configure_with_backend(backend, monkeypatch, mock_write_config):
import bigchaindb
from bigchaindb.commands.bigchain import run_configure
value = {}
def mock_write_config(new_config, filename=None):
value['return'] = new_config
monkeypatch.setattr('os.path.exists', lambda path: False)
monkeypatch.setattr('builtins.input', lambda: '\n')
monkeypatch.setattr('bigchaindb.config_utils.write_config',
mock_write_config)
args = Namespace(config='foo', backend=backend, yes=True)
expected_config = bigchaindb.config
run_configure(args)
# update the expected config with the correct backend and keypair
backend_conf = getattr(bigchaindb, '_database_' + backend)
expected_config.update({'database': backend_conf,
'keypair': value['return']['keypair']})
assert value['return'] == expected_config
@patch('bigchaindb.common.crypto.generate_key_pair',
return_value=('private_key', 'public_key'))
@pytest.mark.usefixtures('ignore_local_config_file')

13
tests/conftest.py
View File

@ -109,26 +109,23 @@ def _restore_dbs(request):
@pytest.fixture(scope='session')
def _configure_bigchaindb(request):
import bigchaindb
from bigchaindb import config_utils
test_db_name = TEST_DB_NAME
# Put a suffix like _gw0, _gw1 etc on xdist processes
xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid')
if xdist_suffix:
test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix)
backend = request.config.getoption('--database-backend')
config = {
'database': {
'name': test_db_name,
'backend': request.config.getoption('--database-backend'),
},
'database': bigchaindb._database_map[backend],
'keypair': {
'private': '31Lb1ZGKTyHnmVK3LUMrAUrPNfd4sE2YyBt3UA4A25aA',
'public': '4XYfCbabAWVUCbjTmRTFEu2sc3dFEdkse4r6X498B1s8',
}
}
# FIXME
if config['database']['backend'] == 'mongodb':
# not a great way to do this
config['database']['port'] = 27017
config['database']['name'] = test_db_name
config_utils.set_config(config)

84
tests/test_config_utils.py
View File

@ -10,24 +10,32 @@ ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config)
@pytest.fixture(scope='function', autouse=True)
def clean_config(monkeypatch):
monkeypatch.setattr('bigchaindb.config', copy.deepcopy(ORIGINAL_CONFIG))
def clean_config(monkeypatch, request):
import bigchaindb
original_config = copy.deepcopy(ORIGINAL_CONFIG)
backend = request.config.getoption('--database-backend')
original_config['database'] = bigchaindb._database_map[backend]
monkeypatch.setattr('bigchaindb.config', original_config)
def test_bigchain_instance_is_initialized_when_conf_provided():
def test_bigchain_instance_is_initialized_when_conf_provided(request):
import bigchaindb
from bigchaindb import config_utils
assert 'CONFIGURED' not in bigchaindb.config
config_utils.set_config({'keypair': {'public': 'a', 'private': 'b'}})
assert bigchaindb.config['CONFIGURED'] is True
b = bigchaindb.Bigchain()
assert b.me
assert b.me_private
def test_bigchain_instance_raises_when_not_configured(monkeypatch):
def test_bigchain_instance_raises_when_not_configured(request, monkeypatch):
import bigchaindb
from bigchaindb import config_utils
from bigchaindb.common import exceptions
assert 'CONFIGURED' not in bigchaindb.config
@ -101,42 +109,64 @@ def test_env_config(monkeypatch):
def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request):
# constants
DATABASE_HOST = 'test-host'
DATABASE_NAME = 'test-dbname'
DATABASE_PORT = 4242
DATABASE_BACKEND = request.config.getoption('--database-backend')
SERVER_BIND = '1.2.3.4:56'
KEYRING = 'pubkey_0:pubkey_1:pubkey_2'
file_config = {
'database': {
'host': 'test-host',
'backend': request.config.getoption('--database-backend')
'host': DATABASE_HOST
},
'backlog_reassign_delay': 5
}
monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config)
monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname',
'BIGCHAINDB_DATABASE_PORT': '4242',
'BIGCHAINDB_SERVER_BIND': '1.2.3.4:56',
'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'})
monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': DATABASE_NAME,
'BIGCHAINDB_DATABASE_PORT': str(DATABASE_PORT),
'BIGCHAINDB_DATABASE_BACKEND': DATABASE_BACKEND,
'BIGCHAINDB_SERVER_BIND': SERVER_BIND,
'BIGCHAINDB_KEYRING': KEYRING})
import bigchaindb
from bigchaindb import config_utils
config_utils.autoconfigure()
database_rethinkdb = {
'backend': 'rethinkdb',
'host': DATABASE_HOST,
'port': DATABASE_PORT,
'name': DATABASE_NAME,
}
database_mongodb = {
'backend': 'mongodb',
'host': DATABASE_HOST,
'port': DATABASE_PORT,
'name': DATABASE_NAME,
'replicaset': 'bigchain-rs',
}
database = {}
if DATABASE_BACKEND == 'mongodb':
database = database_mongodb
elif DATABASE_BACKEND == 'rethinkdb':
database = database_rethinkdb
assert bigchaindb.config == {
'CONFIGURED': True,
'server': {
'bind': '1.2.3.4:56',
'bind': SERVER_BIND,
'workers': None,
'threads': None,
},
'database': {
'backend': request.config.getoption('--database-backend'),
'host': 'test-host',
'port': 4242,
'name': 'test-dbname',
'replicaset': 'bigchain-rs'
},
'database': database,
'keypair': {
'public': None,
'private': None,
},
'keyring': ['pubkey_0', 'pubkey_1', 'pubkey_2'],
'keyring': KEYRING.split(':'),
'statsd': {
'host': 'localhost',
'port': 8125,
@ -215,7 +245,6 @@ def test_write_config():
('BIGCHAINDB_DATABASE_HOST', 'test-host', 'host'),
('BIGCHAINDB_DATABASE_PORT', 4242, 'port'),
('BIGCHAINDB_DATABASE_NAME', 'test-db', 'name'),
('BIGCHAINDB_DATABASE_REPLICASET', 'test-replicaset', 'replicaset')
))
def test_database_envs(env_name, env_value, config_key, monkeypatch):
import bigchaindb
@ -227,3 +256,18 @@ def test_database_envs(env_name, env_value, config_key, monkeypatch):
expected_config['database'][config_key] = env_value
assert bigchaindb.config == expected_config
def test_database_envs_replicaset(monkeypatch):
# the replica set env is only used if the backend is mongodb
import bigchaindb
monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_REPLICASET':
'test-replicaset'})
bigchaindb.config['database'] = bigchaindb._database_mongodb
bigchaindb.config_utils.autoconfigure()
expected_config = copy.deepcopy(bigchaindb.config)
expected_config['database']['replicaset'] = 'test-replicaset'
assert bigchaindb.config == expected_config