From 78bd1ecd965eaea34b54bf9a95ae40658829b63e Mon Sep 17 00:00:00 2001 From: tymlez Date: Tue, 24 Jan 2017 09:12:16 +0000 Subject: [PATCH 1/3] consensus plugin --- bigchaindb/config_utils.py | 40 +++++++++++++++++++ bigchaindb/consensus.py | 5 +++ bigchaindb/core.py | 9 ++++- bigchaindb/pipelines/vote.py | 10 ++++- .../source/server-reference/configuration.md | 23 +++++++++-- tests/test_config_utils.py | 28 +++++++++++++ 6 files changed, 109 insertions(+), 6 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index e678f4e9..2b0fede8 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -16,11 +16,15 @@ import copy import json import logging import collections +from functools import lru_cache + +from pkg_resources import iter_entry_points, ResolutionError from bigchaindb.common import exceptions import bigchaindb +from bigchaindb.consensus import BaseConsensusRules # TODO: move this to a proper configuration file for logging logging.getLogger('requests').setLevel(logging.WARNING) @@ -240,3 +244,39 @@ def autoconfigure(filename=None, config=None, force=False): newconfig = update(newconfig, config) set_config(newconfig) # sets bigchaindb.config + +@lru_cache() +def load_consensus_plugin(name=None): + """Find and load the chosen consensus plugin. + + Args: + name (string): the name of the entry_point, as advertised in the + setup.py of the providing package. + + Returns: + an uninstantiated subclass of ``bigchaindb.consensus.AbstractConsensusRules`` + """ + if not name: + return BaseConsensusRules; + + # TODO: This will return the first plugin with group `bigchaindb.consensus` + # and name `name` in the active WorkingSet. + # We should probably support Requirements specs in the config, e.g. + # consensus_plugin: 'my-plugin-package==0.0.1;default' + plugin = None + for entry_point in iter_entry_points('bigchaindb.consensus', name): + plugin = entry_point.load() + + # No matching entry_point found + if not plugin: + raise ResolutionError( + 'No plugin found in group `bigchaindb.consensus` with name `{}`'. + format(name)) + + # Is this strictness desireable? + # It will probably reduce developer headaches in the wild. + if not issubclass(plugin, (BaseConsensusRules)): + raise TypeError("object of type '{}' does not implement `bigchaindb." + "consensus.BaseConsensusRules`".format(type(plugin))) + + return plugin diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 9e7a11a1..be7fffd3 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -10,6 +10,11 @@ logger = logging.getLogger(__name__) class BaseConsensusRules(): """Base consensus rules for Bigchain. + + A consensus plugin must expose a class inheriting from this one via an entry_point. + + All methods listed below must be implemented. + """ @staticmethod diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 3c62e65d..38f78c77 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -56,7 +56,14 @@ class Bigchain(object): self.me_private = private_key or bigchaindb.config['keypair']['private'] self.nodes_except_me = keyring or bigchaindb.config['keyring'] self.backlog_reassign_delay = backlog_reassign_delay or bigchaindb.config['backlog_reassign_delay'] - self.consensus = BaseConsensusRules + + consensusPlugin = bigchaindb.config.get('consensus_plugin') + + if consensusPlugin: + self.consensus = config_utils.load_consensus_plugin(consensusPlugin) + else: + self.consensus = BaseConsensusRules + self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) if not self.me or not self.me_private: raise exceptions.KeypairNotFoundException() diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index dd138d41..5d1555fe 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -13,6 +13,7 @@ from bigchaindb.common import exceptions import bigchaindb from bigchaindb import Bigchain from bigchaindb import backend +from bigchaindb import config_utils from bigchaindb.backend.changefeed import ChangeFeed from bigchaindb.consensus import BaseConsensusRules from bigchaindb.models import Transaction, Block @@ -31,7 +32,14 @@ class Vote: # Since cannot share a connection to RethinkDB using multiprocessing, # we need to create a temporary instance of BigchainDB that we use # only to query RethinkDB - self.consensus = BaseConsensusRules + #self.consensus = BaseConsensusRules + + consensusPlugin = bigchaindb.config.get('consensus_plugin') + + if consensusPlugin: + self.consensus = config_utils.load_consensus_plugin(consensusPlugin) + else: + self.consensus = BaseConsensusRules # This is the Bigchain instance that will be "shared" (aka: copied) # by all the subprocesses diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index b1238ffa..68fbe940 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -23,6 +23,7 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_STATSD_RATE`
`BIGCHAINDB_CONFIG_PATH`
`BIGCHAINDB_BACKLOG_REASSIGN_DELAY`
+`BIGCHAINDB_CONSENSUS_PLUGIN`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -56,7 +57,7 @@ Internally (i.e. in the Python code), both keys have a default value of `None`, ## keyring -A list of the public keys of all the nodes in the cluster, excluding the public key of this node. +A list of the public keys of all the nodes in the cluster, excluding the public key of this node. **Example using an environment variable** ```text @@ -67,7 +68,7 @@ Note how the keys in the list are separated by colons. **Example config file snippet** ```js -"keyring": ["BnCsre9MPBeQK8QZBFznU2dJJ2GwtvnSMdemCmod2XPB", +"keyring": ["BnCsre9MPBeQK8QZBFznU2dJJ2GwtvnSMdemCmod2XPB", "4cYQHoQrvPiut3Sjs8fVR1BMZZpJjMTC4bsMTt9V71aQ"] ``` @@ -170,9 +171,23 @@ Specifies how long, in seconds, transactions can remain in the backlog before be **Example using environment variables** ```text export BIGCHAINDB_BACKLOG_REASSIGN_DELAY=30 -``` +``` **Default value (from a config file)** ```js -"backlog_reassign_delay": 120 +"backlog_reassign_delay": 120 +``` + +## consensus_plugin + +The [consensus plugin](../appendices/consensus.html) to use. + +**Example using an environment variable** +```text +export BIGCHAINDB_CONSENSUS_PLUGIN=default +``` + +**Example config file snippet: the default** +```js +"consensus_plugin": "default" ``` diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 2a326147..dcf83c02 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -39,6 +39,34 @@ def test_bigchain_instance_raises_when_not_configured(monkeypatch): with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() +def test_load_consensus_plugin_loads_default_rules_without_name(): + from bigchaindb import config_utils + from bigchaindb.consensus import BaseConsensusRules + + assert config_utils.load_consensus_plugin() == BaseConsensusRules + + +def test_load_consensus_plugin_raises_with_unknown_name(): + from pkg_resources import ResolutionError + from bigchaindb import config_utils + + with pytest.raises(ResolutionError): + config_utils.load_consensus_plugin('bogus') + + +def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): + # Monkeypatch entry_point.load to return something other than a + # ConsensusRules instance + from bigchaindb import config_utils + import time + monkeypatch.setattr(config_utils, + 'iter_entry_points', + lambda *args: [type('entry_point', (object), {'load': lambda: object})]) + + with pytest.raises(TypeError): + # Since the function is decorated with `lru_cache`, we need to + # "miss" the cache using a name that has not been used previously + config_utils.load_consensus_plugin(str(time.time())) def test_map_leafs_iterator(): from bigchaindb import config_utils From 750304b575f6543a54361cea3033a57a3cd94429 Mon Sep 17 00:00:00 2001 From: tymlez Date: Thu, 26 Jan 2017 18:36:38 +0000 Subject: [PATCH 2/3] resolve travis errors --- bigchaindb/config_utils.py | 3 ++- bigchaindb/pipelines/vote.py | 6 +++--- tests/test_config_utils.py | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 2b0fede8..8b8208ac 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -245,6 +245,7 @@ def autoconfigure(filename=None, config=None, force=False): set_config(newconfig) # sets bigchaindb.config + @lru_cache() def load_consensus_plugin(name=None): """Find and load the chosen consensus plugin. @@ -257,7 +258,7 @@ def load_consensus_plugin(name=None): an uninstantiated subclass of ``bigchaindb.consensus.AbstractConsensusRules`` """ if not name: - return BaseConsensusRules; + return BaseConsensusRules # TODO: This will return the first plugin with group `bigchaindb.consensus` # and name `name` in the active WorkingSet. diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index 5d1555fe..75187efa 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -32,17 +32,17 @@ class Vote: # Since cannot share a connection to RethinkDB using multiprocessing, # we need to create a temporary instance of BigchainDB that we use # only to query RethinkDB - #self.consensus = BaseConsensusRules consensusPlugin = bigchaindb.config.get('consensus_plugin') if consensusPlugin: - self.consensus = config_utils.load_consensus_plugin(consensusPlugin) + self.consensus = config_utils.load_consensus_plugin(consensusPlugin) else: - self.consensus = BaseConsensusRules + self.consensus = BaseConsensusRules # This is the Bigchain instance that will be "shared" (aka: copied) # by all the subprocesses + self.bigchain = Bigchain() self.last_voted_id = Bigchain().get_last_voted_block().id diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index dcf83c02..25290ee9 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -39,6 +39,7 @@ def test_bigchain_instance_raises_when_not_configured(monkeypatch): with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() + def test_load_consensus_plugin_loads_default_rules_without_name(): from bigchaindb import config_utils from bigchaindb.consensus import BaseConsensusRules @@ -68,6 +69,7 @@ def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): # "miss" the cache using a name that has not been used previously config_utils.load_consensus_plugin(str(time.time())) + def test_map_leafs_iterator(): from bigchaindb import config_utils From 0d9881fc237c9c8dba367eb598fd615a136d9350 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 22 Feb 2017 14:09:12 +0100 Subject: [PATCH 3/3] style fix --- bigchaindb/config_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 8b8208ac..87a25d3f 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -276,8 +276,8 @@ def load_consensus_plugin(name=None): # Is this strictness desireable? # It will probably reduce developer headaches in the wild. - if not issubclass(plugin, (BaseConsensusRules)): - raise TypeError("object of type '{}' does not implement `bigchaindb." - "consensus.BaseConsensusRules`".format(type(plugin))) + if not issubclass(plugin, (BaseConsensusRules,)): + raise TypeError('object of type "{}" does not implement `bigchaindb.' + 'consensus.BaseConsensusRules`'.format(type(plugin))) return plugin