Merge remote-tracking branch 'origin/develop' into feat/17/provide-a-restful-api

bigchaindb/__init__.py

@@ -1,7 +1,6 @@
 import os
 import copy
 
-from bigchaindb.core import Bigchain  # noqa
 
 
 def e(key, default=None, conv=None):
@@ -35,10 +34,17 @@ config = {
         'private': e('BIGCHAIN_KEYPAIR_PRIVATE')
     },
     'keyring': [
-    ]
+    ],
+    'statsd': {
+        'host': e('BIGCHAIN_STATSD_HOST', default='localhost'),
+        'port': e('BIGCHAIN_STATSD_PORT', default=8125),
+        'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01)
+    }
 }
 
 # We need to maintain a backup copy of the original config dict in case
 # the user wants to reconfigure the node. Check ``bigchaindb.config_utils``
 # for more info.
 _config = copy.deepcopy(config)
+from bigchaindb.core import Bigchain  # noqa
+

bigchaindb/block.py

@@ -4,11 +4,16 @@ import queue
 
 import rethinkdb as r
 
+import bigchaindb
 from bigchaindb import Bigchain
+from bigchaindb.monitor import Monitor
+
 
 
 logger = logging.getLogger(__name__)
 
+monitor = Monitor()
+
 
 class Block(object):
 
@@ -52,6 +57,7 @@ class Block(object):
         b = Bigchain()
 
         while True:
+            monitor.gauge('tx_queue_gauge', self.q_tx_to_validate.qsize(), rate=bigchaindb.config['statsd']['rate'])
             tx = self.q_tx_to_validate.get()
 
             # poison pill

bigchaindb/commands/bigchain.py

@@ -59,6 +59,10 @@ def run_configure(args, skip_if_exists=False):
             val = conf['database'][key]
             conf['database'][key] = input('Database {}? (default `{}`): '.format(key, val)) or val
 
+        for key in ('host', 'port', 'rate'):
+            val = conf['statsd'][key]
+            conf['statsd'][key] = input('Statsd {}? (default `{}`): '.format(key, val)) or val
+
     bigchaindb.config_utils.write_config(conf, config_path)
     print('Ready to go!')
 

bigchaindb/core.py

@@ -9,6 +9,10 @@ from bigchaindb import util
 from bigchaindb import config_utils
 from bigchaindb import exceptions
 from bigchaindb import crypto
+from bigchaindb.monitor import Monitor
+
+
+monitor = Monitor()
 
 
 class GenesisBlockAlreadyExistsError(Exception):
@@ -63,6 +67,7 @@ class Bigchain(object):
     def reconnect(self):
         return r.connect(host=self.host, port=self.port, db=self.dbname)
 
+    @monitor.timer('create_transaction', rate=bigchaindb.config['statsd']['rate'])
     def create_transaction(self, current_owner, new_owner, tx_input, operation, payload=None):
         """Create a new transaction
 
@@ -96,6 +101,7 @@ class Bigchain(object):
         public_key = crypto.PublicKey(public_key_base58)
         return public_key.verify(util.serialize(data), signature)
 
+    @monitor.timer('write_transaction', rate=bigchaindb.config['statsd']['rate'])
     def write_transaction(self, signed_transaction, durability='soft'):
         """Write the transaction to bigchain.
 
@@ -230,6 +236,7 @@ class Bigchain(object):
 
         return owned
 
+    @monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate'])
     def validate_transaction(self, transaction):
         """Validate a transaction.
 
@@ -336,6 +343,7 @@ class Bigchain(object):
 
         return block
 
+    @monitor.timer('validate_block')
     # TODO: check that the votings structure is correctly constructed
     def validate_block(self, block):
         """Validate a block.
@@ -379,6 +387,7 @@ class Bigchain(object):
         except Exception:
             return False
 
+    @monitor.timer('write_block')
     def write_block(self, block, durability='soft'):
         """Write a block to bigchain.
 

bigchaindb/monitor.py

@@ -0,0 +1,30 @@
+import statsd
+from platform import node
+
+import bigchaindb
+from bigchaindb import config_utils
+
+class Monitor(statsd.StatsClient):
+    """Set up statsd monitoring
+
+    """
+    def __init__(self, *args, **kwargs):
+        """Overrides statsd client, fixing prefix to messages and loading configuration
+
+        Args:
+            *args: arguments (identical to Statsclient)
+            **kwargs: keyword arguments (identical to Statsclient)
+        """
+        config_utils.autoconfigure()
+
+        if not kwargs:
+            kwargs = {}
+
+        # set prefix, parameters from configuration file
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = '{hostname}.'.format(hostname=node())
+        if 'host' not in kwargs:
+            kwargs['host'] = bigchaindb.config['statsd']['host']
+        if 'port' not in kwargs:
+            kwargs['port'] = bigchaindb.config['statsd']['port']
+        super().__init__(*args, **kwargs)

docker-compose-monitor.yml

@@ -0,0 +1,25 @@
+influxdb:
+  image: tutum/influxdb
+  ports:
+    - "8083:8083"
+    - "8086:8086"
+  expose:
+    - "8090"
+    - "8099"
+  environment:
+    PRE_CREATE_DB: "telegraf"
+
+grafana:
+  image: rhsimplex/grafana-bigchaindb-docker
+  tty: true
+  ports:
+    - "3000:3000"
+  links:
+    - influxdb:localhost
+
+statsd:
+  image: rhsimplex/docker-telegraf-statsd
+  ports:
+    - "8125:8125/udp"
+  links:
+    - influxdb:localhost

docs/source/index.rst

@@ -21,6 +21,7 @@ Table of Contents
    models
    json-serialization
    developer-interface
+   monitoring
    contributing
    faq
    release-notes

docs/source/monitoring.md

@@ -0,0 +1,22 @@
+# Monitoring
+
+BigchainDB uses [statsd](https://github.com/etsy/statsd) for monitoring.  To fully take advantage of this functionality requires some additional infrastructure: an agent to listen for metrics (e.g. [telegraf](https://github.com/influxdata/telegraf)), a time-series database (e.g. [influxdb](https://influxdata.com/time-series-platform/influxdb/)), and a frontend to display analytics (e.g. [Grafana](http://grafana.org/)).
+
+For ease of use, we've provided a docker compose file that sets up all these services for testing. Simply run in the BigchainDB directory:
+
+```text
+$ docker-compose -f docker-compose-monitor.yml build
+$ docker-compose -f docker-compose-monitor.yml up
+```
+
+and point a browser tab to `http://localhost:3000/dashboard/script/bigchaindb_dashboard.js`.  Login and password are `admin` by default.  If BigchainDB is running and processing transactions, you should see analytics—if not, [start BigchainDB](installing.html#run-bigchaindb) and load some test transactions:
+
+```text
+$ bigchaindb-benchmark load
+```
+
+and refresh the page after a few seconds.
+
+If you're not interested in monitoring, don't worry: BigchainDB will function just fine without any monitoring setup.
+
+Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking!

tests/test_commands.py

@@ -60,6 +60,7 @@ def mock_bigchaindb_backup_config(monkeypatch):
     config = {
         'keypair': {},
         'database': {'host': 'host', 'port': 12345, 'name': 'adbname'},
+        'statsd': {'host': 'host', 'port': 12345, 'rate': 0.1},
     }
     monkeypatch.setattr('bigchaindb._config', config)
 

tests/test_monitor.py

@@ -0,0 +1,14 @@
+from platform import node
+
+
+def test_monitor_class_init_defaults():
+    import bigchaindb
+    from bigchaindb.monitor import Monitor
+    monitor = Monitor()
+    assert monitor
+    assert len(monitor._addr) == 2
+    # TODO get value from config
+    # assert monitor._addr[0] == bigchaindb.config['statsd']['host']
+    assert monitor._addr[0] == '127.0.0.1'
+    assert monitor._addr[1] == bigchaindb.config['statsd']['port']
+    assert monitor._prefix == node() + '.'

tests/utils/test_config_utils.py

@@ -6,7 +6,7 @@ import bigchaindb
 from bigchaindb import exceptions
 
 
-ORIGINAL_CONFIG = copy.deepcopy(bigchaindb.config)
+ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config)
 
 
 @pytest.fixture(scope='function', autouse=True)