diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh index cc1061fe..97cfc977 100755 --- a/.ci/travis_script.sh +++ b/.ci/travis_script.sh @@ -7,11 +7,14 @@ if [[ -n ${TOXENV} ]]; then elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then # Run the full suite of tests for MongoDB over an unsecure connection - pytest -sv --database-backend=mongodb --cov=bigchaindb + pytest -sv --database-backend=mongodb -m "serial" + pytest -sv --database-backend=mongodb --cov=bigchaindb -m "not serial" elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == mongodb && \ "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'. pytest -sv --database-backend=mongodb-ssl --cov=bigchaindb -m bdb_ssl else - pytest -sv -n auto --cov=bigchaindb + # Run the full suite of tests for RethinkDB (the default backend when testing) + pytest -sv -m "serial" + pytest -sv --cov=bigchaindb -m "not serial" fi diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 00000000..1446da07 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,26 @@ +* BigchainDB version: +* Operating System: +* Deployment Type: `[Docker|Host|IPDB|Other]` + * If you are using IPDB, please specify your network type `[test, prod]` + and the `bdb_url(BigchainDB URL)` you are using. + * For every other type of deployment, please specify the documentation/instructions + you are following. +* BigchainDB driver: `[yes|no]` + * If using a driver please specify, driver type `[python|js|java]` + and version. + +### Description + +Describe what you were trying to get done. +Tell us what happened, what went wrong, and what you expected to happen. + +### Steps to Reproduce +If you have the precise steps to reproduce, please specify. If you can reproduce +ocassionally, please provide additional information; e.g. screenshots, commands, logs, etc. + +### What I Did + +``` +Paste the command(s) you ran and the output. +If there was a crash, please include the traceback/error message here. +``` \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..31636a13 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,34 @@ +## Description +A few sentences describing the overall goals of the pull request's commits. + +## Issues This PR Fixes +Fixes #NNNN +Fixes #NNNN + +## Related PRs +List related PRs against other branches e.g. for backporting features/bugfixes +to previous release branches: + +Repo/Branch | PR +------ | ------ +some_other_PR | [link]() + + +## Todos +- [ ] Tested and working on development environment +- [ ] Unit tests (if appropriate) +- [ ] Added/Updated all related documentation. Add [link]() if different from this PR +- [ ] DevOps Support needed e.g. create Runscope API test if new endpoint added or + update deployment docs. Create a ticket and add [link]() + +## Deployment Notes +Notes about how to deploy this work. For example, running a migration against the production DB. + +## How to QA +Outline the steps to test or reproduce the PR here. + +## Impacted Areas in Application +List general components of the application that this PR will affect: +- Scale +- Performance +- Security etc. \ No newline at end of file diff --git a/.gitignore b/.gitignore index 20d71296..c58d0db8 100644 --- a/.gitignore +++ b/.gitignore @@ -77,7 +77,6 @@ ntools/one-m/ansible/hosts ntools/one-m/ansible/ansible.cfg # Just in time documentation -docs/server/source/schema docs/server/source/http-samples # Terraform state files diff --git a/CHANGELOG.md b/CHANGELOG.md index ee7240aa..65086567 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,62 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [1.3] - 2017-11-21 +Tag name: v1.3.0 + +### Added +* Metadata full-text search. [Pull request #1812](https://github.com/bigchaindb/bigchaindb/pull/1812) + +### Notes +* Improved documentation about blocks and votes. [Pull request #1855](https://github.com/bigchaindb/bigchaindb/pull/1855) + + +## [1.2] - 2017-11-13 +Tag name: v1.2.0 + +### Added +* New and improved installation setup docs and code. Pull requests [#1775](https://github.com/bigchaindb/bigchaindb/pull/1775) and [#1785](https://github.com/bigchaindb/bigchaindb/pull/1785) +* New BigchainDB configuration setting to set the port number of the log server: `log.port`. [Pull request #1796](https://github.com/bigchaindb/bigchaindb/pull/1796) +* New secondary index on `id` in the bigchain table. That will make some queries execute faster. [Pull request #1803](https://github.com/bigchaindb/bigchaindb/pull/1803) +* When using MongoDB, there are some restrictions on allowed names for keys (JSON keys). Those restrictions were always there but now BigchainDB checks key names explicitly, rather than leaving that to MongoDB. Pull requests [#1807](https://github.com/bigchaindb/bigchaindb/pull/1807) and [#1811](https://github.com/bigchaindb/bigchaindb/pull/1811) +* When using MongoDB, there are some restrictions on the allowed values of "language" (if that key is used in the values of `metadata` or `asset.data`). Those restrictions were always there but now BigchainDB checks the values explicitly, rather than leaving that to MongoDB. Pull requests [#1806](https://github.com/bigchaindb/bigchaindb/pull/1806) and [#1811](https://github.com/bigchaindb/bigchaindb/pull/1811) +* There's a new page in the root docs about permissions in BigchainDB. [Pull request #1788](https://github.com/bigchaindb/bigchaindb/pull/1788) +* There's a new option in the `bigchaindb start` command: `bigchaindb start --no-init` will avoid doing `bigchaindb init` if it wasn't done already. [Pull request #1814](https://github.com/bigchaindb/bigchaindb/pull/1814) + +### Fixed +* Fixed a bug where setting the log level in a BigchainDB config file didn't have any effect. It does now. [Pull request #1797](https://github.com/bigchaindb/bigchaindb/pull/1797) +* The docs were wrong about there being no Ping/Pong support in the Events API. There is, so the docs were fixed. [Pull request #1799](https://github.com/bigchaindb/bigchaindb/pull/1799) +* Fixed an issue with closing WebSocket connections properly. [Pull request #1819](https://github.com/bigchaindb/bigchaindb/pull/1819) + +### Notes +* Many changes were made to the Kubernetes-based production deployment template and code. + + +## [1.1] - 2017-09-26 +Tag name: v1.1.0 + +### Added +* Support for server-side plugins that can add support for alternate event consumers (other than the WebSocket API). [Pull request #1707](https://github.com/bigchaindb/bigchaindb/pull/1707) +* New configuration settings to set the *advertised* wsserver scheme, host and port. (The *advertised* ones are the ones that external users use to connect to the WebSocket API.) [Pull request #1703](https://github.com/bigchaindb/bigchaindb/pull/1703) +* Support for secure (TLS) WebSocket connections. [Pull request #1619](https://github.com/bigchaindb/bigchaindb/pull/1619) +* A new page of documentation about the contents of a condition (inside a transaction). [Pull request #1668](https://github.com/bigchaindb/bigchaindb/pull/1668) + +### Changed +* We updated our definition of the **public API** (at the top of this document). [Pull request #1700](https://github.com/bigchaindb/bigchaindb/pull/1700) +* The HTTP API Logger now logs the request path and method as well. [Pull request #1644](https://github.com/bigchaindb/bigchaindb/pull/1644) + +### External Contributors +* @carchrae - [Pull request #1731](https://github.com/bigchaindb/bigchaindb/pull/1731) +* @ivanbakel - [Pull request #1706](https://github.com/bigchaindb/bigchaindb/pull/1706) +* @ketanbhatt - Pull requests [#1643](https://github.com/bigchaindb/bigchaindb/pull/1643) and [#1644](https://github.com/bigchaindb/bigchaindb/pull/1644) + +### Notes +* New drivers & tools from our community: + * [Java driver](https://github.com/authenteq/java-bigchaindb-driver), by [Authenteq](https://authenteq.com/) + * [Ruby library](https://rubygems.org/gems/bigchaindb), by @nileshtrivedi +* Many improvements to our production deployment template (which uses Kubernetes). +* The production deployment template for the multi-node case was out of date. We updated that and verified it. [Pull request #1713](https://github.com/bigchaindb/bigchaindb/pull/1713) + ## [1.0.1] - 2017-07-13 Tag name: v1.0.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b8fc9c9..145efe95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -145,6 +145,20 @@ Once you accept and submit the CLA, we'll email you with further instructions. ( Someone will then merge your branch or suggest changes. If we suggest changes, you won't have to open a new pull request, you can just push new code to the same branch (on `origin`) as you did before creating the pull request. +### Pull Request Guidelines + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.5, and pass the flake8 check. + Check https://travis-ci.org/bigchaindb/bigchaindb-driver/pull_requests + and make sure that the tests pass for all supported Python versions. +4. Follow the pull request template while creating new PRs, the template will + be visible to you when you create a new pull request. + ### Tip: Upgrading All BigchainDB Dependencies Over time, your versions of the Python packages used by BigchainDB will get out of date. You can upgrade them using: diff --git a/LICENSES.md b/LICENSES.md index 82d62018..a62ca81e 100644 --- a/LICENSES.md +++ b/LICENSES.md @@ -1,23 +1,27 @@ # Code Licenses -For all code in this repository, BigchainDB GmbH ("We") either: +Except as noted in the **Exceptions** section below, for all code in this repository, BigchainDB GmbH ("We") either: 1. owns the copyright, or -2. owns the right to sublicense it (because all external contributors must agree to a Contributor License Agreement). +2. owns the right to sublicense it under any license (because all external contributors must agree to a Contributor License Agreement). -Therefore We can choose how to license all the code in this repository. We can license it to Joe Xname under one license and Company Yname under a different license. +Therefore We can choose how to license all the code in this repository (except for the Exceptions). We can license it to Joe Xname under one license and Company Yname under a different license. The two general options are: 1. You can get it under a commercial license for a fee. We can negotiate the terms of that license. It's not like we have some standard take-it-or-leave it commercial license. If you want to modify it and keep your modifications private, then that's certainly possible. Just ask. 2. You can get it under the AGPLv3 license for free. You don't even have to ask us. That's because all code in _this_ repository is licensed under the GNU Affero General Public License version 3 (AGPLv3), the full text of which can be found at [http://www.gnu.org/licenses/agpl.html](http://www.gnu.org/licenses/agpl.html). -If you don't like the AGPL license, then just ignore it. It doesn't affect any other license. +If you don't like the AGPL license, then contact us to get a different license. -All short code snippets embedded in the official BigchainDB _documentation_ are also licensed under the Apache License, Version 2.0, the full text of which can be found at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). +All short code snippets embedded in the official BigchainDB _documentation_ are licensed under the Apache License, Version 2.0, the full text of which can be found at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). For the licenses on all other BigchainDB-related code, see the LICENSE file in the associated repository. # Documentation Licenses The official BigchainDB documentation, _except for the short code snippets embedded within it_, is licensed under a Creative Commons Attribution-ShareAlike 4.0 International license, the full text of which can be found at [http://creativecommons.org/licenses/by-sa/4.0/legalcode](http://creativecommons.org/licenses/by-sa/4.0/legalcode). + +# Exceptions + +The contents of the `k8s/nginx-openresty/` directory are licensed as described in the `LICENSE.md` file in that directory. diff --git a/README.md b/README.md index 118d6f85..7bcde402 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,10 @@ BigchainDB is a scalable blockchain database. [The whitepaper](https://www.bigch ## Get Started with BigchainDB Server ### [Quickstart](https://docs.bigchaindb.com/projects/server/en/latest/quickstart.html) -### [Set Up & Run a Dev/Test Node](https://docs.bigchaindb.com/projects/server/en/latest/dev-and-test/setup-run-node.html) +### [Set Up & Run a Dev/Test Node](https://docs.bigchaindb.com/projects/server/en/latest/dev-and-test/index.html) ### [Run BigchainDB Server with Docker](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-docker.html) +### [Run BigchainDB Server with Vagrant](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-vagrant.html) +### [Run BigchainDB Server with Ansible](https://docs.bigchaindb.com/projects/server/en/latest/appendices/run-with-ansible.html) ## Links for Everyone diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index ad54f164..776eecba 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -16,13 +16,13 @@ except release candidates are labelled like A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server. 1. Update the `CHANGELOG.md` file in master -1. In `k8s/bigchaindb/bigchaindb-dep.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.) Commit that change to master +1. In `k8s/bigchaindb/bigchaindb-dep.yaml` AND in `k8s/dev-setup/bigchaindb.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.8.1` and change the version number to the new version number, e.g. `0.9.0`. (This is the Docker image that Kubernetes should pull from Docker Hub.) Commit that change to master 1. Create and checkout a new branch for the minor release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` (*not* 0.9.0, this new branch will be for e.g. 0.9.0, 0.9.1, 0.9.2, etc. each of which will be identified by a tagged commit) 1. Push the new branch to GitHub, e.g. `git push origin 0.9` 1. Create and checkout a new branch off of the 0.9 branch. Let's call it branch T for now 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) 1. Commit those changes, push the new branch T to GitHub, and use the pushed branch T to create a new pull request merging the T branch into the 0.9 branch. -1. Wait for all the tests to pass! +1. Wait for all the tests to pass! Then merge T into 0.9. 1. Follow steps outlined in [Common Steps](#common-steps) 1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev`. (Exception: If you just released `X.Y.Zrc1` then increment the minor version to `X.Y.Zrc2`.) This step is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available). 1. Go to [Docker Hub](https://hub.docker.com/), sign in, go to bigchaindb/bigchaindb, go to Settings - Build Settings, and under the build with Docker Tag Name equal to `latest`, change the Name to the number of the new release, e.g. `0.9` @@ -38,7 +38,7 @@ A patch release is similar to a minor release, but piggybacks on an existing min 1. Update the `CHANGELOG.md` file 1. Increment the patch version in `bigchaindb/version.py`, e.g. `0.9.1` 1. Commit that change -1. In `k8s/bigchaindb/bigchaindb-dep.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.9.0` and change the version number to the new version number, e.g. `0.9.1`. (This is the Docker image that Kubernetes should pull from Docker Hub.) +1. In `k8s/bigchaindb/bigchaindb-dep.yaml` AND in `k8s/dev-setup/bigchaindb.yaml`, find the line of the form `image: bigchaindb/bigchaindb:0.9.0` and change the version number to the new version number, e.g. `0.9.1`. (This is the Docker image that Kubernetes should pull from Docker Hub.) 1. Commit that change 1. Push the updated minor release branch to GitHub 1. Follow steps outlined in [Common Steps](#common-steps) @@ -59,8 +59,7 @@ These steps are common between minor and patch releases: 1. Make sure your local Git is in the same state as the release: e.g. `git fetch ` and `git checkout v0.9.1` 1. Make sure you have a `~/.pypirc` file containing credentials for PyPI 1. Do a `make release` to build and publish the new `bigchaindb` package on PyPI -1. [Login to readthedocs.org](https://readthedocs.org/accounts/login/) - as a maintainer of the BigchainDB Server docs, and: +1. [Login to readthedocs.org](https://readthedocs.org/accounts/login/) and go to the **BigchainDB Server** project (*not* **BigchainDB**), then: - Go to Admin --> Advanced Settings and make sure that "Default branch:" (i.e. what "latest" points to) is set to the new release's tag, e.g. `v0.9.1`. diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index a65893ec..b7d759ca 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -97,6 +97,7 @@ config = { 'fmt_console': log_config['formatters']['console']['format'], 'fmt_logfile': log_config['formatters']['file']['format'], 'granular_levels': {}, + 'port': log_config['root']['port'] }, 'graphite': { 'host': os.environ.get('BIGCHAINDB_GRAPHITE_HOST', 'localhost'), diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 673f643f..2abd770d 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -265,6 +265,16 @@ def write_assets(conn, assets): return +@register_query(MongoDBConnection) +def write_metadata(conn, metadata): + try: + return conn.run( + conn.collection('metadata') + .insert_many(metadata, ordered=False)) + except OperationError: + return + + @register_query(MongoDBConnection) def get_assets(conn, asset_ids): return conn.run( @@ -273,6 +283,14 @@ def get_assets(conn, asset_ids): projection={'_id': False})) +@register_query(MongoDBConnection) +def get_metadata(conn, txn_ids): + return conn.run( + conn.collection('metadata') + .find({'id': {'$in': txn_ids}}, + projection={'_id': False})) + + @register_query(MongoDBConnection) def count_blocks(conn): return conn.run( @@ -348,9 +366,9 @@ def get_new_blocks_feed(conn, start_block_id): @register_query(MongoDBConnection) def text_search(conn, search, *, language='english', case_sensitive=False, - diacritic_sensitive=False, text_score=False, limit=0): + diacritic_sensitive=False, text_score=False, limit=0, table='assets'): cursor = conn.run( - conn.collection('assets') + conn.collection(table) .find({'$text': { '$search': search, '$language': language, @@ -363,7 +381,7 @@ def text_search(conn, search, *, language='english', case_sensitive=False, if text_score: return cursor - return (_remove_text_score(asset) for asset in cursor) + return (_remove_text_score(obj) for obj in cursor) def _remove_text_score(asset): diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index e398560f..e55d01a8 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -27,7 +27,7 @@ def create_database(conn, dbname): @register_schema(MongoDBConnection) def create_tables(conn, dbname): - for table_name in ['bigchain', 'backlog', 'votes', 'assets']: + for table_name in ['bigchain', 'backlog', 'votes', 'assets', 'metadata']: logger.info('Create `%s` table.', table_name) # create the table # TODO: read and write concerns can be declared here @@ -40,6 +40,7 @@ def create_indexes(conn, dbname): create_backlog_secondary_index(conn, dbname) create_votes_secondary_index(conn, dbname) create_assets_secondary_index(conn, dbname) + create_metadata_secondary_index(conn, dbname) @register_schema(MongoDBConnection) @@ -50,6 +51,11 @@ def drop_database(conn, dbname): def create_bigchain_secondary_index(conn, dbname): logger.info('Create `bigchain` secondary index.') + # secondary index on block id which should be unique + conn.conn[dbname]['bigchain'].create_index('id', + name='block_id', + unique=True) + # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', ASCENDING)], @@ -116,3 +122,17 @@ def create_assets_secondary_index(conn, dbname): # full text search index conn.conn[dbname]['assets'].create_index([('$**', TEXT)], name='text') + + +def create_metadata_secondary_index(conn, dbname): + logger.info('Create `metadata` secondary index.') + + # unique index on the id of the metadata. + # the id is the txid of the transaction for which the metadata + # was specified + conn.conn[dbname]['metadata'].create_index('id', + name='transaction_id', + unique=True) + + # full text search index + conn.conn[dbname]['metadata'].create_index([('$**', TEXT)], name='text') diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 0528da3b..7c0318ce 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -254,6 +254,19 @@ def write_assets(connection, assets): raise NotImplementedError +@singledispatch +def write_metadata(connection, metadata): + """Write a list of metadata to the metadata table. + + Args: + metadata (list): a list of metadata to write. + + Returns: + The database response. + """ + raise NotImplementedError + + @singledispatch def get_assets(connection, asset_ids): """Get a list of assets from the assets table. @@ -268,6 +281,20 @@ def get_assets(connection, asset_ids): raise NotImplementedError +@singledispatch +def get_metadata(connection, txn_ids): + """Get a list of metadata from the metadata table. + + Args: + txn_ids (list): a list of ids for the metadata to be retrieved from + the database. + + Returns: + metadata (list): the list of returned metadata. + """ + raise NotImplementedError + + @singledispatch def count_blocks(connection): """Count the number of blocks in the bigchain table. @@ -360,7 +387,7 @@ def get_new_blocks_feed(connection, start_block_id): @singledispatch def text_search(conn, search, *, language='english', case_sensitive=False, - diacritic_sensitive=False, text_score=False, limit=0): + diacritic_sensitive=False, text_score=False, limit=0, table=None): """Return all the assets that match the text search. The results are sorted by text score. diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index cac9cc94..6a572d0c 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -173,6 +173,13 @@ def write_assets(connection, assets): .insert(assets, durability=WRITE_DURABILITY)) +@register_query(RethinkDBConnection) +def write_metadata(connection, metadata): + return connection.run( + r.table('metadata') + .insert(metadata, durability=WRITE_DURABILITY)) + + @register_query(RethinkDBConnection) def get_assets(connection, asset_ids): return connection.run( @@ -180,6 +187,13 @@ def get_assets(connection, asset_ids): .get_all(*asset_ids)) +@register_query(RethinkDBConnection) +def get_metadata(connection, txn_ids): + return connection.run( + r.table('metadata', read_mode=READ_MODE) + .get_all(*txn_ids)) + + @register_query(RethinkDBConnection) def count_blocks(connection): return connection.run( diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index ea6f4e25..04534857 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -23,7 +23,7 @@ def create_database(connection, dbname): @register_schema(RethinkDBConnection) def create_tables(connection, dbname): - for table_name in ['bigchain', 'backlog', 'votes', 'assets']: + for table_name in ['bigchain', 'backlog', 'votes', 'assets', 'metadata']: logger.info('Create `%s` table.', table_name) connection.run(r.db(dbname).table_create(table_name)) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index f6ce466f..80bbfa59 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -16,10 +16,17 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect +from bigchaindb.common.exceptions import ValidationError +from bigchaindb.common.utils import validate_all_values_for_key logger = logging.getLogger(__name__) -TABLES = ('bigchain', 'backlog', 'votes', 'assets') +TABLES = ('bigchain', 'backlog', 'votes', 'assets', 'metadata') +VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', + 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', + 'russian', 'spanish', 'swedish', 'turkish', 'none', + 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', + 'ro', 'ru', 'es', 'sv', 'tr') @singledispatch @@ -99,3 +106,44 @@ def init_database(connection=None, dbname=None): create_database(connection, dbname) create_tables(connection, dbname) create_indexes(connection, dbname) + + +def validate_language_key(obj, key): + """Validate all nested "language" key in `obj`. + + Args: + obj (dict): dictionary whose "language" key is to be validated. + + Returns: + None: validation successful + + Raises: + ValidationError: will raise exception in case language is not valid. + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_values_for_key(data, 'language', validate_language) + + +def validate_language(value): + """Check if `value` is a valid language. + https://docs.mongodb.com/manual/reference/text-search-languages/ + + Args: + value (str): language to validated + + Returns: + None: validation successful + + Raises: + ValidationError: will raise exception in case language is not valid. + """ + if value not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(value) + raise ValidationError(error_str) diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 146dab91..6320d279 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -196,7 +196,9 @@ def run_start(args): logger.info('RethinkDB started with PID %s' % proc.pid) try: - _run_init() + if not args.skip_initialize_database: + logger.info('Initializing database') + _run_init() except DatabaseAlreadyExists: pass except KeypairNotFoundException: @@ -300,6 +302,12 @@ def create_parser(): action='store_true', help='Run RethinkDB on start') + start_parser.add_argument('--no-init', + dest='skip_initialize_database', + default=False, + action='store_true', + help='Skip database initialization') + # parser for configuring the number of shards sharding_parser = subparsers.add_parser('set-shards', help='Configure number of shards') diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index d6840d68..11d97395 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -34,16 +34,18 @@ def configure_bigchaindb(command): """ @functools.wraps(command) def configure(args): + config_from_cmdline = None try: - config_from_cmdline = { - 'log': { - 'level_console': args.log_level, - 'level_logfile': args.log_level, - }, - 'server': {'loglevel': args.log_level}, - } + if args.log_level is not None: + config_from_cmdline = { + 'log': { + 'level_console': args.log_level, + 'level_logfile': args.log_level, + }, + 'server': {'loglevel': args.log_level}, + } except AttributeError: - config_from_cmdline = None + pass bigchaindb.config_utils.autoconfigure( filename=args.config, config=config_from_cmdline, force=True) command(args) @@ -238,10 +240,11 @@ base_parser.add_argument('-c', '--config', help='Specify the location of the configuration file ' '(use "-" for stdout)') +# NOTE: this flag should not have any default value because that will override +# the environment variables provided to configure the logger. base_parser.add_argument('-l', '--log-level', type=str.upper, # convert to uppercase for comparison to choices choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - default='INFO', help='Log level') base_parser.add_argument('-y', '--yes', '--yes-please', diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md index 3c8451b0..4b2cf873 100644 --- a/bigchaindb/common/schema/README.md +++ b/bigchaindb/common/schema/README.md @@ -3,11 +3,12 @@ This directory contains the schemas for the different JSON documents BigchainDB uses. The aim is to provide: - - a strict definition/documentation of the data structures used in BigchainDB - - a language independent tool to validate the structure of incoming/outcoming - data (there are several ready to use - [implementations](http://json-schema.org/implementations.html) written in - different languages) + +- a strict definition of the data structures used in BigchainDB +- a language independent tool to validate the structure of incoming/outcoming + data (there are several ready to use + [implementations](http://json-schema.org/implementations.html) written in + different languages) ## Learn about JSON Schema diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index fc048e0b..b2e8129a 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -13,24 +13,11 @@ from bigchaindb.common.exceptions import SchemaValidationError logger = logging.getLogger(__name__) -def drop_schema_descriptions(node): - """ Drop descriptions from schema, since they clutter log output """ - if 'description' in node: - del node['description'] - for n in node.get('properties', {}).values(): - drop_schema_descriptions(n) - for n in node.get('definitions', {}).values(): - drop_schema_descriptions(n) - for n in node.get('anyOf', []): - drop_schema_descriptions(n) - - def _load_schema(name): """ Load a schema from disk """ path = os.path.join(os.path.dirname(__file__), name + '.yaml') with open(path) as handle: schema = yaml.safe_load(handle) - drop_schema_descriptions(schema) fast_schema = rapidjson_schema.loads(rapidjson.dumps(schema)) return path, (schema, fast_schema) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index fbab5eb1..33f6a1f8 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -4,8 +4,6 @@ id: "http://www.bigchaindb.com/schema/transaction.json" type: object additionalProperties: false title: Transaction Schema -description: | - A transaction represents the creation or transfer of assets in BigchainDB. required: - id - inputs @@ -17,48 +15,24 @@ required: properties: id: "$ref": "#/definitions/sha3_hexdigest" - description: | - A sha3 digest of the transaction. The ID is calculated by removing all - derived hashes and signatures from the transaction, serializing it to - JSON with keys in sorted order and then hashing the resulting string - with sha3. operation: "$ref": "#/definitions/operation" asset: "$ref": "#/definitions/asset" - description: | - Description of the asset being transacted. - - See: `Asset`_. inputs: type: array title: "Transaction inputs" - description: | - Array of the inputs of a transaction. - - See: Input_. items: "$ref": "#/definitions/input" outputs: type: array - description: | - Array of outputs provided by this transaction. - - See: Output_. items: "$ref": "#/definitions/output" metadata: "$ref": "#/definitions/metadata" - description: | - User provided transaction metadata. This field may be ``null`` or may - contain an id and an object with freeform metadata. - - See: `Metadata`_. version: type: string pattern: "^1\\.0$" - description: | - BigchainDB transaction schema version. definitions: offset: type: integer @@ -78,53 +52,25 @@ definitions: uuid4: pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" type: string - description: | - A `UUID `_ - of type 4 (random). operation: type: string - description: | - Type of the transaction: - - A ``CREATE`` transaction creates an asset in BigchainDB. This - transaction has outputs but no inputs, so a dummy input is created. - - A ``TRANSFER`` transaction transfers ownership of an asset, by providing - an input that meets the conditions of an earlier transaction's outputs. - - A ``GENESIS`` transaction is a special case transaction used as the - sole member of the first block in a BigchainDB ledger. enum: - CREATE - TRANSFER - GENESIS asset: type: object - description: | - Description of the asset being transacted. In the case of a ``TRANSFER`` - transaction, this field contains only the ID of asset. In the case - of a ``CREATE`` transaction, this field contains only the user-defined - payload. additionalProperties: false properties: id: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID of the transaction that created the asset. data: - description: | - User provided metadata associated with the asset. May also be ``null``. anyOf: - type: object additionalProperties: true - type: 'null' output: type: object - description: | - A transaction output. Describes the quantity of an asset and the - requirements that must be met to spend the output. - - See also: Input_. additionalProperties: false required: - amount @@ -134,15 +80,7 @@ definitions: amount: type: string pattern: "^[0-9]{1,20}$" - description: | - Integral amount of the asset represented by this output. - In the case of a non divisible asset, this will always be 1. condition: - description: | - Describes the condition that needs to be met to spend the output. Has the properties: - - - **details**: Details of the condition. - - **uri**: Condition encoded as an ASCII string. type: object additionalProperties: false required: @@ -158,13 +96,8 @@ definitions: subtypes=ed25519-sha-256(&)?){2,3}$" public_keys: "$ref": "#/definitions/public_keys" - description: | - List of public keys associated with the conditions on an output. input: type: "object" - description: - An input spends a previous output, by providing one or more fulfillments - that fulfill the conditions of the previous output. additionalProperties: false required: - owners_before @@ -172,13 +105,7 @@ definitions: properties: owners_before: "$ref": "#/definitions/public_keys" - description: | - List of public keys of the previous owners of the asset. fulfillment: - description: | - Fulfillment of an `Output.condition`_, or, put a different way, a payload - that satisfies the condition of a previous output to prove that the - creator(s) of this transaction have control over the listed asset. anyOf: - type: string pattern: "^[a-zA-Z0-9_-]*$" @@ -186,8 +113,6 @@ definitions: fulfills: anyOf: - type: 'object' - description: | - Reference to the output that is being spent. additionalProperties: false required: - output_index @@ -195,26 +120,16 @@ definitions: properties: output_index: "$ref": "#/definitions/offset" - description: | - Index of the output containing the condition being fulfilled transaction_id: "$ref": "#/definitions/sha3_hexdigest" - description: | - Transaction ID containing the output to spend - type: 'null' metadata: anyOf: - type: object - description: | - User provided transaction metadata. This field may be ``null`` or may - contain an non empty object with freeform metadata. additionalProperties: true minProperties: 1 - type: 'null' condition_details: - description: | - Details needed to reconstruct the condition associated with an output. - Currently, BigchainDB only supports ed25519 and threshold condition types. anyOf: - type: object additionalProperties: false diff --git a/bigchaindb/common/schema/transaction_transfer.yaml b/bigchaindb/common/schema/transaction_transfer.yaml index b8b79696..538ec5e6 100644 --- a/bigchaindb/common/schema/transaction_transfer.yaml +++ b/bigchaindb/common/schema/transaction_transfer.yaml @@ -10,8 +10,6 @@ properties: properties: id: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID of the transaction that created the asset. required: - id inputs: diff --git a/bigchaindb/common/schema/vote.yaml b/bigchaindb/common/schema/vote.yaml index 49e5ae98..8f16562f 100644 --- a/bigchaindb/common/schema/vote.yaml +++ b/bigchaindb/common/schema/vote.yaml @@ -4,13 +4,6 @@ id: "http://www.bigchaindb.com/schema/vote.json" type: object additionalProperties: false title: Vote Schema -description: | - A Vote is an endorsement of a Block (identified by a hash) by - a node (identified by a public key). - - The outer Vote object contains the details of the vote being made - as well as the signature and identifying information of the node - passing the vote. required: - node_pubkey - signature @@ -19,18 +12,12 @@ properties: node_pubkey: type: "string" pattern: "[1-9a-zA-Z^OIl]{43,44}" - description: | - Ed25519 public key identifying the voting node. signature: type: "string" pattern: "[1-9a-zA-Z^OIl]{86,88}" - description: - Ed25519 signature of the `Vote Details`_ object. vote: type: "object" additionalProperties: false - description: | - `Vote Details`_ to be signed. required: - invalid_reason - is_block_valid @@ -40,33 +27,17 @@ properties: properties: previous_block: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID (SHA3 hash) of the block that precedes the block being voted on. - The notion of a "previous" block is subject to vote. voting_for_block: "$ref": "#/definitions/sha3_hexdigest" - description: | - ID (SHA3 hash) of the block being voted on. is_block_valid: type: "boolean" - description: | - This field is ``true`` if the block was deemed valid by the node. invalid_reason: anyOf: - type: "string" - description: | - Reason the block is voted invalid, or ``null``. - - .. container:: notice - - **Note**: The invalid_reason was not being used and may be dropped in a future version of BigchainDB. See Issue `#217 `_ on GitHub. - type: "null" timestamp: type: "string" pattern: "[0-9]{10}" - description: | - Unix timestamp that the vote was created by the node, according - to the system time of the node. definitions: sha3_hexdigest: pattern: "[0-9a-f]{64}" diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index f6f671db..9ad448f5 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -1,7 +1,10 @@ import time - +import re import rapidjson +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + def gen_timestamp(): """The Unix time, rounded to the nearest second. @@ -46,3 +49,90 @@ def deserialize(data): string. """ return rapidjson.loads(data) + + +def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validate value of `key` in `obj` using `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictionary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successful + + Raises: + ValidationError: `validation_fun` will raise exception on failure + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_keys(obj_name, data, validation_fun) + + +def validate_all_keys(obj_name, obj, validation_fun): + """Validate all (nested) keys in `obj` by using `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictionary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successful + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for key, value in obj.items(): + validation_fun(obj_name, key) + if isinstance(value, dict): + validate_all_keys(obj_name, value, validation_fun) + + +def validate_all_values_for_key(obj, key, validation_fun): + """Validate value for all (nested) occurrence of `key` in `obj` + using `validation_fun`. + + Args: + obj (dict): dictionary object. + key (str): key whose value is to be validated. + validation_fun (function): function used to validate the value + of `key`. + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for vkey, value in obj.items(): + if vkey == key: + validation_fun(value) + elif isinstance(value, dict): + validate_all_values_for_key(value, key, validation_fun) + + +def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters. + + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: validation successful + + Raises: + ValidationError: will raise exception in case of regex match. + """ + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 92552207..25b59d4d 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -192,10 +192,15 @@ class Bigchain(object): # get the asset ids from the block if block_dict: asset_ids = Block.get_asset_ids(block_dict) + txn_ids = Block.get_txn_ids(block_dict) # get the assets from the database assets = self.get_assets(asset_ids) + # get the metadata from the database + metadata = self.get_metadata(txn_ids) # add the assets to the block transactions block_dict = Block.couple_assets(block_dict, assets) + # add the metadata to the block transactions + block_dict = Block.couple_metadata(block_dict, metadata) status = None if include_status: @@ -381,8 +386,8 @@ class Bigchain(object): for transaction in transactions: # ignore transactions in invalid blocks # FIXME: Isn't there a faster solution than doing I/O again? - _, status = self.get_transaction(transaction['id'], - include_status=True) + txn, status = self.get_transaction(transaction['id'], + include_status=True) if status == self.TX_VALID: num_valid_transactions += 1 # `txid` can only have been spent in at most on valid block. @@ -392,6 +397,7 @@ class Bigchain(object): ' with the chain'.format(txid)) # if its not and invalid transaction if status is not None: + transaction.update({'metadata': txn.metadata}) non_invalid_transactions.append(transaction) if non_invalid_transactions: @@ -510,10 +516,15 @@ class Bigchain(object): # Decouple assets from block assets, block_dict = block.decouple_assets() + metadatas, block_dict = block.decouple_metadata(block_dict) + # write the assets if assets: self.write_assets(assets) + if metadatas: + self.write_metadata(metadatas) + # write the block return backend.query.write_block(self.connection, block_dict) @@ -624,6 +635,19 @@ class Bigchain(object): """ return backend.query.get_assets(self.connection, asset_ids) + def get_metadata(self, txn_ids): + """ + Return a list of metadata that match the transaction ids (txn_ids) + + Args: + txn_ids (:obj:`list` of :obj:`str`): A list of txn_ids to + retrieve from the database. + + Returns: + list: The list of metadata returned from the database. + """ + return backend.query.get_metadata(self.connection, txn_ids) + def write_assets(self, assets): """ Writes a list of assets into the database. @@ -634,7 +658,17 @@ class Bigchain(object): """ return backend.query.write_assets(self.connection, assets) - def text_search(self, search, *, limit=0): + def write_metadata(self, metadata): + """ + Writes a list of metadata into the database. + + Args: + metadata (:obj:`list` of :obj:`dict`): A list of metadata to write to + the database. + """ + return backend.query.write_metadata(self.connection, metadata) + + def text_search(self, search, *, limit=0, table='assets'): """ Return an iterator of assets that match the text search @@ -645,12 +679,13 @@ class Bigchain(object): Returns: iter: An iterator of assets that match the text search. """ - assets = backend.query.text_search(self.connection, search, limit=limit) + objects = backend.query.text_search(self.connection, search, limit=limit, + table=table) # TODO: This is not efficient. There may be a more efficient way to # query by storing block ids with the assets and using fastquery. # See https://github.com/bigchaindb/bigchaindb/issues/1496 - for asset in assets: - tx, status = self.get_transaction(asset['id'], True) + for obj in objects: + tx, status = self.get_transaction(obj['id'], True) if status == self.TX_VALID: - yield asset + yield obj diff --git a/bigchaindb/log/configs.py b/bigchaindb/log/configs.py index 034256a4..6746edf9 100644 --- a/bigchaindb/log/configs.py +++ b/bigchaindb/log/configs.py @@ -62,6 +62,7 @@ SUBSCRIBER_LOGGING_CONFIG = { 'loggers': {}, 'root': { 'level': logging.DEBUG, - 'handlers': ['console', 'file', 'errors'] + 'handlers': ['console', 'file', 'errors'], + 'port': DEFAULT_SOCKET_LOGGING_PORT }, } diff --git a/bigchaindb/log/loggers.py b/bigchaindb/log/loggers.py index f8c18320..156a4c41 100644 --- a/bigchaindb/log/loggers.py +++ b/bigchaindb/log/loggers.py @@ -22,11 +22,14 @@ class HttpServerLogger(Logger): object. *Ignored*. """ + log_cfg = self.cfg.env_orig.get('custom_log_config', {}) + self.log_port = log_cfg.get('port', DEFAULT_SOCKET_LOGGING_PORT) + self._set_socklog_handler(self.error_log) self._set_socklog_handler(self.access_log) def _set_socklog_handler(self, log): socket_handler = logging.handlers.SocketHandler( - DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT) + DEFAULT_SOCKET_LOGGING_HOST, self.log_port) socket_handler._gunicorn = True log.addHandler(socket_handler) diff --git a/bigchaindb/log/setup.py b/bigchaindb/log/setup.py index b6b45b00..61ada66f 100644 --- a/bigchaindb/log/setup.py +++ b/bigchaindb/log/setup.py @@ -25,17 +25,25 @@ def _normalize_log_level(level): raise ConfigurationError('Log level must be a string!') from exc -def setup_pub_logger(): +def setup_pub_logger(logging_port=None): + logging_port = logging_port or DEFAULT_SOCKET_LOGGING_PORT + dictConfig(PUBLISHER_LOGGING_CONFIG) socket_handler = logging.handlers.SocketHandler( - DEFAULT_SOCKET_LOGGING_HOST, DEFAULT_SOCKET_LOGGING_PORT) + DEFAULT_SOCKET_LOGGING_HOST, logging_port) socket_handler.setLevel(logging.DEBUG) logger = logging.getLogger() logger.addHandler(socket_handler) def setup_sub_logger(*, user_log_config=None): - server = LogRecordSocketServer() + kwargs = {} + log_port = user_log_config.get('port') if user_log_config is not None else None + + if log_port is not None: + kwargs['port'] = log_port + + server = LogRecordSocketServer(**kwargs) with server: server_proc = Process( target=server.serve_forever, @@ -45,7 +53,8 @@ def setup_sub_logger(*, user_log_config=None): def setup_logging(*, user_log_config=None): - setup_pub_logger() + port = user_log_config.get('port') if user_log_config is not None else None + setup_pub_logger(logging_port=port) setup_sub_logger(user_log_config=user_log_config) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 1769296f..e6d0abc0 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -5,11 +5,12 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, DoubleSpend, InputDoesNotExist, TransactionNotInValidBlock, AssetIdMismatch, AmountError, - SybilError, - DuplicateTransaction) + SybilError, DuplicateTransaction) from bigchaindb.common.transaction import Transaction -from bigchaindb.common.utils import gen_timestamp, serialize +from bigchaindb.common.utils import (gen_timestamp, serialize, + validate_txn_obj, validate_key) from bigchaindb.common.schema import validate_transaction_schema +from bigchaindb.backend.schema import validate_language_key class Transaction(Transaction): @@ -96,6 +97,9 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) + validate_txn_obj('asset', tx_body['asset'], 'data', validate_key) + validate_txn_obj('metadata', tx_body, 'metadata', validate_key) + validate_language_key(tx_body['asset'], 'data') return super().from_dict(tx_body) @classmethod @@ -121,6 +125,15 @@ class Transaction(Transaction): del asset['id'] tx_dict.update({'asset': asset}) + # get metadata of the transaction + metadata = list(bigchain.get_metadata([tx_dict['id']])) + if 'metadata' not in tx_dict: + metadata = metadata[0] if metadata else None + if metadata: + metadata = metadata.get('metadata') + + tx_dict.update({'metadata': metadata}) + return cls.from_dict(tx_dict) @@ -359,11 +372,15 @@ class Block(object): """ asset_ids = cls.get_asset_ids(block_dict) assets = bigchain.get_assets(asset_ids) + txn_ids = cls.get_txn_ids(block_dict) + metadata = bigchain.get_metadata(txn_ids) + # reconstruct block block_dict = cls.couple_assets(block_dict, assets) + block_dict = cls.couple_metadata(block_dict, metadata) kwargs = from_dict_kwargs or {} return cls.from_dict(block_dict, **kwargs) - def decouple_assets(self): + def decouple_assets(self, block_dict=None): """ Extracts the assets from the ``CREATE`` transactions in the block. @@ -372,7 +389,9 @@ class Block(object): the block being the dict of the block with no assets in the CREATE transactions. """ - block_dict = deepcopy(self.to_dict()) + if block_dict is None: + block_dict = deepcopy(self.to_dict()) + assets = [] for transaction in block_dict['block']['transactions']: if transaction['operation'] in [Transaction.CREATE, @@ -383,6 +402,27 @@ class Block(object): return (assets, block_dict) + def decouple_metadata(self, block_dict=None): + """ + Extracts the metadata from transactions in the block. + + Returns: + tuple: (metadatas, block) with the metadatas being a list of dict/null and + the block being the dict of the block with no metadata in any transaction. + """ + if block_dict is None: + block_dict = deepcopy(self.to_dict()) + + metadatas = [] + for transaction in block_dict['block']['transactions']: + metadata = transaction.pop('metadata') + if metadata: + metadata_new = {'id': transaction['id'], + 'metadata': metadata} + metadatas.append(metadata_new) + + return (metadatas, block_dict) + @staticmethod def couple_assets(block_dict, assets): """ @@ -408,6 +448,34 @@ class Block(object): transaction.update({'asset': assets.get(transaction['id'])}) return block_dict + @staticmethod + def couple_metadata(block_dict, metadatal): + """ + Given a block_dict with no metadata (as returned from a database call) + and a list of metadata, reconstruct the original block by putting the + metadata of each transaction back into its original transaction. + + NOTE: Till a transaction gets accepted the `metadata` of the transaction + is not moved outside of the transaction. So, if a transaction is found to + have metadata then it should not be overridden. + + Args: + block_dict (:obj:`dict`): The block dict as returned from a + database call. + metadata (:obj:`list` of :obj:`dict`): A list of metadata returned from + a database call. + + Returns: + dict: The dict of the reconstructed block. + """ + # create a dict with {'': metadata} + metadatal = {m.pop('id'): m.pop('metadata') for m in metadatal} + # add the metadata to their corresponding transactions + for transaction in block_dict['block']['transactions']: + metadata = metadatal.get(transaction['id'], None) + transaction.update({'metadata': metadata}) + return block_dict + @staticmethod def get_asset_ids(block_dict): """ @@ -431,6 +499,25 @@ class Block(object): return asset_ids + @staticmethod + def get_txn_ids(block_dict): + """ + Given a block_dict return all the transaction ids. + + Args: + block_dict (:obj:`dict`): The block dict as returned from a + database call. + + Returns: + list: The list of txn_ids in the block. + + """ + txn_ids = [] + for transaction in block_dict['block']['transactions']: + txn_ids.append(transaction['id']) + + return txn_ids + def to_str(self): return serialize(self.to_dict()) diff --git a/bigchaindb/processes.py b/bigchaindb/processes.py index 550342e5..f93cd5ce 100644 --- a/bigchaindb/processes.py +++ b/bigchaindb/processes.py @@ -63,7 +63,8 @@ def start(): election.start(events_queue=exchange.get_publisher_queue()) # start the web api - app_server = server.create_server(bigchaindb.config['server']) + app_server = server.create_server(settings=bigchaindb.config['server'], + log_config=bigchaindb.config['log']) p_webapi = mp.Process(name='webapi', target=app_server.run) p_webapi.start() diff --git a/bigchaindb/version.py b/bigchaindb/version.py index e20993d7..919193d2 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '1.1.0.dev' -__short_version__ = '1.1.dev' +__version__ = '1.3.0' +__short_version__ = '1.3' diff --git a/bigchaindb/web/routes.py b/bigchaindb/web/routes.py index d1fa6d26..e74c58cd 100644 --- a/bigchaindb/web/routes.py +++ b/bigchaindb/web/routes.py @@ -2,6 +2,7 @@ from flask_restful import Api from bigchaindb.web.views import ( assets, + metadata, blocks, info, statuses, @@ -27,6 +28,7 @@ def r(*args, **kwargs): ROUTES_API_V1 = [ r('/', info.ApiV1Index), r('assets/', assets.AssetListApi), + r('metadata/', metadata.MetadataApi), r('blocks/', blocks.BlockApi), r('blocks/', blocks.BlockListApi), r('statuses/', statuses.StatusApi), diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 91892807..a776afd4 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -37,6 +37,11 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication): super().__init__() def load_config(self): + # find a better way to pass this such that + # the custom logger class can access it. + custom_log_config = self.options.get('custom_log_config') + self.cfg.env_orig['custom_log_config'] = custom_log_config + config = dict((key, value) for key, value in self.options.items() if key in self.cfg.settings and value is not None) @@ -74,7 +79,7 @@ def create_app(*, debug=False, threads=1): return app -def create_server(settings): +def create_server(settings, log_config=None): """Wrap and return an application ready to be run. Args: @@ -97,6 +102,7 @@ def create_server(settings): settings['threads'] = 1 settings['logger_class'] = 'bigchaindb.log.loggers.HttpServerLogger' + settings['custom_log_config'] = log_config app = create_app(debug=settings.get('debug', False), threads=settings['threads']) standalone = StandaloneApplication(app, options=settings) diff --git a/bigchaindb/web/views/metadata.py b/bigchaindb/web/views/metadata.py new file mode 100644 index 00000000..1879fabe --- /dev/null +++ b/bigchaindb/web/views/metadata.py @@ -0,0 +1,50 @@ +"""This module provides the blueprint for some basic API endpoints. + +For more information please refer to the documentation: http://bigchaindb.com/http-api +""" +import logging + +from flask_restful import reqparse, Resource +from flask import current_app + +from bigchaindb.backend.exceptions import OperationError +from bigchaindb.web.views.base import make_error + +logger = logging.getLogger(__name__) + + +class MetadataApi(Resource): + def get(self): + """API endpoint to perform a text search on transaction metadata. + + Args: + search (str): Text search string to query the text index + limit (int, optional): Limit the number of returned documents. + + Return: + A list of metadata that match the query. + """ + parser = reqparse.RequestParser() + parser.add_argument('search', type=str, required=True) + parser.add_argument('limit', type=int) + args = parser.parse_args() + + if not args['search']: + return make_error(400, 'text_search cannot be empty') + if not args['limit']: + del args['limit'] + + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + args['table'] = 'metadata' + metadata = bigchain.text_search(**args) + + try: + # This only works with MongoDB as the backend + return list(metadata) + except OperationError as e: + return make_error( + 400, + '({}): {}'.format(type(e).__name__, e) + ) diff --git a/bigchaindb/web/websocket_server.py b/bigchaindb/web/websocket_server.py index bcf14f35..930ebc94 100644 --- a/bigchaindb/web/websocket_server.py +++ b/bigchaindb/web/websocket_server.py @@ -70,6 +70,15 @@ class Dispatcher: self.subscribers[uuid] = websocket + def unsubscribe(self, uuid): + """Remove a websocket from the list of subscribers. + + Args: + uuid (str): a unique identifier for the websocket. + """ + + del self.subscribers[uuid] + @asyncio.coroutine def publish(self): """Publish new events to the subscribers.""" @@ -115,11 +124,16 @@ def websocket_handler(request): msg = yield from websocket.receive() except RuntimeError as e: logger.debug('Websocket exception: %s', str(e)) - return websocket - - if msg.type == aiohttp.WSMsgType.ERROR: + break + if msg.type == aiohttp.WSMsgType.CLOSED: + logger.debug('Websocket closed') + break + elif msg.type == aiohttp.WSMsgType.ERROR: logger.debug('Websocket exception: %s', websocket.exception()) - return websocket + break + + request.app['dispatcher'].unsubscribe(uuid) + return websocket def init_app(event_source, *, loop=None): diff --git a/docker-compose.yml b/docker-compose.yml index 81be1c69..cd6aa2aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: ports: - "27017" command: mongod --replSet=bigchain-rs - + bdb: build: context: . @@ -22,7 +22,6 @@ services: - ./setup.cfg:/usr/src/app/setup.cfg - ./pytest.ini:/usr/src/app/pytest.ini - ./tox.ini:/usr/src/app/tox.ini - - ../cryptoconditions:/usr/src/app/cryptoconditions environment: BIGCHAINDB_DATABASE_BACKEND: mongodb BIGCHAINDB_DATABASE_HOST: mdb diff --git a/docs/root/source/bft.md b/docs/root/source/bft.md index 1f1a9bab..fce8ca3d 100644 --- a/docs/root/source/bft.md +++ b/docs/root/source/bft.md @@ -1,6 +1,8 @@ # BigchainDB and Byzantine Fault Tolerance -While BigchainDB is not currently [Byzantine fault tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance), we plan to offer it as an option. -We anticipate that turning it on will cause a severe dropoff in performance. See [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). +While BigchainDB is not currently [Byzantine fault tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance), we plan to offer it as an option. +Update Nov 2017: we're actively working on this, the next release or two will likely have support. More details to come in blog form and github issues + +Related issue: [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). We anticipate that turning on BFT will cause a dropoff in performance (for a gain in security). In the meantime, there are practical things that one can do to increase security (e.g. firewalls, key management, and access controls). diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py index 0d799fed..990fa3ce 100644 --- a/docs/root/source/conf.py +++ b/docs/root/source/conf.py @@ -34,7 +34,9 @@ from recommonmark.parser import CommonMarkParser # ones. import sphinx_rtd_theme -extensions = [] +extensions = [ + 'sphinx.ext.autosectionlabel', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst index 1dd71003..71fdd022 100644 --- a/docs/root/source/index.rst +++ b/docs/root/source/index.rst @@ -58,6 +58,9 @@ At a high level, one can communicate with a BigchainDB cluster (set of nodes) us
Python Driver Docs
+
+ JavaScript Driver Docs +
Command Line Transaction Tool
@@ -85,5 +88,6 @@ More About BigchainDB assets smart-contracts transaction-concepts + permissions timestamps Data Models diff --git a/docs/root/source/permissions.rst b/docs/root/source/permissions.rst new file mode 100644 index 00000000..d5972ac0 --- /dev/null +++ b/docs/root/source/permissions.rst @@ -0,0 +1,74 @@ +Permissions in BigchainDB +------------------------- + +BigchainDB lets users control what other users can do, to some extent. That ability resembles "permissions" in the \*nix world, "privileges" in the SQL world, and "access control" in the security world. + + +Permission to Spend/Transfer an Output +====================================== + +In BigchainDB, every output has an associated condition (crypto-condition). + +To spend/transfer an unspent output, a user (or group of users) must fulfill the condition. Another way to say that is that only certain users have permission to spend the output. The simplest condition is of the form, "Only someone with the private key corresponding to this public key can spend this output." Much more elaborate conditions are possible, e.g. "To spend this output, …" + +- "…anyone in the Accounting Group can sign." +- "…three of these four people must sign." +- "…either Bob must sign, or both Tom and Sylvia must sign." + +For details, see `the documentation about conditions in BigchainDB `_. + +Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output. + + +Write Permissions +================= + +When someone builds a TRANSFER transaction, they can put an arbitrary JSON object in the ``metadata`` field (within reason; real BigchainDB networks put a limit on the size of transactions). That is, they can write just about anything they want in a TRANSFER transaction. + +Does that mean there are no "write permissions" in BigchainDB? Not at all! + +A TRANSFER transaction will only be valid (allowed) if its inputs fulfill some previous outputs. The conditions on those outputs will control who can build valid TRANSFER transactions. In other words, one can interpret the condition on an output as giving "write permissions" to certain users to write something into the history of the associated asset. + +As a concrete example, you could use BigchainDB to write a public journal where only you have write permissions. Here's how: First you'd build a CREATE transaction with the ``asset.data`` being something like ``{"title": "The Journal of John Doe"}``, with one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. +Each time you want to append something to your journal, you'd build a new TRANSFER transaction with your latest entry in the ``metadata`` field, e.g. + +.. code-block:: json + + {"timestamp": "1508319582", + "entry": "I visited Marmot Lake with Jane."} + +The TRANSFER transaction would have one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. And so on. Only you would be able to append to the history of that asset (your journal). + +The same technique could be used for scientific notebooks, supply-chain records, government meeting minutes, and so on. + +You could do more elaborate things too. As one example, each time someone writes a TRANSFER transaction, they give *someone else* permission to spend it, setting up a sort of writers-relay or chain letter. + +.. note:: + + Anyone can write any JSON (again, within reason) in the ``asset.data`` field of a CREATE transaction. They don't need permission. + + +Read Permissions +================ + +All the data stored in a BigchainDB network can be read by anyone with access to that network. One *can* store encrypted data, but if the decryption key ever leaks out, then the encrypted data can be read, decrypted, and leak out too. (Deleting the encrypted data is :doc:`not an option `.) + +The permission to read some specific information (e.g. a music file) can be thought of as an *asset*. (In many countries, that permission or "right" is a kind of intellectual property.) +BigchainDB can be used to register that asset and transfer it from owner to owner. +Today, BigchainDB does not have a way to restrict read access of data stored in a BigchainDB network, but many third-party services do offer that (e.g. Google Docs, Dropbox). +In principle, a third party service could ask a BigchainDB network to determine if a particular user has permission to read some particular data. Indeed they could use BigchainDB to keep track of *all* the rights a user has for some data (not just the right to read it). +That third party could also use BigchainDB to store audit logs, i.e. records of every read, write or other operation on stored data. + +BigchainDB can be used in other ways to help parties exchange private data: + +- It can be used to publicly disclose the *availability* of some private data (stored elsewhere). For example, there might be a description of the data and a price. +- It can be used to record the TLS handshakes which two parties sent to each other to establish an encrypted and authenticated TLS connection, which they could use to exchange private data with each other. (The stored handshake information wouldn't be enough, by itself, to decrypt the data.) It would be a "proof of TLS handshake." +- See the BigchainDB `Privacy Protocols repository `_ for more techniques. + + +Role-Based Access Control (RBAC) +================================ + +In September 2017, we published a `blog post about how one can define an RBAC sub-system on top of BigchainDB `_. +At the time of writing (October 2017), doing so required the use of a plugin, so it's not possible using standard BigchainDB (which is what's available on `IPDB `_). That may change in the future. +If you're interested, `contact BigchainDB `_. diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py deleted file mode 100644 index c94fe3a9..00000000 --- a/docs/server/generate_schema_documentation.py +++ /dev/null @@ -1,241 +0,0 @@ -""" Script to render transaction schema into .rst document """ - -from collections import OrderedDict -import os.path - -import yaml - -from bigchaindb.common.schema import TX_SCHEMA_PATH, VOTE_SCHEMA_PATH - - -TPL_PROP = """\ -%(title)s -%(underline)s - -**type:** %(type)s - -%(description)s -""" - - -TPL_STYLES = """ -.. raw:: html - - -""" - - -TPL_TRANSACTION = TPL_STYLES + """\ -.. This file was auto generated by %(file)s - -================== -Transaction Schema -================== - -* `Transaction`_ - -* Input_ - -* Output_ - -* Asset_ - -* Metadata_ - - -Transaction ------------ - -%(transaction)s - -Input ------ - -%(input)s - -Output ------- - -%(output)s - -Asset ------ - -%(asset)s - -Metadata --------- - -%(metadata)s -""" - - -def generate_transaction_docs(): - schema = load_schema(TX_SCHEMA_PATH) - defs = schema['definitions'] - - doc = TPL_TRANSACTION % { - 'transaction': render_section('Transaction', schema), - 'output': render_section('Output', defs['output']), - 'input': render_section('Input', defs['input']), - 'asset': render_section('Asset', defs['asset']), - 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), - 'container': 'transaction-schema', - 'file': os.path.basename(__file__), - } - - write_schema_doc('transaction', doc) - - -TPL_VOTE = TPL_STYLES + """\ -.. This file was auto generated by %(file)s - -=========== -Vote Schema -=========== - -Vote ----- - -%(vote)s - -Vote Details ------------- - -%(vote_details)s - -""" - - -def generate_vote_docs(): - schema = load_schema(VOTE_SCHEMA_PATH) - - doc = TPL_VOTE % { - 'vote': render_section('Vote', schema), - 'vote_details': render_section('Vote', schema['properties']['vote']), - 'container': 'vote-schema', - 'file': os.path.basename(__file__), - } - - write_schema_doc('vote', doc) - - -def ordered_load_yaml(path): - """ Custom YAML loader to preserve key order """ - class OrderedLoader(yaml.SafeLoader): - pass - - def construct_mapping(loader, node): - loader.flatten_mapping(node) - return OrderedDict(loader.construct_pairs(node)) - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping) - with open(path) as handle: - return yaml.load(handle, OrderedLoader) - - -def load_schema(path): - global DEFS - schema = ordered_load_yaml(path) - DEFS = schema['definitions'] - return schema - - -def write_schema_doc(name, doc): - # Check base path exists - base_path = os.path.join(os.path.dirname(__file__), 'source/schema') - if not os.path.exists(base_path): - os.makedirs(base_path) - # Write doc - path = os.path.join(base_path, '%s.rst' % name) - with open(path, 'w') as handle: - handle.write(doc) - - -def render_section(section_name, obj): - """ Render a domain object and it's properties """ - out = [obj['description']] - for name, prop in obj.get('properties', {}).items(): - try: - title = '%s.%s' % (section_name, name) - out += [TPL_PROP % { - 'title': title, - 'underline': '^' * len(title), - 'description': property_description(prop), - 'type': property_type(prop), - }] - except Exception as exc: - raise ValueError('Error rendering property: %s' % name, exc) - return '\n\n'.join(out + ['']) - - -def property_description(prop): - """ Get description of property """ - if 'description' in prop: - return prop['description'] - if '$ref' in prop: - return property_description(resolve_ref(prop['$ref'])) - if 'anyOf' in prop: - return property_description(prop['anyOf'][0]) - raise KeyError('description') - - -def property_type(prop): - """ Resolve a string representing the type of a property """ - if 'type' in prop: - if prop['type'] == 'array': - return 'array (%s)' % property_type(prop['items']) - return prop['type'] - if 'anyOf' in prop: - return ' or '.join(property_type(p) for p in prop['anyOf']) - if '$ref' in prop: - return property_type(resolve_ref(prop['$ref'])) - raise ValueError('Could not resolve property type') - - -DEFINITION_BASE_PATH = '#/definitions/' - - -def resolve_ref(ref): - """ Resolve definition reference """ - assert ref.startswith(DEFINITION_BASE_PATH) - return DEFS[ref[len(DEFINITION_BASE_PATH):]] - - -def main(): - """ Main function """ - generate_transaction_docs() - generate_vote_docs() - - -def setup(*_): - """ Fool sphinx into think it's an extension muahaha """ - main() - - -if __name__ == '__main__': - main() diff --git a/docs/server/requirements.txt b/docs/server/requirements.txt index 4321f44b..cd06eab9 100644 --- a/docs/server/requirements.txt +++ b/docs/server/requirements.txt @@ -3,3 +3,5 @@ recommonmark>=0.4.0 sphinx-rtd-theme>=0.1.9 sphinxcontrib-napoleon>=0.4.4 sphinxcontrib-httpdomain>=1.5.0 +pyyaml>=3.12 +bigchaindb diff --git a/docs/server/source/_static/arch.jpg b/docs/server/source/_static/arch.jpg new file mode 100644 index 00000000..570d63c1 Binary files /dev/null and b/docs/server/source/_static/arch.jpg differ diff --git a/docs/server/source/_static/cloud9.pdf b/docs/server/source/_static/cloud9.pdf deleted file mode 100644 index 22c6891e..00000000 Binary files a/docs/server/source/_static/cloud9.pdf and /dev/null differ diff --git a/docs/server/source/appendices/docker-on-mac.md b/docs/server/source/appendices/docker-on-mac.md deleted file mode 100644 index 7f87540f..00000000 --- a/docs/server/source/appendices/docker-on-mac.md +++ /dev/null @@ -1,101 +0,0 @@ -# Run BigchainDB with Docker On Mac - -**NOT for Production Use** - -Those developing on Mac can follow this document to run BigchainDB in docker -containers for a quick dev setup. -Running BigchainDB on Mac (Docker or otherwise) is not officially supported. - -Support is very much limited as there are certain things that work differently -in Docker for Mac than Docker for other platforms. -Also, we do not use mac for our development and testing. :) - -This page may not be up to date with various settings and docker updates at -all the times. - -These steps work as of this writing (2017.Mar.09) and might break in the -future with updates to Docker for mac. -Community contribution to make BigchainDB run on Docker for Mac will always be -welcome. - - -## Prerequisite - -Install Docker for Mac. - -## (Optional) For a clean start - -1. Stop all BigchainDB and RethinkDB/MongoDB containers. -2. Delete all BigchainDB docker images. -3. Delete the ~/bigchaindb_docker folder. - - -## Pull the images - -Pull the bigchaindb and other required docker images from docker hub. - -```text -docker pull bigchaindb/bigchaindb:master -docker pull [rethinkdb:2.3|mongo:3.4.1] -``` - -## Create the BigchainDB configuration file on Mac -```text -docker run \ - --rm \ - --volume $HOME/bigchaindb_docker:/data \ - bigchaindb/bigchaindb:master \ - -y configure \ - [mongodb|rethinkdb] -``` - -To ensure that BigchainDB connects to the backend database bound to the virtual -interface `172.17.0.1`, you must edit the BigchainDB configuration file -(`~/bigchaindb_docker/.bigchaindb`) and change database.host from `localhost` -to `172.17.0.1`. - - -## Run the backend database on Mac - -From v0.9 onwards, you can run RethinkDB or MongoDB. - -We use the virtual interface created by the Docker daemon to allow -communication between the BigchainDB and database containers. -It has an IP address of 172.17.0.1 by default. - -You can also use docker host networking or bind to your primary (eth) -interface, if needed. - -### For RethinkDB backend -```text -docker run \ - --name=rethinkdb \ - --publish=28015:28015 \ - --publish=8080:8080 \ - --restart=always \ - --volume $HOME/bigchaindb_docker:/data \ - rethinkdb:2.3 -``` - -### For MongoDB backend -```text -docker run \ - --name=mongodb \ - --publish=27017:27017 \ - --restart=always \ - --volume=$HOME/bigchaindb_docker/db:/data/db \ - --volume=$HOME/bigchaindb_docker/configdb:/data/configdb \ - mongo:3.4.1 --replSet=bigchain-rs -``` - -### Run BigchainDB on Mac -```text -docker run \ - --name=bigchaindb \ - --publish=9984:9984 \ - --restart=always \ - --volume=$HOME/bigchaindb_docker:/data \ - bigchaindb/bigchaindb \ - start -``` - diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index 864b32e4..6f560458 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -10,7 +10,6 @@ Appendices install-os-level-deps install-latest-pip run-with-docker - docker-on-mac json-serialization cryptography the-Bigchain-class @@ -27,3 +26,7 @@ Appendices rethinkdb-backup licenses install-with-lxd + run-with-vagrant + run-with-ansible + tx-yaml-files + vote-yaml diff --git a/docs/server/source/appendices/rethinkdb-backup.md b/docs/server/source/appendices/rethinkdb-backup.md index 732323ed..dc33231e 100644 --- a/docs/server/source/appendices/rethinkdb-backup.md +++ b/docs/server/source/appendices/rethinkdb-backup.md @@ -22,8 +22,6 @@ That's just one possible way of setting up the file system so as to provide extr Another way to get similar reliability would be to mount the RethinkDB data directory on an [Amazon EBS](https://aws.amazon.com/ebs/) volume. Each Amazon EBS volume is, "automatically replicated within its Availability Zone to protect you from component failure, offering high availability and durability." -See [the section on setting up storage for RethinkDB](../dev-and-test/setup-run-node.html#set-up-storage-for-rethinkdb-data) for more details. - As with shard replication, live file-system replication protects against many failure modes, but it doesn't protect against them all. You should still consider having normal, "cold" backups. @@ -108,7 +106,7 @@ Considerations for BigchainDB: Although it's not advertised as such, RethinkDB's built-in replication feature is similar to continous backup, except the "backup" (i.e. the set of replica shards) is spread across all the nodes. One could take that idea a bit farther by creating a set of backup-only servers with one full backup: -* Give all the original BigchainDB nodes (RethinkDB nodes) the server tag `original`. This is the default if you used the RethinkDB config file suggested in the section titled [Configure RethinkDB Server](../dev-and-test/setup-run-node.html#configure-rethinkdb-server). +* Give all the original BigchainDB nodes (RethinkDB nodes) the server tag `original`. * Set up a group of servers running RethinkDB only, and give them the server tag `backup`. The `backup` servers could be geographically separated from all the `original` nodes (or not; it's up to the consortium to decide). * Clients shouldn't be able to read from or write to servers in the `backup` set. * Send a RethinkDB reconfigure command to the RethinkDB cluster to make it so that the `original` set has the same number of replicas as before (or maybe one less), and the `backup` set has one replica. Also, make sure the `primary_replica_tag='original'` so that all primary shards live on the `original` nodes. diff --git a/docs/server/source/appendices/run-with-ansible.md b/docs/server/source/appendices/run-with-ansible.md new file mode 100644 index 00000000..b1ec6769 --- /dev/null +++ b/docs/server/source/appendices/run-with-ansible.md @@ -0,0 +1,167 @@ +# Run BigchainDB with Ansible + +**NOT for Production Use** + +You can use the following instructions to deploy a single or multi node +BigchainDB setup for dev/test using Ansible. Ansible will setup BigchainDB node(s) along with +[Docker](https://www.docker.com/), [Docker Compose](https://docs.docker.com/compose/), +[MongoDB](https://www.mongodb.com/), [BigchainDB Python driver](https://docs.bigchaindb.com/projects/py-driver/en/latest/). + +Currently, this workflow is only supported for the following distributions: +- Ubuntu >= 16.04 +- CentOS >= 7 +- Fedora >= 24 + +## Minimum Requirements | Ansible +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 +## Clone the BigchainDB repository | Ansible +```text +$ git clone https://github.com/bigchaindb/bigchaindb.git +``` + +## Install dependencies | Ansible +- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) + +You can also install `ansible` and other dependencies, if any, using the `boostrap.sh` script +inside the BigchainDB repository. +Navigate to `bigchaindb/pkg/scripts` and run the `bootstrap.sh` script to install the dependecies +for your OS. The script also checks if the OS you are running is compatible with the +supported versions. + +**Note**: `bootstrap.sh` only supports Ubuntu >= 16.04, CentOS >= 7 and Fedora >=24. + +```text +$ cd bigchaindb/pkg/scripts/ +$ sudo ./bootstrap.sh +``` + +### BigchainDB Setup Configuration(s) | Ansible +#### Local Setup | Ansible +You can run the Ansible playbook `bdb-deploy.yml` on your local dev machine and set up the BigchainDB node where +BigchainDB can be run as a process or inside a Docker container(s) depending on your configuration. + +Before, running the playbook locally, you need to update the `hosts` and `bdb-config.yml` configuration, which will notify Ansible that we need to run the play locally. + +##### Update Hosts | Local +Navigate to `bigchaindb/pkg/configuration/hosts` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/hosts +``` + +Edit `all` configuration file: +```text +# Delete any existing configuration in this file and insert +# Hostname of dev machine + ansible_connection=local +``` +##### Update Configuration | Local +Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/vars/bdb-config.yml +``` + +Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): +```text +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 # Only needed if `deploy_docker` is true +bdb_hosts: + - name: "" # Hostname of dev machine +``` +**Note**: You can also orchestrate a multi-node BigchainDB cluster on a local dev host using Docker containers. +Here is a sample `bdb-config.yml` +```text +--- +deploy_docker: true #[true, false] +docker_cluster_size: 3 +bdb_hosts: + - name: "" +``` + +### BigchainDB Setup | Ansible +Now, You can safely run the `bdb-deploy.yml` playbook and everything will be taken care of by `Ansible`. To run the playbook please navigate to the `bigchaindb/pkg/configuration` directory inside the BigchainDB repository and run the `bdb-deploy.yml` playbook. + +```text +$ cd bigchaindb/pkg/configuration/ + +$ sudo ansible-playbook bdb-deploy.yml -i hosts/all +``` + +After successful execution of the playbook, you can verify that BigchainDB docker(s)/process(es) is(are) running. + +Verify BigchainDB process(es): +```text +$ ps -ef | grep bigchaindb +``` + +OR + +Verify BigchainDB Docker(s): +```text +$ docker ps | grep bigchaindb +``` + +The playbook also installs the BigchainDB Python Driver, +so you can use it to make transactions +and verify the functionality of your BigchainDB node. +See the [BigchainDB Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) +for details on how to use it. + + +**Note**: The `bdb_root_url` can be be one of the following: +```text +# BigchainDB is running as a process +bdb_root_url = http://:9984 + +OR + +# BigchainDB is running inside a docker container +bdb_root_url = http://: +``` + +**Note**: BigchainDB has [other drivers as well](../drivers-clients/index.html). + +### Experimental: Running Ansible a Remote Dev/Host +#### Remote Setup | Ansible +You can also run the Ansible playbook `bdb-deploy.yml` on remote machine(s) and set up the BigchainDB node where +BigchainDB can run as a process or inside a Docker container(s) depending on your configuration. + +Before, running the playbook on a remote host, you need to update the `hosts` and `bdb-config.yml` configuration, which will notify Ansible that we need to +run the play on a remote host. + +##### Update Hosts | Remote +Navigate to `bigchaindb/pkg/configuration/hosts` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/hosts +``` + +Edit `all` configuration file: +```text +# Delete any existing configuration in this file and insert + ansible_ssh_user= ansible_sudo_pass= +``` + +**Note**: You can add multiple hosts to the `all` configuration file. Root password is needed because ansible +will run some tasks that require root permissions. + +**Note**: You can also use other methods to get inside the remote machines instead of password based SSH. For other methods +please consult [Ansible Documentation](http://docs.ansible.com/ansible/latest/intro_getting_started.html). + +##### Update Configuration | Remote +Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/vars/bdb-config.yml +``` + +Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): +```text +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 # Only needed if `deploy_docker` is true +bdb_hosts: + - name: "" +``` + +After, the configuration of remote hosts, [run the Ansible playbook and verify your deployment](#bigchaindb-setup-ansible). diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index d80f58b1..b75c9300 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -6,9 +6,12 @@ For those who like using Docker and wish to experiment with BigchainDB in non-production environments, we currently maintain a Docker image and a `Dockerfile` that can be used to build an image for `bigchaindb`. +## Prerequisite(s) +- [Docker](https://docs.docker.com/engine/installation/) + ## Pull and Run the Image from Docker Hub -Assuming you have Docker installed, you would proceed as follows. +With Docker installed, you can proceed as follows. In a terminal shell, pull the latest version of the BigchainDB Docker image using: ```text @@ -26,6 +29,7 @@ docker run \ --rm \ --tty \ --volume $HOME/bigchaindb_docker:/data \ + --env BIGCHAINDB_DATABASE_HOST=172.17.0.1 \ bigchaindb/bigchaindb \ -y configure \ [mongodb|rethinkdb] @@ -46,24 +50,18 @@ Let's analyze that command: this allows us to have the data persisted on the host machine, you can read more in the [official Docker documentation](https://docs.docker.com/engine/tutorials/dockervolumes) +* `--env BIGCHAINDB_DATABASE_HOST=172.17.0.1`, `172.17.0.1` is the default `docker0` bridge +IP address, for fresh Docker installations. It is used for the communication between BigchainDB and database +containers. * `bigchaindb/bigchaindb` the image to use. All the options after the container name are passed on to the entrypoint inside the container. * `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values * `mongodb` or `rethinkdb` specifies the database backend to use with bigchaindb -To ensure that BigchainDB connects to the backend database bound to the virtual -interface `172.17.0.1`, you must edit the BigchainDB configuration file -(`~/bigchaindb_docker/.bigchaindb`) and change database.host from `localhost` -to `172.17.0.1`. - ### Run the backend database From v0.9 onwards, you can run either RethinkDB or MongoDB. -We use the virtual interface created by the Docker daemon to allow -communication between the BigchainDB and database containers. -It has an IP address of 172.17.0.1 by default. - You can also use docker host networking or bind to your primary (eth) interface, if needed. @@ -73,8 +71,8 @@ You can also use docker host networking or bind to your primary (eth) docker run \ --detach \ --name=rethinkdb \ - --publish=172.17.0.1:28015:28015 \ - --publish=172.17.0.1:58080:8080 \ + --publish=28015:28015 \ + --publish=58080:8080 \ --restart=always \ --volume $HOME/bigchaindb_docker:/data \ rethinkdb:2.3 @@ -102,11 +100,11 @@ group. docker run \ --detach \ --name=mongodb \ - --publish=172.17.0.1:27017:27017 \ + --publish=27017:27017 \ --restart=always \ --volume=$HOME/mongodb_docker/db:/data/db \ --volume=$HOME/mongodb_docker/configdb:/data/configdb \ - mongo:3.4.1 --replSet=bigchain-rs + mongo:3.4.9 --replSet=bigchain-rs ``` ### Run BigchainDB diff --git a/docs/server/source/appendices/run-with-vagrant.md b/docs/server/source/appendices/run-with-vagrant.md new file mode 100644 index 00000000..8bde0a38 --- /dev/null +++ b/docs/server/source/appendices/run-with-vagrant.md @@ -0,0 +1,170 @@ +# Run BigchainDB with Vagrant + +**NOT for Production Use** + +You can use the following instructions to deploy a single or multi node +BigchainDB setup for dev/test using Vagrant. Vagrant will set up the BigchainDB node(s) +with all the dependencies along with MongoDB and BigchainDB Python driver. You +can also tweak the following configurations for the BigchainDB node(s). +- Vagrant Box + - Currently, we support the following boxes: + - `ubuntu/xenial64 # >=16.04` + - `centos/7 # >=7` + - `fedora/24 # >=24` + - **NOTE** : You can choose any other vagrant box of your choice but these are + the minimum versioning requirements. +- Resources and specs for your box. + - RAM + - VCPUs + - Network Type + - Currently, only `private_network` is supported. + - IP Address +- Deploy node with Docker + - Deploy all the services in Docker containers or as processes. +- Number of BigchainDB nodes + - If you want to deploy the services inside Docker containers, you + can specify number of member(s) in the BigchainDB cluster. +- Upstart Script +- Vagrant Provider + - Virtualbox + - VMware + +## Minimum Requirements | Vagrant +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 + +## Install dependencies | Vagrant +1. [VirtualBox](https://www.virtualbox.org/wiki/Downloads) >= 5.0.0 +2. [Vagrant](https://www.vagrantup.com/downloads.html) >= 1.16.0 + +## Clone the BigchainDB repository | Vagrant +```text +$ git clone https://github.com/bigchaindb/bigchaindb.git +``` + +## Configuration | Vagrant +Navigate to `bigchaindb/pkg/configuration/vars/` inside the BigchainDB repository. +```text +$ cd bigchaindb/pkg/configuration/vars/ +``` + +Edit `bdb-config.yml` as per your requirements. Sample `bdb-config.yml`: + +```text +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.40" + type: "private_network" +``` + +**Note**: You can spawn multiple instances to orchestrate a multi-node BigchainDB cluster. +Here is a sample `bdb-config.yml`: +```text +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.40" + type: "private_network" + - name: "bdb-node-02" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.50" + type: "private_network" +``` +**Note**: You can also orchestrate a multi-node BigchainDB cluster on a single dev host using Docker containers. +Here is a sample `bdb-config.yml` +```text +--- +deploy_docker: true #[true, false] +docker_cluster_size: 3 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "8192" + vcpus: "4" + network: + ip: "10.20.30.40" + type: "private_network" +``` +The above mentioned configuration will deploy a 3 node BigchainDB cluster with Docker containers +on your specified host. + +## BigchainDB Setup | Vagrant + +**Note**: There are some vagrant plugins required for the installation, +user will be prompted to install them if they are not present. To install +the required plugins, run the following command: + +```text +$ vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts +``` + +To bring up the BigchainDB node(s), run the following command: + +```text +$ vagrant up +``` + +After successful execution of Vagrant, you can log in to your fresh BigchainDB node. + +```text +$ vagrant ssh +``` + +## Make your first transaction +Once you are inside the BigchainDB node, you can verify that BigchainDB +docker(s)/process(es) is(are) running. + +Verify BigchainDB process(es): +```text +$ ps -ef | grep bigchaindb +``` + +OR + +Verify BigchainDB Docker(s): +```text +$ docker ps | grep bigchaindb +``` + +The BigchainDB Python Driver is pre-installed in the instance, +so you can use it to make transactions +and verify the functionality of your BigchainDB node. +See the [BigchainDB Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) +for details on how to use it. + +Note 1: The `bdb_root_url` can be one of the following: +```text +# BigchainDB is running as a process +bdb_root_url = http://:9984 + +OR + +# BigchainDB is running inside a docker container +bdb_root_url = http://: +``` + +Note 2: BigchainDB has [other drivers as well](../drivers-clients/index.html). diff --git a/docs/server/source/appendices/tx-yaml-files.rst b/docs/server/source/appendices/tx-yaml-files.rst new file mode 100644 index 00000000..f9a51685 --- /dev/null +++ b/docs/server/source/appendices/tx-yaml-files.rst @@ -0,0 +1,37 @@ +The Transaction Schema Files +============================ + +BigchainDB checks all :ref:`transactions ` +(JSON documents) against a formal schema +defined in some JSON Schema files named +transaction.yaml, +transaction_create.yaml and +transaction_transfer.yaml. +The contents of those files are copied below. +To understand those contents +(i.e. JSON Schema), check out +`"Understanding JSON Schema" +`_ +by Michael Droettboom or +`json-schema.org `_. + + +transaction.yaml +---------------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/transaction.yaml + :language: yaml + + +transaction_create.yaml +----------------------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/transaction_create.yaml + :language: yaml + + +transaction_transfer.yaml +------------------------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/transaction_transfer.yaml + :language: yaml diff --git a/docs/server/source/appendices/vote-yaml.rst b/docs/server/source/appendices/vote-yaml.rst new file mode 100644 index 00000000..6613827e --- /dev/null +++ b/docs/server/source/appendices/vote-yaml.rst @@ -0,0 +1,20 @@ +The Vote Schema File +==================== + +BigchainDB checks all :ref:`votes ` +(JSON documents) against a formal schema +defined in a JSON Schema file named vote.yaml. +The contents of that file are copied below. +To understand those contents +(i.e. JSON Schema), check out +`"Understanding JSON Schema" +`_ +by Michael Droettboom or +`json-schema.org `_. + + +vote.yaml +--------- + +.. literalinclude:: ../../../../bigchaindb/common/schema/vote.yaml + :language: yaml diff --git a/docs/server/source/conf.py b/docs/server/source/conf.py index 756a8d13..e272baf4 100644 --- a/docs/server/source/conf.py +++ b/docs/server/source/conf.py @@ -51,7 +51,6 @@ extensions = [ 'sphinx.ext.autosectionlabel', # Below are actually build steps made to look like sphinx extensions. # It was the easiest way to get it running with ReadTheDocs. - 'generate_schema_documentation', 'generate_http_server_api_documentation', ] diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index eefa81bb..054a4083 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -2,6 +2,8 @@ To avoid redundant data in transactions, the asset model is different for `CREATE` and `TRANSFER` transactions. +## In CREATE Transactions + In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair. The key must be `"data"` and the value can be any valid JSON document, or `null`. For example: ```json { @@ -12,6 +14,15 @@ In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair } ``` +When using MongoDB for storage, certain restriction apply to all (including nested) keys of the `"data"` JSON document: + +* Keys (i.e. key names, not values) must **not** begin with the `$` character. +* Keys must not contain `.` or the null character (Unicode code point 0000). +* The key `"language"` (at any level in the hierarchy) is a special key and used for specifying text search language. Its value must be one of the allowed values; see the valid [Text Search Languages](https://docs.mongodb.com/manual/reference/text-search-languages/) in the MongoDB Docs. In BigchainDB, only the languages supported by _MongoDB community edition_ are allowed. + + +## In TRANSFER Transactions + In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pair. They key must be `"id"` and the value must contain a transaction ID (i.e. a SHA3-256 hash: the ID of the `CREATE` transaction which created the asset, which also serves as the asset ID). For example: ```json { diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index 8b184261..36b7204e 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -1,36 +1,90 @@ The Block Model =============== -A block has the following structure: +A block is a JSON object with a particular schema, +as outlined in this page. +A block must contain the following JSON keys +(also called names or fields): .. code-block:: json { - "id": "", + "id": "", "block": { - "timestamp": "", - "transactions": [""], - "node_pubkey": "", - "voters": [""] + "timestamp": "", + "transactions": [""], + "node_pubkey": "", + "voters": [""] }, - "signature": "" + "signature": "" } -- ``id``: The :ref:`hash ` of the serialized inner ``block`` (i.e. the ``timestamp``, ``transactions``, ``node_pubkey``, and ``voters``). It's used as a unique index in the database backend (e.g. RethinkDB or MongoDB). +The JSON Keys in a Block +------------------------ -- ``block``: - - ``timestamp``: The Unix time when the block was created. It's provided by the node that created the block. - - ``transactions``: A list of the transactions included in the block. - - ``node_pubkey``: The public key of the node that created the block. - - ``voters``: A list of the public keys of all cluster nodes at the time the block was created. - It's the list of nodes which can cast a vote on this block. - This list can change from block to block, as nodes join and leave the cluster. +**id** -- ``signature``: :ref:`Cryptographic signature ` of the block by the node that created the block (i.e. the node with public key ``node_pubkey``). To generate the signature, the node signs the serialized inner ``block`` (the same thing that was hashed to determine the ``id``) using the private key corresponding to ``node_pubkey``. +The transaction ID and also the SHA3-256 hash +of the inner ``block`` object, loosely speaking. +It's a string. +To compute it, 1) construct an :term:`associative array` ``d`` containing +``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, +``block.voters``, and their values. 2) compute ``id = hash_of_aa(d)``. +There's pseudocode for the ``hash_of_aa()`` function +in the `IPDB Protocol documentation page about cryptographic hashes +`_. +The result (``id``) is a string: the block ID. +An example is ``"b60adf655932bf47ef58c0bfb2dd276d4795b94346b36cbb477e10d7eb02cea8"`` -Working with Blocks -------------------- +**block.timestamp** -There's a **Block** class for creating and working with Block objects; look in `/bigchaindb/models.py `_. (The link is to the latest version on the master branch on GitHub.) +The `Unix time `_ +when the block was created, according to the node which created it. +It's a string representation of an integer. +An example is ``"1507294217"``. + + +**block.transactions** + +A list of the :ref:`transactions ` included in the block. +(Each transaction is a JSON object.) + + +**block.node_pubkey** + +The public key of the node that created the block. +It's a string. +See the `IPDB Protocol documentation page about cryptographic keys & signatures +`_. + + +**block.voters** + +A list of the public keys of all cluster nodes at the time the block was created. +It's a list of strings. +This list can change from block to block, as nodes join and leave the cluster. + + +**signature** + +The cryptographic signature of the inner ``block`` +by the node that created the block +(i.e. the node with public key ``node_pubkey``). +To compute that: + +#. Construct an :term:`associative array` ``d`` containing the contents + of the inner ``block`` + (i.e. ``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, + ``block.voters``, and their values). +#. Compute ``signature = sig_of_aa(d, private_key)``, + where ``private_key`` is the node's private key + (i.e. ``node_pubkey`` and ``private_key`` are a key pair). There's pseudocode + for the ``sig_of_aa()`` function + on `the IPDB Protocol documentation page about cryptographic keys and signatures + `_. + +.. note:: + + The ``d_bytes`` computed when computing the block ID will be the *same* as the ``d_bytes`` computed when computing the block signature. This can be used to avoid redundant calculations. diff --git a/docs/server/source/data-models/inputs-outputs.rst b/docs/server/source/data-models/inputs-outputs.rst index 4e246bfe..52214421 100644 --- a/docs/server/source/data-models/inputs-outputs.rst +++ b/docs/server/source/data-models/inputs-outputs.rst @@ -26,7 +26,6 @@ An input has the following structure: You can think of the ``fulfills`` object as a pointer to an output on another transaction: the output that this input is spending/transferring. A CREATE transaction should have exactly one input. That input can contain one or more ``owners_before``, a ``fulfillment`` (with one signature from each of the owners-before), and the value of ``fulfills`` should be ``null``). A TRANSFER transaction should have at least one input, and the value of ``fulfills`` should not be ``null``. -See the reference on :ref:`inputs ` for more description about the meaning of each field. The ``fulfillment`` string fulfills the condition in the output that is being spent (transferred). To calculate it: @@ -62,7 +61,6 @@ An output has the following structure: The :ref:`page about conditions ` explains the contents of a ``condition``. The list of ``public_keys`` is always the "owners" of the asset at the time the transaction completed, but before the next transaction started. -See the reference on :ref:`outputs ` for more description about the meaning of each field. Note that ``amount`` must be a string (e.g. ``"7"``). In a TRANSFER transaction, the sum of the output amounts must be the same as the sum of the outputs that it transfers (i.e. the sum of the input amounts). For example, if a TRANSFER transaction has two outputs, one with ``"amount": "2"`` and one with ``"amount": "3"``, then the sum of the outputs is 5 and so the sum of the outputs-being-transferred must also be 5. diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 38e523bd..631399e2 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -19,20 +19,18 @@ Here's some explanation of the contents: - **id**: The ID of the transaction and also the hash of the transaction (loosely speaking). See below for an explanation of how it's computed. It's also the database primary key. -- **version**: The version-number of :ref:`the transaction schema `. As of BigchainDB Server 1.0.0, the only allowed value is ``"1.0"``. +- **version**: The version-number of the transaction schema. As of BigchainDB Server 1.0.0, the only allowed value is ``"1.0"``. - **inputs**: List of inputs. Each input spends/transfers a previous output by satisfying/fulfilling the crypto-conditions on that output. A CREATE transaction should have exactly one input. A TRANSFER transaction should have at least one input (i.e. ≥1). - For more details, see the subsection about :ref:`inputs `. - **outputs**: List of outputs. Each output indicates the crypto-conditions which must be satisfied by anyone wishing to spend/transfer that output. It also indicates the number of shares of the asset tied to that output. - For more details, see the subsection about :ref:`outputs `. - **operation**: A string indicating what kind of transaction this is, and how it should be validated. @@ -46,6 +44,10 @@ Here's some explanation of the contents: - **metadata**: User-provided transaction metadata. It can be any valid JSON document, or ``null``. + **NOTE:** When using MongoDB for storage, certain restriction apply + to all (including nested) keys of the ``"data"`` JSON document: + 1) keys (i.e. key names, not values) must **not** begin with the ``$`` character, and + 2) keys must not contain ``.`` or the null character (Unicode code point 0000). **How the transaction ID is computed.** 1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, @@ -60,3 +62,13 @@ There are example BigchainDB transactions in :ref:`the HTTP API documentation ` and `the Python Driver documentation `_. + + +The Transaction Schema +---------------------- + +BigchainDB checks all transactions (JSON documents) +against a formal schema defined in :ref:`some JSON Schema files named +transaction.yaml, +transaction_create.yaml and +transaction_transfer.yaml `. diff --git a/docs/server/source/data-models/vote-model.md b/docs/server/source/data-models/vote-model.md deleted file mode 100644 index daa66a94..00000000 --- a/docs/server/source/data-models/vote-model.md +++ /dev/null @@ -1,27 +0,0 @@ -# The Vote Model - -A vote has the following structure: - -```json -{ - "node_pubkey": "", - "vote": { - "voting_for_block": "", - "previous_block": "", - "is_block_valid": "", - "invalid_reason": null, - "timestamp": "" - }, - "signature": "" -} -``` - -**Notes** - -* Votes have no ID (or `"id"`), as far as users are concerned. (The backend database uses one internally, but it's of no concern to users and it's never reported to them via BigchainDB APIs.) - -* At the time of writing, the value of `"invalid_reason"` was always `null`. In other words, it wasn't being used. It may be used or dropped in a future version of BigchainDB. See [Issue #217](https://github.com/bigchaindb/bigchaindb/issues/217) on GitHub. - -* For more information about the vote `"timestamp"`, see [the page about timestamps in BigchainDB](https://docs.bigchaindb.com/en/latest/timestamps.html). - -* For more information about how the `"signature"` is calculated, see [the page about cryptography in BigchainDB](../appendices/cryptography.html). diff --git a/docs/server/source/data-models/vote-model.rst b/docs/server/source/data-models/vote-model.rst new file mode 100644 index 00000000..45db9680 --- /dev/null +++ b/docs/server/source/data-models/vote-model.rst @@ -0,0 +1,121 @@ +The Vote Model +============== + +A vote is a JSON object with a particular schema, +as outlined in this page. +A vote must contain the following JSON keys +(also called names or fields): + +.. code-block:: json + + { + "node_pubkey": "", + "vote": { + "voting_for_block": "", + "previous_block": "", + "is_block_valid": "", + "invalid_reason": null, + "timestamp": "" + }, + "signature": "" + } + +.. note:: + + Votes have no ID (or ``"id"``), as far as users are concerned. + The backend database may use one internally, + but it's of no concern to users and it's never reported to them via APIs. + + +The JSON Keys in a Vote +----------------------- + +**node_pubkey** + +The public key of the node which cast this vote. +It's a string. +For more information about public keys, +see the `IPDB Protocol documentation page about cryptographic keys and signatures +`_. + + +**vote.voting_for_block** + +The block ID that this vote is for. +It's a string. +For more information about block IDs, +see the page about :ref:`blocks `. + + +**vote.previous_block** + +The block ID of the block "before" the block that this vote is for, +according to the node which cast this vote. +It's a string. +(It's possible for different nodes to see different block orders.) +For more information about block IDs, +see the page about :ref:`blocks `. + + +**vote.is_block_valid** + +``true`` if the node which cast this vote considered the block in question to be valid, +and ``false`` otherwise. +Note that it's a *boolean* (i.e. ``true`` or ``false``), not a string. + + +**vote.invalid_reason** + +Always ``null``, that is, it's not being used. +It may be used or dropped in a future version. +See `bigchaindb/bigchaindb issue #217 +`_ on GitHub. + + +**vote.timestamp** + +The `Unix time `_ +when the vote was created, according to the node which created it. +It's a string representation of an integer. + + +**signature** + +The cryptographic signature of the inner ``vote`` +by the node that created the vote +(i.e. the node with public key ``node_pubkey``). +To compute that: + +#. Construct an :term:`associative array` ``d`` containing the contents of the inner ``vote`` + (i.e. ``vote.voting_for_block``, ``vote.previous_block``, ``vote.is_block_valid``, + ``vote.invalid_reason``, ``vote.timestamp``, and their values). +#. Compute ``signature = sig_of_aa(d, private_key)``, where ``private_key`` + is the node's private key (i.e. ``node_pubkey`` and ``private_key`` are a key pair). + There's pseudocode for the ``sig_of_aa()`` function + on `the IPDB Protocol documentation page about cryptographic keys and signatures + `_. + + +The Vote Schema +--------------- + +BigchainDB checks all votes (JSON documents) against a formal schema +defined in a :ref:`JSON Schema file named vote.yaml `. + + +An Example Vote +--------------- + +.. code-block:: json + + { + "node_pubkey": "3ZCsVWPAhPTqHx9wZVxp9Se54pcNeeM5mQvnozDWyDR9", + "vote": { + "voting_for_block": "11c3a3fcc9efa4fc4332a0849fc39b58e403ff37794a7d1fdfb9e7703a94a274", + "previous_block": "3dd1441018b782a50607dc4c7f83a0f0a23eb257f4b6a8d99330dfff41271e0d", + "is_block_valid": true, + "invalid_reason": null, + "timestamp": "1509977988" + }, + "signature": "3tW2EBVgxaZTE6nixVd9QEQf1vUxqPmQaNAMdCHc7zHik5KEosdkwScGYt4VhiHDTB6BCxTUzmqu3P7oP93tRWfj" + } diff --git a/docs/server/source/dev-and-test/index.rst b/docs/server/source/dev-and-test/index.rst index f79c20a5..b9ddaf58 100644 --- a/docs/server/source/dev-and-test/index.rst +++ b/docs/server/source/dev-and-test/index.rst @@ -1,8 +1,13 @@ Develop & Test BigchainDB Server ================================ +This section outlines some ways that you could set up a minimal BigchainDB node for development and testing purposes. For additional guidance on how you could help develop BigchainDB, see the `CONTRIBUTING.md file on GitHub `_. + .. toctree:: :maxdepth: 1 - setup-run-node - running-all-tests + Using a Local Dev Machine + Using a Local Dev Machine and Docker <../appendices/run-with-docker> + Using Vagrant <../appendices/run-with-vagrant> + Using Ansible <../appendices/run-with-ansible> + running-all-tests \ No newline at end of file diff --git a/docs/server/source/dev-and-test/setup-bdb-host.md b/docs/server/source/dev-and-test/setup-bdb-host.md new file mode 100644 index 00000000..cdee3c0b --- /dev/null +++ b/docs/server/source/dev-and-test/setup-bdb-host.md @@ -0,0 +1,61 @@ +# Set Up BigchainDB Node on Local Dev Machine + +The BigchainDB core dev team develops BigchainDB on recent Ubuntu, Fedora and CentOS distributions, so we recommend you use one of those. BigchainDB Server doesn't work on Windows or macOS (unless you use a VM or containers). + + +## With MongoDB + +First read the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to set up a machine for developing and testing BigchainDB. + +Create a default BigchainDB config file (in `$HOME/.bigchaindb`): +```text +$ bigchaindb -y configure mongodb +``` + +Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) + +Start MongoDB __3.4+__ using: +```text +$ mongod --replSet=bigchain-rs +``` + +You can verify that MongoDB is running correctly by checking the output of the +previous command for the line: +```text +waiting for connections on port 27017 +``` + +To run BigchainDB Server, do: +```text +$ bigchaindb start +``` + +You can [run all the unit tests](running-all-tests.html) to test your installation. + + +## With RethinkDB + +First read the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to set up a machine for developing and testing BigchainDB. + +Create a default BigchainDB config file (in `$HOME/.bigchaindb`): +```text +$ bigchaindb -y configure rethinkdb +``` + +Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) + +Start RethinkDB using: +```text +$ rethinkdb +``` + +You can verify that RethinkDB is running by opening the RethinkDB web interface in your web browser. It should be at http://localhost:8080/ + + + +To run BigchainDB Server, do: +```text +$ bigchaindb start +``` + +You can [run all the unit tests](running-all-tests.html) to test your installation. diff --git a/docs/server/source/dev-and-test/setup-run-node.md b/docs/server/source/dev-and-test/setup-run-node.md deleted file mode 100644 index b9271cf8..00000000 --- a/docs/server/source/dev-and-test/setup-run-node.md +++ /dev/null @@ -1,189 +0,0 @@ -# Set Up & Run a Dev/Test Node - -This page explains how to set up a minimal local BigchainDB node for development and testing purposes. - -The BigchainDB core dev team develops BigchainDB on recent Ubuntu and Fedora distributions, so we recommend you use one of those. BigchainDB Server doesn't work on Windows and Mac OS X (unless you use a VM or containers). - - -## Option A: Using a Local Dev Machine - -Read through the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to setup a machine for developing and testing BigchainDB. - -### With RethinkDB - -Create a default BigchainDB config file (in `$HOME/.bigchaindb`): -```text -$ bigchaindb -y configure rethinkdb -``` - -Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) - -Start RethinkDB using: -```text -$ rethinkdb -``` - -You can verify that RethinkDB is running by opening the RethinkDB web interface in your web browser. It should be at http://localhost:8080/ - - - -To run BigchainDB Server, do: -```text -$ bigchaindb start -``` - -You can [run all the unit tests](running-unit-tests.html) to test your installation. - -The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md) has more details about how to contribute. - - -### With MongoDB - -Create a default BigchainDB config file (in `$HOME/.bigchaindb`): -```text -$ bigchaindb -y configure mongodb -``` - -Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) - -Start MongoDB __3.4+__ using: -```text -$ mongod --replSet=bigchain-rs -``` - -You can verify that MongoDB is running correctly by checking the output of the -previous command for the line: -```text -waiting for connections on port 27017 -``` - -To run BigchainDB Server, do: -```text -$ bigchaindb start -``` - -You can [run all the unit tests](running-unit-tests.html) to test your installation. - -The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md) has more details about how to contribute. - - -## Option B: Using a Local Dev Machine and Docker - -You need to have recent versions of [Docker Engine](https://docs.docker.com/engine/installation/) -and (Docker) [Compose](https://docs.docker.com/compose/install/). - -Build the images: - -```bash -docker-compose build -``` - -### Docker with RethinkDB - -**Note**: If you're upgrading BigchainDB and have previously already built the images, you may need -to rebuild them after the upgrade to install any new dependencies. - -Start RethinkDB: - -```bash -docker-compose -f docker-compose.rdb.yml up -d rdb -``` - -The RethinkDB web interface should be accessible at http://localhost:58080/. -Depending on which platform, and/or how you are running docker, you may need -to change `localhost` for the `ip` of the machine that is running docker. As a -dummy example, if the `ip` of that machine was `0.0.0.0`, you would access the -web interface at: http://0.0.0.0:58080/. - -Start a BigchainDB node: - -```bash -docker-compose -f docker-compose.rdb.yml up -d bdb-rdb -``` - -You can monitor the logs: - -```bash -docker-compose -f docker-compose.rdb.yml logs -f bdb-rdb -``` - -If you wish to run the tests: - -```bash -docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb pytest -v -n auto -``` - -### Docker with MongoDB - -Start MongoDB: - -```bash -docker-compose up -d mdb -``` - -MongoDB should now be up and running. You can check the port binding for the -MongoDB driver port using: -```bash -$ docker-compose port mdb 27017 -``` - -Start a BigchainDB node: - -```bash -docker-compose up -d bdb -``` - -You can monitor the logs: - -```bash -docker-compose logs -f bdb -``` - -If you wish to run the tests: - -```bash -docker-compose run --rm bdb py.test -v --database-backend=mongodb -``` - -### Accessing the HTTP API - -You can do quick check to make sure that the BigchainDB server API is operational: - -```bash -curl $(docker-compose port bdb 9984) -``` - -The result should be a JSON object (inside braces like { }) -containing the name of the software ("BigchainDB"), -the version of BigchainDB, the node's public key, and other information. - -How does the above curl command work? Inside the Docker container, BigchainDB -exposes the HTTP API on port `9984`. First we get the public port where that -port is bound: - -```bash -docker-compose port bdb 9984 -``` - -The port binding will change whenever you stop/restart the `bdb` service. You -should get an output similar to: - -```bash -0.0.0.0:32772 -``` - -but with a port different from `32772`. - - -Knowing the public port we can now perform a simple `GET` operation against the -root: - -```bash -curl 0.0.0.0:32772 -``` - -## Option C: Using a Dev Machine on Cloud9 - -Ian Worrall of [Encrypted Labs](http://www.encryptedlabs.com/) wrote a document (PDF) explaining how to set up a BigchainDB (Server) dev machine on Cloud9: - -[Download that document from GitHub](https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/docs/server/source/_static/cloud9.pdf) diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 382e68b6..407fe688 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -23,6 +23,6 @@ Community-Driven Libraries and Tools * `Haskell transaction builder `_ * `Go driver `_ -* `Java driver `_ +* `Java driver `_ * `Ruby driver `_ * `Ruby library for preparing/signing transactions and submitting them or querying a BigchainDB/IPDB node (MIT licensed) `_ diff --git a/docs/server/source/events/websocket-event-stream-api.rst b/docs/server/source/events/websocket-event-stream-api.rst index 0000107b..7eb3f3f5 100644 --- a/docs/server/source/events/websocket-event-stream-api.rst +++ b/docs/server/source/events/websocket-event-stream-api.rst @@ -40,11 +40,14 @@ response contains a ``streams`` property: Connection Keep-Alive --------------------- -The Event Stream API initially does not provide any mechanisms for connection -keep-alive other than enabling TCP keepalive on each open WebSocket connection. -In the future, we may add additional functionality to handle ping/pong frames -or payloads designed for keep-alive. +The Event Stream API supports Ping/Pong frames as descibed in +`RFC 6455 `_. +.. note:: + + It might not be possible to send PING/PONG frames via web browsers because + of non availability of Javascript API on different browsers to achieve the + same. Streams ------- diff --git a/docs/server/source/glossary.rst b/docs/server/source/glossary.rst new file mode 100644 index 00000000..8fc03ac2 --- /dev/null +++ b/docs/server/source/glossary.rst @@ -0,0 +1,19 @@ +Glossary +======== + +.. glossary:: + :sorted: + + associative array + A collection of key/value (or name/value) pairs + such that each possible key appears at most once + in the collection. + In JavaScript (and JSON), all objects behave as associative arrays + with string-valued keys. + In Python and .NET, associative arrays are called *dictionaries*. + In Java and Go, they are called *maps*. + In Ruby, they are called *hashes*. + See also: Wikipedia's articles for + `Associative array `_ + and + `Comparison of programming languages (associative array) `_ diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 618903a8..58ec5617 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -452,6 +452,118 @@ Assets text search. +Transaction Metadata +-------------------------------- + +.. http:get:: /api/v1/metadata + + Return all the metadata that match a given text search. + + :query string text search: Text search string to query. + :query int limit: (Optional) Limit the number of returned metadata objects. Defaults + to ``0`` meaning return all matching objects. + + .. note:: + + Currently this enpoint is only supported if the server is running + MongoDB as the backend. + +.. http:get:: /api/v1/metadata/?search={text_search} + + Return all metadata that match a given text search. The ``id`` of the metadata + is the same ``id`` of the transaction where it was defined. + + If no metadata match the text search it returns an empty list. + + If the text string is empty or the server does not support text search, + a ``400`` is returned. + + The results are sorted by text score. + For more information about the behavior of text search see `MongoDB text + search behavior `_ + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/metadata/?search=bigchaindb HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "metadata": {"metakey1": "Hello BigchainDB 1!"}, + "id": "51ce82a14ca274d43e4992bbce41f6fdeb755f846e48e710a3bbb3b0cf8e4204" + }, + { + "metadata": {"metakey2": "Hello BigchainDB 2!"}, + "id": "b4e9005fa494d20e503d916fa87b74fe61c079afccd6e084260674159795ee31" + }, + { + "metadata": {"metakey3": "Hello BigchainDB 3!"}, + "id": "fa6bcb6a8fdea3dc2a860fcdc0e0c63c9cf5b25da8b02a4db4fb6a2d36d27791" + } + ] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully. + :statuscode 400: The query was not executed successfully. Returned if the + text string is empty or the server does not support + text search. + +.. http:get:: /api/v1/metadata/?search={text_search}&limit={n_documents} + + Return at most ``n`` metadata objects that match a given text search. + + If no metadata match the text search it returns an empty list. + + If the text string is empty or the server does not support text search, + a ``400`` is returned. + + The results are sorted by text score. + For more information about the behavior of text search see `MongoDB text + search behavior `_ + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/metadata/?search=bigchaindb&limit=2 HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + { + "metadata": {"msg": "Hello BigchainDB 1!"}, + "id": "51ce82a14ca274d43e4992bbce41f6fdeb755f846e48e710a3bbb3b0cf8e4204" + }, + { + "metadata": {"msg": "Hello BigchainDB 2!"}, + "id": "b4e9005fa494d20e503d916fa87b74fe61c079afccd6e084260674159795ee31" + }, + ] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: The query was executed successfully. + :statuscode 400: The query was not executed successfully. Returned if the + text string is empty or the server does not support + text search. + + Advanced Usage -------------------------------- diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index ec6efef0..65bd8774 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -16,7 +16,6 @@ BigchainDB Server Documentation events/index drivers-clients/index data-models/index - schema/transaction - schema/vote release-notes + glossary appendices/index diff --git a/docs/server/source/introduction.md b/docs/server/source/introduction.md index 58d7de16..1ef47922 100644 --- a/docs/server/source/introduction.md +++ b/docs/server/source/introduction.md @@ -15,14 +15,13 @@ Note that there are a few kinds of nodes: ## Setup Instructions for Various Cases -* [Set up a local stand-alone BigchainDB node for learning and experimenting: Quickstart](quickstart.html) -* [Set up and run a local dev/test node for developing and testing BigchainDB Server](dev-and-test/setup-run-node.html) +* [Quickstart](quickstart.html) +* [Set up a local BigchainDB node for development, experimenting and testing](dev-and-test/index.html) * [Set up and run a BigchainDB cluster](clusters.html) There are some old RethinkDB-based deployment instructions as well: * [Deploy a bare-bones RethinkDB-based node on Azure](appendices/azure-quickstart-template.html) -* [Deploy a bare-bones RethinkDB-based node on any Ubuntu machine with Ansible](appendices/template-ansible.html) * [Deploy a RethinkDB-based testing cluster on AWS](appendices/aws-testing-cluster.html) Instructions for setting up a client will be provided once there's a public test net. diff --git a/docs/server/source/production-deployment-template/architecture.rst b/docs/server/source/production-deployment-template/architecture.rst new file mode 100644 index 00000000..beb03d7e --- /dev/null +++ b/docs/server/source/production-deployment-template/architecture.rst @@ -0,0 +1,75 @@ +Architecture of an IPDB Node +============================ + +An IPDB Production deployment is hosted on a Kubernetes cluster and includes: + +* NGINX, OpenResty, BigchainDB and MongoDB + `Kubernetes Services `_. +* NGINX, OpenResty, BigchainDB, Monitoring Agent and Backup Agent + `Kubernetes Deployments `_. +* MongoDB `Kubernetes StatefulSet `_. +* Third party services like `3scale `_, + `MongoDB Cloud Manager `_ and the + `Azure Operations Management Suite + `_. + +.. image:: ../_static/arch.jpg + +.. note:: + The arrows in the diagram represent the client-server communication. For + example, A-->B implies that A initiates the connection to B. + It does not represent the flow of data; the communication channel is always + fully duplex. + + +NGINX +----- + +We use an NGINX as HTTP proxy on port 443 (configurable) at the cloud +entrypoint for: + +#. Rate Limiting: We configure NGINX to allow only a certain number of requests + (configurable) which prevents DoS attacks. + +#. HTTPS Termination: The HTTPS connection does not carry through all the way + to BigchainDB and terminates at NGINX for now. + +#. Request Routing: For HTTPS connections on port 443 (or the configured BigchainDB public api port), + the connection is proxied to: + + #. OpenResty Service if it is a POST request. + #. BigchainDB Service if it is a GET request. + + +We use an NGINX TCP proxy on port 27017 (configurable) at the cloud +entrypoint for: + +#. Rate Limiting: We configure NGINX to allow only a certain number of requests + (configurable) which prevents DoS attacks. + +#. Request Routing: For connections on port 27017 (or the configured MongoDB + public api port), the connection is proxied to the MongoDB Service. + + +OpenResty +--------- + +We use `OpenResty `_ to perform authorization checks +with 3scale using the ``app_id`` and ``app_key`` headers in the HTTP request. + +OpenResty is NGINX plus a bunch of other +`components `_. We primarily depend +on the LuaJIT compiler to execute the functions to authenticate the ``app_id`` +and ``app_key`` with the 3scale backend. + + +MongoDB +------- + +We use MongoDB as the backend database for BigchainDB. +In a multi-node deployment, MongoDB members communicate with each other via the +public port exposed by the NGINX Service. + +We achieve security by avoiding DoS attacks at the NGINX proxy layer and by +ensuring that MongoDB has TLS enabled for all its connections. + diff --git a/docs/server/source/production-deployment-template/client-tls-certificate.rst b/docs/server/source/production-deployment-template/client-tls-certificate.rst index 80483b83..d0a67006 100644 --- a/docs/server/source/production-deployment-template/client-tls-certificate.rst +++ b/docs/server/source/production-deployment-template/client-tls-certificate.rst @@ -92,7 +92,7 @@ consolidated file containing both the public and private keys. .. code:: bash - cat /path/to/mdb-instance-0.crt /path/to/mdb-instance-0.key > mdb-instance-0.pem + cat /path/to/bdb-instance-0.crt /path/to/bdb-instance-0.key > bdb-instance-0.pem OR diff --git a/docs/server/source/production-deployment-template/cloud-manager.rst b/docs/server/source/production-deployment-template/cloud-manager.rst index 0a1030a3..c407ceb1 100644 --- a/docs/server/source/production-deployment-template/cloud-manager.rst +++ b/docs/server/source/production-deployment-template/cloud-manager.rst @@ -16,7 +16,7 @@ Configure MongoDB Cloud Manager for Monitoring * Select the group from the dropdown box on the page. - * Go to Settings, Group Settings and add a ``Preferred Hostnames`` entry as + * Go to Settings and add a ``Preferred Hostnames`` entry as a regexp based on the ``mdb-instance-name`` of the nodes in your cluster. It may take up to 5 mins till this setting takes effect. You may refresh the browser window and verify whether the changes have diff --git a/docs/server/source/production-deployment-template/index.rst b/docs/server/source/production-deployment-template/index.rst index 8852dd2c..aa966677 100644 --- a/docs/server/source/production-deployment-template/index.rst +++ b/docs/server/source/production-deployment-template/index.rst @@ -28,3 +28,5 @@ Feel free change things to suit your needs or preferences. add-node-on-kubernetes restore-from-mongodb-cloud-manager tectonic-azure + troubleshoot + architecture diff --git a/docs/server/source/production-deployment-template/node-on-kubernetes.rst b/docs/server/source/production-deployment-template/node-on-kubernetes.rst index d415e44a..492d15c6 100644 --- a/docs/server/source/production-deployment-template/node-on-kubernetes.rst +++ b/docs/server/source/production-deployment-template/node-on-kubernetes.rst @@ -322,6 +322,18 @@ Step 9.1: Vanilla NGINX ``cluster-health-check-port``. Set them to the values specified in the ConfigMap. + * The configuration uses the following values set in the ConfigMap: + + - ``cluster-frontend-port`` + - ``cluster-health-check-port`` + - ``cluster-dns-server-ip`` + - ``mongodb-frontend-port`` + - ``ngx-mdb-instance-name`` + - ``mongodb-backend-port`` + - ``ngx-bdb-instance-name`` + - ``bigchaindb-api-port`` + - ``bigchaindb-ws-port`` + * Start the Kubernetes Deployment: .. code:: bash @@ -346,6 +358,25 @@ Step 9.2: NGINX with HTTPS ``cluster-health-check-port``. Set them to the values specified in the ConfigMap. + * The configuration uses the following values set in the ConfigMap: + + - ``cluster-frontend-port`` + - ``cluster-health-check-port`` + - ``cluster-fqdn`` + - ``cluster-dns-server-ip`` + - ``mongodb-frontend-port`` + - ``ngx-mdb-instance-name`` + - ``mongodb-backend-port`` + - ``openresty-backend-port`` + - ``ngx-openresty-instance-name`` + - ``ngx-bdb-instance-name`` + - ``bigchaindb-api-port`` + - ``bigchaindb-ws-port`` + + * The configuration uses the following values set in the Secret: + + - ``https-certs`` + * Start the Kubernetes Deployment: .. code:: bash @@ -383,8 +414,8 @@ First, you need an Azure storage account. If you deployed your Kubernetes cluster on Azure using the Azure CLI 2.0 (as per :doc:`our template `), -then the `az acs create` command already created two -storage accounts in the same location and resource group +then the `az acs create` command already created a +storage account in the same location and resource group as your Kubernetes cluster. Both should have the same "storage account SKU": ``Standard_LRS``. Standard storage is lower-cost and lower-performance. @@ -393,13 +424,14 @@ LRS means locally-redundant storage: three replicas in the same data center. Premium storage is higher-cost and higher-performance. It uses solid state drives (SSD). -At the time of writing, -when we created a storage account with SKU ``Premium_LRS`` -and tried to use that, -the PersistentVolumeClaim would get stuck in a "Pending" state. +You can create a `storage account `_ +for Premium storage and associate it with your Azure resource group. For future reference, the command to create a storage account is `az storage account create `_. +.. Note:: + Please refer to `Azure documentation `_ + for the list of VMs that are supported by Premium Storage. The Kubernetes template for configuration of Storage Class is located in the file ``mongodb/mongo-sc.yaml``. @@ -407,6 +439,10 @@ file ``mongodb/mongo-sc.yaml``. You may have to update the ``parameters.location`` field in the file to specify the location you are using in Azure. +If you want to use a custom storage account with the Storage Class, you +can also update `parameters.storageAccount` and provide the Azure storage +account name. + Create the required storage classes using: .. code:: bash @@ -416,15 +452,6 @@ Create the required storage classes using: You can check if it worked using ``kubectl get storageclasses``. -**Azure.** Note that there is no line of the form -``storageAccount: `` -under ``parameters:``. When we included one -and then created a PersistentVolumeClaim based on it, -the PersistentVolumeClaim would get stuck -in a "Pending" state. -Kubernetes just looks for a storageAccount -with the specified skuName and location. - Step 11: Create Kubernetes Persistent Volume Claims --------------------------------------------------- @@ -457,6 +484,27 @@ You can check its status using: ``kubectl get pvc -w`` Initially, the status of persistent volume claims might be "Pending" but it should become "Bound" fairly quickly. +.. Note:: + The default Reclaim Policy for dynamically created persistent volumes is ``Delete`` + which means the PV and its associated Azure storage resource will be automatically + deleted on deletion of PVC or PV. In order to prevent this from happening do + the following steps to change default reclaim policy of dyanmically created PVs + from ``Delete`` to ``Retain`` + + * Run the following command to list existing PVs + + .. Code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 get pv + + * Run the following command to update a PV's reclaim policy to + + .. Code:: bash + + $ kubectl --context k8s-bdb-test-cluster-0 patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + + For notes on recreating a private volume form a released Azure disk resource consult + :ref:`the page about cluster troubleshooting `. Step 12: Start a Kubernetes StatefulSet for MongoDB --------------------------------------------------- @@ -500,6 +548,30 @@ Step 12: Start a Kubernetes StatefulSet for MongoDB backend port. Set it to the value specified for ``mongodb-backend-port`` in the ConfigMap. + * The configuration uses the following values set in the ConfigMap: + + - ``mdb-instance-name`` + - ``mongodb-replicaset-name`` + - ``mongodb-backend-port`` + + * The configuration uses the following values set in the Secret: + + - ``mdb-certs`` + - ``ca-auth`` + + * **Optional**: You can change the value for ``STORAGE_ENGINE_CACHE_SIZE`` in the ConfigMap ``storage-engine-cache-size``, for more information + regarding this configuration, please consult the `MongoDB Official + Documentation `_. + + * **Optional**: If you are not using the **Standard_D2_v2** virtual machines for Kubernetes agents as per the guide, + please update the ``resources`` for ``mongo-ss``. We suggest allocating ``memory`` using the following scheme + for a MongoDB StatefulSet: + + .. code:: bash + + memory = (Total_Memory_Agent_VM_GB - 2GB) + STORAGE_ENGINE_CACHE_SIZE = memory / 2 + * Create the MongoDB StatefulSet using: .. code:: bash @@ -661,6 +733,12 @@ Step 14: Start a Kubernetes Deployment for MongoDB Monitoring Agent ``mdb-mon-instance-name`` is ``mdb-mon-instance-0``, set the fields to the value ``mdb-mon-instance-0-dep``. + * The configuration uses the following values set in the Secret: + + - ``mdb-mon-certs`` + - ``ca-auth`` + - ``cloud-manager-credentials`` + * Start the Kubernetes Deployment using: .. code:: bash @@ -682,6 +760,12 @@ Step 15: Start a Kubernetes Deployment for MongoDB Backup Agent ``mdb-bak-instance-name`` is ``mdb-bak-instance-0``, set the fields to the value ``mdb-bak-instance-0-dep``. + * The configuration uses the following values set in the Secret: + + - ``mdb-bak-certs`` + - ``ca-auth`` + - ``cloud-manager-credentials`` + * Start the Kubernetes Deployment using: .. code:: bash @@ -714,10 +798,34 @@ Step 16: Start a Kubernetes Deployment for BigchainDB richer monitoring and probing becomes available in BigchainDB, we will tweak the ``livenessProbe`` and ``readinessProbe`` parameters. - * Set the ports to be exposed from the pod in the - ``spec.containers[0].ports`` section. We currently expose 2 ports - - ``bigchaindb-api-port`` and ``bigchaindb-ws-port``. Set them to the - values specified in the ConfigMap. + * Set the ports to be exposed from the pod in the + ``spec.containers[0].ports`` section. We currently expose 2 ports - + ``bigchaindb-api-port`` and ``bigchaindb-ws-port``. Set them to the + values specified in the ConfigMap. + + * The configuration uses the following values set in the ConfigMap: + + - ``mdb-instance-name`` + - ``mongodb-backend-port`` + - ``mongodb-replicaset-name`` + - ``bigchaindb-database-name`` + - ``bigchaindb-server-bind`` + - ``bigchaindb-ws-interface`` + - ``cluster-fqdn`` + - ``bigchaindb-ws-port`` + - ``cluster-frontend-port`` + - ``bigchaindb-wsserver-advertised-scheme`` + - ``bdb-public-key`` + - ``bigchaindb-backlog-reassign-delay`` + - ``bigchaindb-database-maxtries`` + - ``bigchaindb-database-connection-timeout`` + - ``bigchaindb-log-level`` + - ``bdb-user`` + + * The configuration uses the following values set in the Secret: + + - ``bdb-certs`` + - ``ca-auth`` * Create the BigchainDB Deployment using: @@ -747,6 +855,17 @@ Step 17: Start a Kubernetes Deployment for OpenResty which OpenResty is listening for requests, ``openresty-backend-port`` in the above ConfigMap. + * The configuration uses the following values set in the Secret: + + - ``threescale-credentials`` + + * The configuration uses the following values set in the ConfigMap: + + - ``cluster-dns-server-ip`` + - ``openresty-backend-port`` + - ``ngx-bdb-instance-name`` + - ``bigchaindb-api-port`` + * Create the OpenResty Deployment using: .. code:: bash diff --git a/docs/server/source/production-deployment-template/tectonic-azure.rst b/docs/server/source/production-deployment-template/tectonic-azure.rst index c59dc241..3803751e 100644 --- a/docs/server/source/production-deployment-template/tectonic-azure.rst +++ b/docs/server/source/production-deployment-template/tectonic-azure.rst @@ -47,7 +47,9 @@ when following the steps above: ``tectonic-cluster-CLUSTER``. #. Set the ``tectonic_base_domain`` to ``""`` if you want to use Azure managed - DNS. You will be assigned a ``cloudapp.azure.com`` sub-domain by default. + DNS. You will be assigned a ``cloudapp.azure.com`` sub-domain by default and + you can skip the ``Configuring Azure DNS`` section from the Tectonic installation + guide. #. Set the ``tectonic_cl_channel`` to ``"stable"`` unless you want to experiment or test with the latest release. @@ -76,6 +78,14 @@ when following the steps above: #. Set the ``tectonic_azure_ssh_key`` to the path of the public key created in the previous step. +#. We recommend setting up or using a CA(Certificate Authority) to generate Tectonic + Console's server certificate(s) and adding it to your trusted authorities on the client side, + accessing the Tectonic Console i.e. Browser. If you already have a CA(self-signed or otherwise), + Set the ``tectonic_ca_cert`` and ``tectonic_ca_key`` configurations with the content + of PEM-encoded certificate and key files, respectively. For more information about, how to set + up a self-signed CA, Please refer to + :doc:`How to Set up self-signed CA `. + #. Note that the ``tectonic_azure_client_secret`` is the same as the ``ARM_CLIENT_SECRET``. @@ -85,6 +95,10 @@ when following the steps above: ``test-cluster`` and specified the datacenter as ``westeurope``, the Tectonic console will be available at ``test-cluster.westeurope.cloudapp.azure.com``. +#. Note that, if you do not specify ``tectonic_ca_cert``, a CA certificate will + be generated automatically and you will encounter the untrusted certificate + message on your client(Browser), when accessing the Tectonic Console. + Step 4: Configure kubectl ------------------------- diff --git a/docs/server/source/production-deployment-template/template-kubernetes-azure.rst b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst index a916012f..7312ba36 100644 --- a/docs/server/source/production-deployment-template/template-kubernetes-azure.rst +++ b/docs/server/source/production-deployment-template/template-kubernetes-azure.rst @@ -105,6 +105,21 @@ Finally, you can deploy an ACS using something like: --orchestrator-type kubernetes \ --debug --output json +.. Note:: + Please refer to `Azure documentation `_ + for a comprehensive list of options available for `az acs create`. + Please tune the following parameters as per your requirement: + + * Master count. + + * Agent count. + + * Agent VM size. + + * **Optional**: Master storage profile. + + * **Optional**: Agent storage profile. + There are more options. For help understanding all the options, use the built-in help: diff --git a/docs/server/source/production-deployment-template/troubleshoot.rst b/docs/server/source/production-deployment-template/troubleshoot.rst new file mode 100644 index 00000000..72e073e0 --- /dev/null +++ b/docs/server/source/production-deployment-template/troubleshoot.rst @@ -0,0 +1,139 @@ +Cluster Troubleshooting +======================= + +This page describes some basic issues we have faced while deploying and +operating the cluster. + +1. MongoDB Restarts +------------------- + +We define the following in the ``mongo-ss.yaml`` file: + +.. code:: bash + + resources: + limits: + cpu: 200m + memory: 5G + +When the MongoDB cache occupies a memory greater than 5GB, it is +terminated by the ``kubelet``. +This can usually be verified by logging in to the worker node running MongoDB +container and looking at the syslog (the ``journalctl`` command should usually +work). + +This issue is resolved in +`PR #1757 `_. + +2. 502 Bad Gateway Error on Runscope Tests +------------------------------------------ + +It means that NGINX could not find the appropriate backed to forward the +requests to. This typically happens when: + +#. MongoDB goes down (as described above) and BigchainDB, after trying for + ``BIGCHAINDB_DATABASE_MAXTRIES`` times, gives up. The Kubernetes BigchainDB + Deployment then restarts the BigchainDB pod. + +#. BigchainDB crashes for some reason. We have seen this happen when updating + BigchainDB from one version to the next. This usually means the older + connections to the service gets disconnected; retrying the request one more + time, forwards the connection to the new instance and succeed. + + +3. Service Unreachable +---------------------- + +Communication between Kubernetes Services and Deployments fail in +v1.6.6 and before due to a trivial key lookup error for non-existent services +in the ``kubelet``. +This error can be reproduced by restarting any public facing (that is, services +using the cloud load balancer) Kubernetes services, and watching the +``kube-proxy`` failure in its logs. +The solution to this problem is to restart ``kube-proxy`` on the affected +worker/agent node. Login to the worker node and run: + +.. code:: bash + + docker stop `docker ps | grep k8s_kube-proxy | cut -d" " -f1` + + docker logs -f `docker ps | grep k8s_kube-proxy | cut -d" " -f1` + +`This issue `_ is +`fixed in Kubernetes v1.7 `_. + + +4. Single Disk Attached to Multiple Mountpoints in a Container +-------------------------------------------------------------- + +This is currently the issue faced in one of the clusters and being debugged by +the support team at Microsoft. + +The issue was first seen on August 29, 2017 on the Test Network and has been +logged in the `Azure/acs-engine repo on GitHub `_. + +This is apparently fixed in Kubernetes v1.7.2 which include a new disk driver, +but is yet to tested by us. + + +5. MongoDB Monitoring Agent throws a dial error while connecting to MongoDB +--------------------------------------------------------------------------- + +You might see something similar to this in the MongoDB Monitoring Agent logs: + +.. code:: bash + + Failure dialing host without auth. Err: `no reachable servers` + at monitoring-agent/components/dialing.go:278 + at monitoring-agent/components/dialing.go:116 + at monitoring-agent/components/dialing.go:213 + at src/runtime/asm_amd64.s:2086 + + +The first thing to check is if the networking is set up correctly. You can use +the (maybe using the `toolbox` container). + +If everything looks fine, it might be a problem with the ``Preferred +Hostnames`` setting in MongoDB Cloud Manager. If you do need to change the +regular expression, ensure that it is correct and saved properly (maybe try +refreshing the MongoDB Cloud Manager web page to see if the setting sticks). + +Once you update the regular expression, you will need to remove the deployment +and add it again for the Monitoring Agent to discover and connect to the +MongoDB instance correctly. + +More information about this configuration is provided in +:doc:`this document `. + +6. Create a Persistent Volume from existing Azure disk storage Resource +--------------------------------------------------------------------------- +When deleting a k8s cluster, all dynamically-created PVs are deleted, along with the +underlying Azure storage disks (so those can't be used in a new cluster). resources +are also deleted thus cannot be used in a new cluster. This workflow will preserve +the Azure storage disks while deleting the k8s cluster and re-use the same disks on a new +cluster for MongoDB persistent storage without losing any data. + +The template to create two PVs for MongoDB Stateful Set (One for MongoDB data store and +the other for MongoDB config store) is located at ``mongodb/mongo-pv.yaml``. + +You need to configure ``diskName`` and ``diskURI`` in ``mongodb/mongo-pv.yaml`` file. You can get +these values by logging into your Azure portal and going to ``Resource Groups`` and click on your +relevant resource group. From the list of resources click on the storage account resource and +click the container (usually named as ``vhds``) that contains storage disk blobs that are available +for PVs. Click on the storage disk file that you wish to use for your PV and you will be able to +see ``NAME`` and ``URL`` parameters which you can use for ``diskName`` and ``diskURI`` values in +your template respectively and run the following command to create PVs: + +.. code:: bash + + $ kubectl --context apply -f mongodb/mongo-pv.yaml + +.. note:: + + Please make sure the storage disks you are using are not already being used by any + other PVs. To check the existing PVs in your cluster, run the following command + to get PVs and Storage disk file mapping. + + .. code:: bash + + $ kubectl --context get pv --output yaml diff --git a/docs/server/source/production-deployment-template/workflow.rst b/docs/server/source/production-deployment-template/workflow.rst index 8b806bcd..aff8d5f9 100644 --- a/docs/server/source/production-deployment-template/workflow.rst +++ b/docs/server/source/production-deployment-template/workflow.rst @@ -110,13 +110,13 @@ secret token, service ID, version header and API service token. ☐ If the cluster uses MongoDB Cloud Manager for monitoring and backup, -you must ask the managing organization for the ``Group ID`` and the +you must ask the managing organization for the ``Project ID`` and the ``Agent API Key``. -(Each Cloud Manager "group" has its own ``Group ID``. A ``Group ID`` can +(Each Cloud Manager "Project" has its own ``Project ID``. A ``Project ID`` can contain a number of ``Agent API Key`` s. It can be found under -**Settings - Group Settings**. It was recently added to the Cloud Manager to +**Settings**. It was recently added to the Cloud Manager to allow easier periodic rotation of the ``Agent API Key`` with a constant -``Group ID``) +``Project ID``) ☐ :doc:`Deploy a Kubernetes cluster on Azure `. diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 62a31fb9..e066a046 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -1,6 +1,6 @@ # Quickstart -This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. If you're not using Linux, then you might try [running BigchainDB with Docker](appendices/run-with-docker.html). +This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. You can also try, [running BigchainDB with Docker](appendices/run-with-docker.html). A. Install MongoDB as the database backend. (There are other options but you can ignore them for now.) @@ -58,9 +58,8 @@ $ bigchaindb start ``` J. Verify BigchainDB Server setup by visiting the BigchainDB Root URL in your browser: -```text -$ http://127.0.0.1:9984/ -``` + +[http://127.0.0.1:9984/](http://127.0.0.1:9984/) A correctly installed installation will show you a JSON object with information about the API, docs, version and your public key. diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 05f321f9..790cb453 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -51,8 +51,6 @@ all database tables/collections, various backend database indexes, and the genesis block. -Note: The `bigchaindb start` command (see below) always starts by trying a `bigchaindb init` first. If it sees that the backend database already exists, then it doesn't re-initialize the database. One doesn't have to do `bigchaindb init` before `bigchaindb start`. `bigchaindb init` is useful if you only want to initialize (but not start). - ## bigchaindb drop @@ -63,7 +61,7 @@ If you want to force-drop the database (i.e. skipping the yes/no prompt), then u ## bigchaindb start -Start BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`. +Start BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`. The database initialization step is optional and can be skipped by passing the `--no-init` flag i.e. `bigchaindb start --no-init`. You can also use the `--dev-start-rethinkdb` command line option to automatically start rethinkdb with bigchaindb if rethinkdb is not already running, e.g. `bigchaindb --dev-start-rethinkdb start`. Note that this will also shutdown rethinkdb when the bigchaindb process stops. The option `--dev-allow-temp-keypair` will generate a keypair on the fly if no keypair is found, this is useful when you want to run a temporary instance of BigchainDB in a Docker container, for example. diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index c4b3f0ef..b7c5526e 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -39,6 +39,7 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_LOG_FMT_CONSOLE`
`BIGCHAINDB_LOG_FMT_LOGFILE`
`BIGCHAINDB_LOG_GRANULAR_LEVELS`
+`BIGCHAINDB_LOG_PORT`
`BIGCHAINDB_DATABASE_SSL`
`BIGCHAINDB_DATABASE_LOGIN`
`BIGCHAINDB_DATABASE_PASSWORD`
@@ -319,7 +320,8 @@ holding the logging configuration. "granular_levels": { "bichaindb.backend": "info", "bichaindb.core": "info" - } + }, + "port": 7070 } ``` @@ -336,7 +338,8 @@ holding the logging configuration. "datefmt_logfile": "%Y-%m-%d %H:%M:%S", "fmt_logfile": "[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)", "fmt_console": "[%(asctime)s] [%(levelname)s] (%(name)s) %(message)s (%(processName)-10s - pid: %(process)d)", - "granular_levels": {} + "granular_levels": {}, + "port": 9020 } ``` @@ -530,7 +533,23 @@ logging of the `core.py` module to be more verbose, you would set the } ``` -**Defaults to**: `"{}"` +**Defaults to**: `{}` + + +### log.port +The port number at which the logging server should listen. + +**Example**: + +``` +{ + "log": { + "port": 7070 + } +} +``` + +**Defaults to**: `9020` ## graphite.host diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 26a47ca2..ed79b122 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -12,7 +12,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:1.0.1 + image: bigchaindb/bigchaindb:1.3.0 imagePullPolicy: IfNotPresent args: - start @@ -158,4 +158,4 @@ spec: - name: ca-auth secret: secretName: ca-auth - defaultMode: 0400 \ No newline at end of file + defaultMode: 0400 diff --git a/k8s/configuration/config-map.yaml b/k8s/configuration/config-map.yaml index 8c30565f..fafe365c 100644 --- a/k8s/configuration/config-map.yaml +++ b/k8s/configuration/config-map.yaml @@ -99,6 +99,11 @@ data: # WebSocket API in BigchainDB; can be 'ws' or 'wss' (default). bigchaindb-wsserver-advertised-scheme: "wss" + # Optional: Optimize storage engine(wired tiger) + # cache size. e.g. (2048MB, 2GB, 1TB), otherwise + # it will use the default cache size; i.e. max((50% RAM - 1GB), 256MB) + storage-engine-cache-size: "" + --- apiVersion: v1 kind: ConfigMap diff --git a/k8s/configuration/secret.yaml b/k8s/configuration/secret.yaml index d7a83135..ad3ca7d7 100644 --- a/k8s/configuration/secret.yaml +++ b/k8s/configuration/secret.yaml @@ -14,9 +14,9 @@ metadata: namespace: default type: Opaque data: - # Base64-encoded Group ID - # Group ID used by MongoDB deployment - group-id: "" + # Base64-encoded Project ID + # Project ID used by MongoDB deployment + group-id: "" # Base64-encoded MongoDB Agent API Key for the group agent-api-key: "" --- diff --git a/k8s/dev-setup/bigchaindb.yaml b/k8s/dev-setup/bigchaindb.yaml index 1186738e..48a0db2b 100644 --- a/k8s/dev-setup/bigchaindb.yaml +++ b/k8s/dev-setup/bigchaindb.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:1.0.0 + image: bigchaindb/bigchaindb:1.3.0 imagePullPolicy: Always args: - start diff --git a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash index fd1d6420..91a0b22c 100755 --- a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash +++ b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/mongodb-backup-agent:3.4 . +docker build -t bigchaindb/mongodb-backup-agent:3.5 . -docker push bigchaindb/mongodb-backup-agent:3.4 +docker push bigchaindb/mongodb-backup-agent:3.5 diff --git a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml index d00194ed..4aeb66bf 100644 --- a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml +++ b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml @@ -24,7 +24,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: mdb-backup - image: bigchaindb/mongodb-backup-agent:3.4 + image: bigchaindb/mongodb-backup-agent:3.5 imagePullPolicy: IfNotPresent env: - name: MMS_API_KEYFILE_PATH diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash index ce861040..680fab22 100755 --- a/k8s/mongodb/container/docker_build_and_push.bash +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/mongodb:3.1 . +docker build -t bigchaindb/mongodb:3.2 . -docker push bigchaindb/mongodb:3.1 +docker push bigchaindb/mongodb:3.2 diff --git a/k8s/mongodb/container/mongod.conf.template b/k8s/mongodb/container/mongod.conf.template index 2ad36344..d8ae1bce 100644 --- a/k8s/mongodb/container/mongod.conf.template +++ b/k8s/mongodb/container/mongod.conf.template @@ -86,6 +86,7 @@ storage: wiredTiger: engineConfig: journalCompressor: snappy + configString: cache_size=STORAGE_ENGINE_CACHE_SIZE collectionConfig: blockCompressor: snappy indexConfig: @@ -98,4 +99,3 @@ operationProfiling: replication: replSetName: REPLICA_SET_NAME enableMajorityReadConcern: true - diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index c9f7b027..213f9989 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -46,6 +46,10 @@ while [[ $# -gt 1 ]]; do MONGODB_IP="$2" shift ;; + --storage-engine-cache-size) + STORAGE_ENGINE_CACHE_SIZE="$2" + shift + ;; *) echo "Unknown option: $1" exit 1 @@ -61,7 +65,8 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ - -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then + -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ + -z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then #-z "${MONGODB_KEY_FILE_PASSWORD:?MongoDB Key File Password not specified. Exiting!}" || \ exit 1 else @@ -72,6 +77,7 @@ else echo MONGODB_KEY_FILE_PATH="$MONGODB_KEY_FILE_PATH" echo MONGODB_CA_FILE_PATH="$MONGODB_CA_FILE_PATH" echo MONGODB_CRL_FILE_PATH="$MONGODB_CRL_FILE_PATH" + echo STORAGE_ENGINE_CACHE_SIZE="$STORAGE_ENGINE_CACHE_SIZE" fi MONGODB_CONF_FILE_PATH=/etc/mongod.conf @@ -84,6 +90,16 @@ sed -i "s|MONGODB_KEY_FILE_PATH|${MONGODB_KEY_FILE_PATH}|g" ${MONGODB_CONF_FILE_ sed -i "s|MONGODB_CA_FILE_PATH|${MONGODB_CA_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_CRL_FILE_PATH|${MONGODB_CRL_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|REPLICA_SET_NAME|${REPLICA_SET_NAME}|g" ${MONGODB_CONF_FILE_PATH} +if [ ! -z "$STORAGE_ENGINE_CACHE_SIZE" ]; then + if [[ "$STORAGE_ENGINE_CACHE_SIZE" =~ ^[0-9]+(G|M|T)B$ ]]; then + sed -i.bk "s|STORAGE_ENGINE_CACHE_SIZE|${STORAGE_ENGINE_CACHE_SIZE}|g" ${MONGODB_CONF_FILE_PATH} + else + echo "Invalid Value for storage engine cache size $STORAGE_ENGINE_CACHE_SIZE" + exit 1 + fi +else + sed -i.bk "/cache_size=/d" ${MONGODB_CONF_FILE_PATH} +fi # add the hostname and ip to hosts file echo "${MONGODB_IP} ${MONGODB_FQDN}" >> $HOSTS_FILE_PATH diff --git a/k8s/mongodb/mongo-pv.yaml b/k8s/mongodb/mongo-pv.yaml new file mode 100644 index 00000000..15b90213 --- /dev/null +++ b/k8s/mongodb/mongo-pv.yaml @@ -0,0 +1,41 @@ +############################################################# +# This YAML section desribes a k8s PV for mongodb dbPath # +############################################################# +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-mongo-db +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 50Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: slow-db +--- +############################################################# +# This YAML section desribes a k8s PV for mongodb configDB # +############################################################# +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-mongdo-configdb +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: + diskURI: + fsType: ext4 + readOnly: false + capacity: + storage: 2Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: slow-configdb diff --git a/k8s/mongodb/mongo-sc.yaml b/k8s/mongodb/mongo-sc.yaml index 2f291ffe..2b155d54 100644 --- a/k8s/mongodb/mongo-sc.yaml +++ b/k8s/mongodb/mongo-sc.yaml @@ -7,8 +7,12 @@ metadata: name: slow-db provisioner: kubernetes.io/azure-disk parameters: - skuName: Standard_LRS + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] location: westeurope + # If you have created a different storage account e.g. for Premium Storage + #storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed --- ###################################################################### # This YAML section desribes a StorageClass for the mongodb configDB # @@ -19,5 +23,9 @@ metadata: name: slow-configdb provisioner: kubernetes.io/azure-disk parameters: - skuName: Standard_LRS + skuName: Premium_LRS #[Premium_LRS, Standard_LRS] location: westeurope + # If you have created a different storage account e.g. for Premium Storage + #storageAccount: + # Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment) + #kind: Managed diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index 4bc3cfa2..1243da26 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -21,7 +21,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: mongodb - image: bigchaindb/mongodb:3.1 + image: bigchaindb/mongodb:3.2 imagePullPolicy: IfNotPresent env: - name: MONGODB_FQDN @@ -43,6 +43,11 @@ spec: configMapKeyRef: name: vars key: mongodb-backend-port + - name: STORAGE_ENGINE_CACHE_SIZE + valueFrom: + configMapKeyRef: + name: vars + key: storage-engine-cache-size args: - --mongodb-port - $(MONGODB_PORT) @@ -58,6 +63,8 @@ spec: - $(MONGODB_FQDN) - --mongodb-ip - $(MONGODB_POD_IP) + - --storage-engine-cache-size + - $(STORAGE_ENGINE_CACHE_SIZE) securityContext: capabilities: add: @@ -80,7 +87,7 @@ spec: resources: limits: cpu: 200m - memory: 3.5G + memory: 5G livenessProbe: tcpSocket: port: mdb-api-port diff --git a/k8s/nginx-http/container/Dockerfile b/k8s/nginx-http/container/Dockerfile index c6b0ccd4..e35dd5e0 100644 --- a/k8s/nginx-http/container/Dockerfile +++ b/k8s/nginx-http/container/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.13.1 +FROM nginx:stable LABEL maintainer "dev@bigchaindb.com" WORKDIR / RUN apt-get update \ diff --git a/k8s/nginx-http/container/docker_build_and_push.bash b/k8s/nginx-http/container/docker_build_and_push.bash index d4b70555..5011eb2d 100755 --- a/k8s/nginx-http/container/docker_build_and_push.bash +++ b/k8s/nginx-http/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/nginx_http:1.0 . +docker build -t bigchaindb/nginx_http:1.1 . -docker push bigchaindb/nginx_http:1.0 +docker push bigchaindb/nginx_http:1.1 diff --git a/k8s/nginx-http/container/nginx.conf.template b/k8s/nginx-http/container/nginx.conf.template index 35d121eb..bc8b8245 100644 --- a/k8s/nginx-http/container/nginx.conf.template +++ b/k8s/nginx-http/container/nginx.conf.template @@ -45,6 +45,12 @@ http { keepalive_timeout 60s; + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + # The following map blocks enable lazy-binding to the backend at runtime, # rather than binding as soon as NGINX starts. map $remote_addr $bdb_backend { @@ -54,7 +60,6 @@ http { # Frontend server for the external clients server { listen CLUSTER_FRONTEND_PORT; - underscores_in_headers on; # Forward websockets to backend BDB at 9985. @@ -86,7 +91,7 @@ http { add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; - + proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; } @@ -100,6 +105,11 @@ http { add_header 'Content-Length' 0; return 204; } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } } } @@ -130,10 +140,10 @@ stream { # Enable logging when connections are being throttled. limit_conn_log_level notice; - + # Allow 16 connections from the same IP address. limit_conn two 16; - + # DNS resolver to use for all the backend names specified in this configuration. resolver DNS_SERVER valid=30s ipv6=off; @@ -142,7 +152,7 @@ stream { map $remote_addr $mdb_backend { default MONGODB_BACKEND_HOST; } - + # Frontend server to forward connections to MDB instance. server { listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5; diff --git a/k8s/nginx-http/nginx-http-dep.yaml b/k8s/nginx-http/nginx-http-dep.yaml index ad97bcdf..5a9359f5 100644 --- a/k8s/nginx-http/nginx-http-dep.yaml +++ b/k8s/nginx-http/nginx-http-dep.yaml @@ -12,7 +12,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: nginx - image: bigchaindb/nginx_http:1.0 + image: bigchaindb/nginx_http:1.1 imagePullPolicy: IfNotPresent env: - name: CLUSTER_FRONTEND_PORT diff --git a/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash b/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash index 69bfd56f..e84e2563 100755 --- a/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash +++ b/k8s/nginx-https-web-proxy/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/nginx-https-web-proxy:0.10 . +docker build -t bigchaindb/nginx-https-web-proxy:0.12 . -docker push bigchaindb/nginx-https-web-proxy:0.10 +docker push bigchaindb/nginx-https-web-proxy:0.12 diff --git a/k8s/nginx-https-web-proxy/container/nginx.conf.template b/k8s/nginx-https-web-proxy/container/nginx.conf.template index 421d7fc0..c379dc8e 100644 --- a/k8s/nginx-https-web-proxy/container/nginx.conf.template +++ b/k8s/nginx-https-web-proxy/container/nginx.conf.template @@ -90,12 +90,6 @@ http { end } - # check if the request originated from the required web page - # use referer header. - if ($http_referer !~ "PROXY_EXPECTED_REFERER_HEADER" ) { - return 403 'Unknown referer'; - } - # check if the request has the expected origin header if ($http_origin !~ "PROXY_EXPECTED_ORIGIN_HEADER" ) { return 403 'Unknown origin'; @@ -108,9 +102,16 @@ http { add_header 'Access-Control-Max-Age' 43200; add_header 'Content-Type' 'text/plain charset=UTF-8'; add_header 'Content-Length' 0; + add_header 'Referrer-Policy' "PROXY_REFERRER_POLICY"; return 204; } + # check if the request originated from the required web page + # use referer header. + if ($http_referer !~ "PROXY_EXPECTED_REFERER_HEADER" ) { + return 403 'Unknown referer'; + } + # No auth for GETs, forward directly to BDB. if ($request_method = GET) { proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; diff --git a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml index b9dbc541..0a255d89 100644 --- a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml +++ b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-conf.yaml @@ -49,6 +49,11 @@ data: # are available to external clients. proxy-frontend-port: "4443" + # proxy-referrer-policy defines the expected behaviour from + # browser while setting the referer header in the HTTP requests to the + # proxy service. + proxy-referrer-policy: "origin-when-cross-origin" + # expected-http-referer is the expected regex expression of the Referer # header in the HTTP requests to the proxy. # The default below accepts the referrer value to be *.bigchaindb.com diff --git a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml index 6ada9347..e92aba7d 100644 --- a/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml +++ b/k8s/nginx-https-web-proxy/nginx-https-web-proxy-dep.yaml @@ -25,6 +25,11 @@ spec: configMapKeyRef: name: proxy-vars key: proxy-frontend-port + - name: PROXY_REFERRER_POLICY + valueFrom: + configMapKeyRef: + name: proxy-vars + key: proxy-referrer-policy - name: PROXY_EXPECTED_REFERER_HEADER valueFrom: configMapKeyRef: diff --git a/k8s/nginx-https/container/Dockerfile b/k8s/nginx-https/container/Dockerfile index 98ec0cfd..3bd6b607 100644 --- a/k8s/nginx-https/container/Dockerfile +++ b/k8s/nginx-https/container/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.13.1 +FROM nginx:stable LABEL maintainer "dev@bigchaindb.com" WORKDIR / RUN apt-get update \ diff --git a/k8s/nginx-https/container/docker_build_and_push.bash b/k8s/nginx-https/container/docker_build_and_push.bash index 3ae71ff9..76494bcb 100755 --- a/k8s/nginx-https/container/docker_build_and_push.bash +++ b/k8s/nginx-https/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/nginx_https:1.0 . +docker build -t bigchaindb/nginx_https:1.1 . -docker push bigchaindb/nginx_https:1.0 +docker push bigchaindb/nginx_https:1.1 diff --git a/k8s/nginx-https/container/nginx.conf.template b/k8s/nginx-https/container/nginx.conf.template index 8a85c894..3ffb4cce 100644 --- a/k8s/nginx-https/container/nginx.conf.template +++ b/k8s/nginx-https/container/nginx.conf.template @@ -42,6 +42,12 @@ http { client_body_timeout 10s; client_header_timeout 10s; + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + # DNS resolver to use for all the backend names specified in this configuration. resolver DNS_SERVER valid=30s ipv6=off; @@ -60,10 +66,11 @@ http { server { listen CLUSTER_FRONTEND_PORT ssl; server_name "CLUSTER_FQDN"; - ssl_certificate /etc/nginx/ssl/cert.pem; - ssl_certificate_key /etc/nginx/ssl/cert.key; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers HIGH:!aNULL:!MD5; + + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/cert.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; underscores_in_headers on; @@ -114,6 +121,11 @@ http { add_header 'Content-Length' 0; return 204; } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } } } diff --git a/k8s/nginx-https/nginx-https-dep.yaml b/k8s/nginx-https/nginx-https-dep.yaml index 57218424..79dfe040 100644 --- a/k8s/nginx-https/nginx-https-dep.yaml +++ b/k8s/nginx-https/nginx-https-dep.yaml @@ -12,7 +12,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: nginx - image: bigchaindb/nginx_https:1.0 + image: bigchaindb/nginx_https:1.1 imagePullPolicy: IfNotPresent env: - name: CLUSTER_FRONTEND_PORT diff --git a/k8s/nginx-openresty/LICENSE.md b/k8s/nginx-openresty/LICENSE.md new file mode 100644 index 00000000..7451bedf --- /dev/null +++ b/k8s/nginx-openresty/LICENSE.md @@ -0,0 +1,74 @@ +# Licenses on the Stuff in this Directory + +All _code_ in this directory is copyright BigchainDB GmbH, +except for the configuration files obtained from 3scale (NGINX configuration +file and NGINX Lua script). + +`nginx.conf.template` and `nginx.lua.template` are based on files we got from +3scale (by going to the 3scale admin site - API - Integration - Download +the NGINX Config files). + +The original files (from 3scale) were licensed under an MIT License, +the text of which can be found below. + +The derived files (`nginx.conf.template` and `nginx.lua.template`), along with +the other files in this directory, are _also_ licensed under an MIT License, +the text of which can be found below. + + +# Documentation Licenses + +The documentation in this directory is licensed under a Creative Commons Attribution-ShareAlike +4.0 International license, the full text of which can be found at +[http://creativecommons.org/licenses/by-sa/4.0/legalcode](http://creativecommons.org/licenses/by-sa/4.0/legalcode). + + +
+ +The MIT License + +Copyright (c) 2016-2017 BigchainDB GmbH. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +
+ +The MIT License + +Copyright (c) 2007-2016 3scale Networks S.L. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/k8s/nginx-openresty/container/Dockerfile b/k8s/nginx-openresty/container/Dockerfile new file mode 100644 index 00000000..da3bfc40 --- /dev/null +++ b/k8s/nginx-openresty/container/Dockerfile @@ -0,0 +1,15 @@ +FROM openresty/openresty:xenial +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /usr/local/openresty/nginx/conf/nginx.conf +COPY nginx.lua.template /usr/local/openresty/nginx/conf/nginx.lua +COPY nginx_openresty_entrypoint.bash / +# The following ports are the values we use to run the NGINX+3scale container. +# 80 for http, 8080 for the 3scale api, 8888 for health-check, 27017 for +# MongoDB +EXPOSE 80 8080 8888 27017 +ENTRYPOINT ["/nginx_openresty_entrypoint.bash"] diff --git a/k8s/nginx-openresty/container/README.md b/k8s/nginx-openresty/container/README.md new file mode 100644 index 00000000..47b37103 --- /dev/null +++ b/k8s/nginx-openresty/container/README.md @@ -0,0 +1,60 @@ +# nginx_3scale agent +nginx_3scale agent is a module that is responsible for providing authentication, +authorization and metering of BigchainDB API users, by communicating with 3scale. +We use the openresty for this, which is nginx bundled with lua libraries. +More information at their [website](openresty.org/en) + +It validates the tokens sent by users in HTTP headers. +The user tokens map directly to the Application Plan specified in 3scale. + +## Build and Push the Latest Container +Use the `docker_build_and_push.bash` script to build the latest docker image +and upload it to Docker Hub. +Ensure that the image tag is updated to a new version number to properly +reflect any changes made to the container. + + +## Working + +* We define a [lua module](./nginx.lua.template) and + custom hooks (lua functions to be executed at certain phases of the nginx + request processing lifecycle) to authenticate an API request. + +* Download the template available from 3scale which pre-defines all the + rules defined using the 3scale UI for monitoring, and the basic nginx + configuration. + +* We heavily modify these templates to add our custom functionality. + +* The nginx_3scale image reads the environment variables and accordingly + creates the nginx.conf and nginx.lua files from the templates. + +* Every request calls the `_M.access()` function. This function extracts the + `app_id` and `app_key` from the HTTP request headers and forwards it to + 3scale to see if a request is allowed to be forwarded to the BigchainDB + backend. The request also contains the + various parameters that one would like to set access policies on. If the + `app_id` and `app_key` is successful, the access rules for the parameters + passed with the request are checked to see if the request can pass through. + For example, we can send a parameter, say `request_body_size`, to the 3scale + auth API. If we have defined a rule in the 3scale dashboard to drop + `request_body_size` above a certain threshold, the authorization will fail + even if the `app_id` and `app_key` are valid. + +* A successful response from the auth API causes the request to be proxied to + the backend. After a backend response, the `_M.post_action_content` hook is + called. We calculate details about all the metrics we are interested in and + form a payload for the 3scale reporting API. This ensures that we update + parameters of every metric defined in the 3scale UI after every request. + +* Note: We do not cache the keys in nginx so that we can validate every request + with 3scale and apply plan rules immediately. We can add auth caching to + improve performance, and in case we move to a fully post-paid billing model. + +* Refer to the references made in the [lua module](./nginx.lua.template) for + more details about how nginx+lua+3scale works + +* For HTTPS support, we also need to add the signed certificate and the + corresponding private key to the folder + `/usr/local/openresty/nginx/conf/ssl/`. Name the pem-encoded certificate as + `cert.pem` and the private key as `cert.key`. diff --git a/k8s/nginx-openresty/container/docker_build_and_push.bash b/k8s/nginx-openresty/container/docker_build_and_push.bash new file mode 100755 index 00000000..21d203cc --- /dev/null +++ b/k8s/nginx-openresty/container/docker_build_and_push.bash @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t bigchaindb/nginx_3scale:3.1 . + +docker push bigchaindb/nginx_3scale:3.1 diff --git a/k8s/nginx-openresty/container/nginx.conf.template b/k8s/nginx-openresty/container/nginx.conf.template new file mode 100644 index 00000000..9a4e56bc --- /dev/null +++ b/k8s/nginx-openresty/container/nginx.conf.template @@ -0,0 +1,197 @@ +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /usr/local/openresty/nginx/logs/error.log; +env THREESCALE_DEPLOYMENT_ENV; + +events { + worker_connections 256; + accept_mutex on; + use epoll; +} + +http { + lua_shared_dict api_keys 10m; + server_names_hash_bucket_size 128; + lua_package_path ";;$prefix/?.lua;$prefix/conf/?.lua"; + init_by_lua 'math.randomseed(ngx.time()) ; cjson = require("cjson")'; + access_log /usr/local/openresty/nginx/logs/access.log combined buffer=16k flush=5s; + + # allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + # enable logging when requests are being throttled + limit_req_log_level notice; + + # the http status code to return to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + resolver DNS_SERVER valid=30s ipv6=off; + + map $remote_addr $bdb_backend { + default BIGCHAINDB_BACKEND_HOST; + } + + upstream backend_SERVICE_ID { + server localhost:9999 max_fails=5 fail_timeout=30; + } + + # Our frontend API server that accepts requests from the external world and + # takes care of authentication and authorization. If auth is successful, it + # forwards the request to the backend_SERVICE_ID upstream where a consortium + # can run a BDB cluster. + server { + lua_code_cache on; + listen OPENRESTY_FRONTEND_PORT; + keepalive_timeout 60s; + + underscores_in_headers on; + set_by_lua $deployment 'return os.getenv("THREESCALE_DEPLOYMENT_ENV")'; + set $threescale_backend "https://su1.3scale.net"; + #set $threescale_backend "http://su1.3scale.net"; + #set $threescale_backend "https://su1.3scale.net:443"; + #set $threescale_backend "https://echo-api.3scale.net"; + + # `slowloris` attack mitigation settings + client_body_timeout 10s; + client_header_timeout 10s; + + location = /out_of_band_authrep_action { + internal; + proxy_pass_request_headers off; + set $service_token "SERVICE_TOKEN"; + content_by_lua "require('nginx').post_action_content()"; + } + + # 3scale auth api that takes the auth credentials and metrics as input, + # and returns 200 OK if both the credentials match and the user has not + # exceeded the limits in his application plan. + location = /threescale_auth { + internal; + set $service_token "SERVICE_TOKEN"; + proxy_pass $threescale_backend/transactions/authorize.xml?service_token=$service_token&service_id=$service_id&$usage&$credentials&log%5Bcode%5D=$arg_code&log%5Brequest%5D=$arg_req&log%5Bresponse%5D=$arg_resp; + proxy_set_header Host "su1.3scale.net"; + #proxy_set_header Host "echo-api.3scale.net"; + proxy_set_header X-3scale-User-Agent "nginx$deployment"; + proxy_set_header X-3scale-Version "THREESCALE_VERSION_HEADER"; + } + + # 3scale reporting api that takes the metrics data and persists the metrics + # in the 3scale backend. + location = /threescale_report { + internal; + set $service_token "SERVICE_TOKEN"; + proxy_pass $threescale_backend/transactions.xml; + proxy_set_header Host "su1.3scale.net"; + #proxy_set_header Host "echo-api.3scale.net"; + # We have a bug in lua-nginx module that does not set + # Content-Type from lua script + proxy_pass_request_headers off; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_set_header X-3scale-User-Agent "nginx$deployment"; + proxy_set_header X-3scale-Version "THREESCALE_VERSION_HEADER"; + } + + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-3scale-proxy-secret-token $secret_token; + + # limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # We do not need the GET handling here as it's done in the other NGINX + # module + #if ($request_method = GET ) { + # proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; + #} + + if ($request_method = POST ) { + set $service_token null; + set $cached_key null; + set $credentials null; + set $usage null; + set $service_id SERVICE_ID; + set $proxy_pass null; + set $secret_token null; + set $resp_body null; + set $resp_headers null; + access_by_lua "require('nginx').access()"; + body_filter_by_lua 'ngx.ctx.buffered = (ngx.ctx.buffered or "") .. string.sub(ngx.arg[1], 1, 1000) + if ngx.arg[2] then ngx.var.resp_body = ngx.ctx.buffered end'; + header_filter_by_lua 'ngx.var.resp_headers = cjson.encode(ngx.resp.get_headers())'; + + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass $proxy_pass ; + post_action /out_of_band_authrep_action; + } + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + } + } + + # Our backend server block that accepts requests from the nginx proxy and + # forwards it to instances of BDB cluster. We currently run only a single + # instance. + server { + sendfile on; + + listen 9999; + + # max client request body size: avg transaction size + client_max_body_size 15k; + + # keepalive connection settings + keepalive_timeout 60s; + + # `slowloris` attack mitigation settings + client_body_timeout 10s; + client_header_timeout 10s; + + if ( $http_x_3scale_proxy_secret_token != "THREESCALE_RESPONSE_SECRET_TOKEN" ) { + return 403; + } + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # enable the following line if and only if you use HTTPS + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; + + # limit requests from the same client, allow `burst` to 20 r/s on avg, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/local/openresty/nginx/html/50x.html; + } + } +} diff --git a/k8s/nginx-openresty/container/nginx.lua.template b/k8s/nginx-openresty/container/nginx.lua.template new file mode 100644 index 00000000..64baeaa1 --- /dev/null +++ b/k8s/nginx-openresty/container/nginx.lua.template @@ -0,0 +1,416 @@ +-- -*- mode: lua; -*- +-- Generated on: 2017-04-10 14:41:18 +0000 -- +-- Version: +-- Error Messages per service + +-- Ref: https://github.com/openresty/lua-nginx-module +-- Ref: https://ipdbtestnet-admin.3scale.net/p/admin/api_docs +-- Ref: http://nginx.org/en/docs/debugging_log.html + +local custom_config = false + +local _M = { + ['services'] = { + ['SERVICE_ID'] = { + error_auth_failed = 'Authentication failed', + error_auth_missing = 'Authentication parameters missing', + auth_failed_headers = 'text/plain; charset=us-ascii', + auth_missing_headers = 'text/plain; charset=us-ascii', + error_no_match = 'No Mapping Rule matched', + no_match_headers = 'text/plain; charset=us-ascii', + no_match_status = 404, + auth_failed_status = 403, + auth_missing_status = 403, + secret_token = 'THREESCALE_RESPONSE_SECRET_TOKEN', + get_credentials = function(service, params) + return ( + (params.app_id and params.app_key) + ) or error_no_credentials(service) + end, + extract_usage = function (service, request) + local method, url = unpack(string.split(request," ")) + local path, querystring = unpack(string.split(url, "?")) + local usage_t = {} + local matched_rules = {} + + local args = get_auth_params(nil, method) + + for i,r in ipairs(service.rules) do + check_rule({path=path, method=method, args=args}, r, usage_t, matched_rules) + end + + -- if there was no match, usage is set to nil and it will respond a 404, this behavior can be changed + return usage_t, table.concat(matched_rules, ", ") + end, + rules = { + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'hits', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'request_body_size', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'response_body_size', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'post_transactions', + delta = 1 + }, + { + method = 'POST', + pattern = '/api/{version}/transactions$', + parameters = { 'version' }, + querystring_params = function(args) + return true + end, + system_name = 'total_body_size', + delta = 1 + }, + } +}, + } +} + +-- Error Codes +function error_no_credentials(service) + ngx.status = service.auth_missing_status + ngx.header.content_type = service.auth_missing_headers + ngx.print(service.error_auth_missing) + ngx.exit(ngx.HTTP_OK) +end + +function error_authorization_failed(service) + ngx.status = service.auth_failed_status + ngx.header.content_type = service.auth_failed_headers + ngx.print(service.error_auth_failed) + ngx.exit(ngx.HTTP_OK) +end + +function error_no_match(service) + ngx.status = service.no_match_status + ngx.header.content_type = service.no_match_headers + ngx.print(service.error_no_match) + ngx.exit(ngx.HTTP_OK) +end +-- End Error Codes + +-- Aux function to split a string + +function string:split(delimiter) + local result = { } + local from = 1 + local delim_from, delim_to = string.find( self, delimiter, from ) + if delim_from == nil then return {self} end + while delim_from do + table.insert( result, string.sub( self, from , delim_from-1 ) ) + from = delim_to + 1 + delim_from, delim_to = string.find( self, delimiter, from ) + end + table.insert( result, string.sub( self, from ) ) + return result +end + +function first_values(a) + r = {} + for k,v in pairs(a) do + if type(v) == "table" then + r[k] = v[1] + else + r[k] = v + end + end + return r +end + +function set_or_inc(t, name, delta) + return (t[name] or 0) + delta +end + +function build_querystring_formatter(fmt) + return function (query) + local function kvmap(f, t) + local res = {} + for k, v in pairs(t) do + table.insert(res, f(k, v)) + end + return res + end + + return table.concat(kvmap(function(k,v) return string.format(fmt, k, v) end, query or {}), "&") + end +end + +local build_querystring = build_querystring_formatter("usage[%s]=%s") +local build_query = build_querystring_formatter("%s=%s") + +function regexpify(path) + return path:gsub('?.*', ''):gsub("{.-}", '([\\w_.-]+)'):gsub("%.", "\\.") +end + +function check_rule(req, rule, usage_t, matched_rules) + local param = {} + local p = regexpify(rule.pattern) + local m = ngx.re.match(req.path, + string.format("^%s",p)) + if m and req.method == rule.method then + local args = req.args + if rule.querystring_params(args) then -- may return an empty table + -- when no querystringparams + -- in the rule. it's fine + for i,p in ipairs(rule.parameters) do + param[p] = m[i] + end + + table.insert(matched_rules, rule.pattern) + usage_t[rule.system_name] = set_or_inc(usage_t, rule.system_name, rule.delta) + end + end +end + +--[[ + Authorization logic + NOTE: We do not use any of the authorization logic defined in the template. + We use custom authentication and authorization logic defined in the + custom_app_id_authorize() function. +]]-- + +function get_auth_params(where, method) + local params = {} + if where == "headers" then + params = ngx.req.get_headers() + elseif method == "GET" then + params = ngx.req.get_uri_args() + else + ngx.req.read_body() + params = ngx.req.get_post_args() + end + return first_values(params) +end + +function get_debug_value() + local h = ngx.req.get_headers() + if h["X-3scale-debug"] == 'SERVICE_TOKEN' then + return true + else + return false + end +end + +function _M.authorize(auth_strat, params, service) + if auth_strat == 'oauth' then + oauth(params, service) + else + authrep(params, service) + end +end + +function oauth(params, service) + ngx.var.cached_key = ngx.var.cached_key .. ":" .. ngx.var.usage + local access_tokens = ngx.shared.api_keys + local is_known = access_tokens:get(ngx.var.cached_key) + + if is_known ~= 200 then + local res = ngx.location.capture("/threescale_oauth_authrep", { share_all_vars = true }) + + -- IN HERE YOU DEFINE THE ERROR IF CREDENTIALS ARE PASSED, BUT THEY ARE NOT VALID + if res.status ~= 200 then + access_tokens:delete(ngx.var.cached_key) + ngx.status = res.status + ngx.header.content_type = "application/json" + ngx.var.cached_key = nil + error_authorization_failed(service) + else + access_tokens:set(ngx.var.cached_key,200) + end + + ngx.var.cached_key = nil + end +end + +function authrep(params, service) + ngx.var.cached_key = ngx.var.cached_key .. ":" .. ngx.var.usage + local api_keys = ngx.shared.api_keys + local is_known = api_keys:get(ngx.var.cached_key) + + if is_known ~= 200 then + local res = ngx.location.capture("/threescale_authrep", { share_all_vars = true }) + + -- IN HERE YOU DEFINE THE ERROR IF CREDENTIALS ARE PASSED, BUT THEY ARE NOT VALID + if res.status ~= 200 then + -- remove the key, if it's not 200 let's go the slow route, to 3scale's backend + api_keys:delete(ngx.var.cached_key) + ngx.status = res.status + ngx.header.content_type = "application/json" + ngx.var.cached_key = nil + error_authorization_failed(service) + else + api_keys:set(ngx.var.cached_key,200) + end + ngx.var.cached_key = nil + end +end + +function _M.access() + local params = {} + local host = ngx.req.get_headers()["Host"] + local auth_strat = "" + local service = {} + local usage = {} + local matched_patterns = '' + + if ngx.status == 403 then + ngx.say("Throttling due to too many requests") + ngx.exit(403) + end + + if ngx.var.service_id == 'SERVICE_ID' then + local parameters = get_auth_params("headers", string.split(ngx.var.request, " ")[1] ) + service = _M.services['SERVICE_ID'] -- + ngx.var.secret_token = service.secret_token + params.app_id = parameters["app_id"] + params.app_key = parameters["app_key"] -- or "" -- Uncoment the first part if you want to allow not passing app_key + service.get_credentials(service, params) + ngx.var.cached_key = "SERVICE_ID" .. ":" .. params.app_id ..":".. params.app_key + auth_strat = "2" + ngx.var.service_id = "SERVICE_ID" + ngx.var.proxy_pass = "http://backend_SERVICE_ID" + usage, matched_patterns = service:extract_usage(ngx.var.request) + end + + usage['post_transactions'] = 0 + usage['request_body_size'] = 0 + usage['total_body_size'] = 0 + usage['response_body_size'] = 0 + ngx.var.credentials = build_query(params) + ngx.var.usage = build_querystring(usage) + + -- WHAT TO DO IF NO USAGE CAN BE DERIVED FROM THE REQUEST. + if ngx.var.usage == '' then + ngx.header["X-3scale-matched-rules"] = '' + error_no_match(service) + end + + if get_debug_value() then + ngx.header["X-3scale-matched-rules"] = matched_patterns + ngx.header["X-3scale-credentials"] = ngx.var.credentials + ngx.header["X-3scale-usage"] = ngx.var.usage + ngx.header["X-3scale-hostname"] = ngx.var.hostname + end + _M.custom_app_id_authorize(params, service) +end + +function _M.custom_app_id_authorize(params, service) + ngx.var.cached_key = ngx.var.cached_key .. ":" .. ngx.var.usage + local api_keys = ngx.shared.api_keys + local res = ngx.location.capture("/threescale_auth", { share_all_vars = true }) + if res.status ~= 200 then + ngx.status = res.status + ngx.header.content_type = "application/json" + ngx.var.cached_key = nil + error_authorization_failed(service) + end + ngx.var.cached_key = nil +end + +function _M.post_action_content() + local report_data = {} + + -- increment POST count + report_data['post_transactions'] = 1 + + -- NOTE: When we are querying for the length of the request here, we already + -- have the complete request data with us and hence can just use the len() + -- function to get the size of the payload in bytes. + -- However, we might not have a complete response from the backend at this + -- stage (esp. if it's a large response size). So, we decipher the payload + -- size by peeking into the content length header of the response. + -- Otherwise, nginx will have to buffer every response and then calculate + -- response payload size. + + -- req data size + local req_data = ngx.req.get_body_data() + if req_data then + report_data['request_body_size'] = req_data:len() + else + report_data['request_body_size'] = 0 + end + + -- res data size + local all_headers = cjson.decode(ngx.var.resp_headers) + local variable_header = "content-length" --<-- case sensitive + if all_headers[variable_header] then + report_data['response_body_size'] = all_headers[variable_header] + else + report_data['response_body_size'] = 0 + end + + -- total data size + report_data['total_body_size'] = report_data['request_body_size'] + report_data['response_body_size'] + + -- get the app_id + local app_id = "" + local credentials = ngx.var.credentials:split("&") + for i in pairs(credentials) do + if credentials[i]:match('app_id') then + local temp = credentials[i]:split("=") + app_id = temp[2] + end + end + + -- form the payload to report to 3scale + local report = {} + report['service_id'] = ngx.var.service_id + report['service_token'] = ngx.var.service_token + report['transactions[0][app_id]'] = app_id + report['transactions[0][usage][post_transactions]'] = report_data['post_transactions'] + report['transactions[0][usage][request_body_size]'] = report_data['request_body_size'] + report['transactions[0][usage][response_body_size]'] = report_data['response_body_size'] + report['transactions[0][usage][total_body_size]'] = report_data['total_body_size'] + local res1 = ngx.location.capture("/threescale_report", {method = ngx.HTTP_POST, body = ngx.encode_args(report), share_all_vars = true }) + --ngx.log(0, ngx.encode_args(report)) + ngx.log(0, "Status: "..res1.status) + ngx.log(0, "Body: "..res1.body) + --if res1.status ~= 200 then + -- local api_keys = ngx.shared.api_keys + -- api_keys:delete(cached_key) + --end + ngx.exit(ngx.HTTP_OK) +end + +if custom_config then + local ok, c = pcall(function() return require(custom_config) end) + if ok and type(c) == 'table' and type(c.setup) == 'function' then + c.setup(_M) + end +end + +return _M + +-- END OF SCRIPT diff --git a/k8s/nginx-openresty/container/nginx_openresty_entrypoint.bash b/k8s/nginx-openresty/container/nginx_openresty_entrypoint.bash new file mode 100755 index 00000000..9a50fc20 --- /dev/null +++ b/k8s/nginx-openresty/container/nginx_openresty_entrypoint.bash @@ -0,0 +1,57 @@ +#!/bin/bash +set -euo pipefail + +# Openresty vars +dns_server=`printenv DNS_SERVER` +openresty_frontend_port=`printenv OPENRESTY_FRONTEND_PORT` + + +# BigchainDB vars +bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` +bdb_api_port=`printenv BIGCHAINDB_API_PORT` + + +# Read the 3scale credentials from the mountpoint +# Should be mounted at the following directory +THREESCALE_CREDENTIALS_DIR=/usr/local/openresty/nginx/conf/threescale + +threescale_secret_token=`cat ${THREESCALE_CREDENTIALS_DIR}/secret-token` +threescale_service_id=`cat ${THREESCALE_CREDENTIALS_DIR}/service-id` +threescale_version_header=`cat ${THREESCALE_CREDENTIALS_DIR}/version-header` +threescale_service_token=`cat ${THREESCALE_CREDENTIALS_DIR}/service-token` + + +if [[ -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${openresty_frontend_port:?OPENRESTY_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${bdb_backend_host:?BIGCHAINDB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?BIGCHAINDB_API_PORT not specified. Exiting!}" || \ + -z "${threescale_secret_token:?3scale secret token not specified. Exiting!}" || \ + -z "${threescale_service_id:?3scale service id not specified. Exiting!}" || \ + -z "${threescale_version_header:?3scale version header not specified. Exiting!}" || \ + -z "${threescale_service_token:?3scale service token not specified. Exiting!}" ]]; then + echo "Invalid environment settings detected. Exiting!" + exit 1 +fi + +NGINX_LUA_FILE=/usr/local/openresty/nginx/conf/nginx.lua +NGINX_CONF_FILE=/usr/local/openresty/nginx/conf/nginx.conf + +# configure the nginx.lua file with env variables +sed -i "s|SERVICE_ID|${threescale_service_id}|g" ${NGINX_LUA_FILE} +sed -i "s|THREESCALE_RESPONSE_SECRET_TOKEN|${threescale_secret_token}|g" ${NGINX_LUA_FILE} +sed -i "s|SERVICE_TOKEN|${threescale_service_token}|g" ${NGINX_LUA_FILE} + +# configure the nginx.conf file with env variables +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|OPENRESTY_FRONTEND_PORT|${openresty_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|THREESCALE_RESPONSE_SECRET_TOKEN|${threescale_secret_token}|g" $NGINX_CONF_FILE +sed -i "s|SERVICE_ID|${threescale_service_id}|g" $NGINX_CONF_FILE +sed -i "s|THREESCALE_VERSION_HEADER|${threescale_version_header}|g" $NGINX_CONF_FILE +sed -i "s|SERVICE_TOKEN|${threescale_service_token}|g" $NGINX_CONF_FILE + + +# start nginx +echo "INFO: starting nginx..." +exec /usr/local/openresty/nginx/sbin/nginx -c ${NGINX_CONF_FILE} diff --git a/pkg/Vagrantfile b/pkg/Vagrantfile new file mode 100644 index 00000000..4031df07 --- /dev/null +++ b/pkg/Vagrantfile @@ -0,0 +1,103 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Required modules +require 'yaml' + +# Minimum Requirements +Vagrant.require_version '>= 1.6.0' +VAGRANTFILE_API_VERSION = '2' + +# Configuration files +CONFIGURATION_FILE = 'configuration/vars/bdb-config.yml' +HOSTS_FILE = 'configuration/hosts/all' +HOST_VARS_PATH = 'configuration/host_vars' + +# Validate if all the required plugins are present +required_plugins = ["vagrant-cachier", "vagrant-vbguest", "vagrant-hosts"] +required_plugins.each do |plugin| + if not Vagrant.has_plugin?(plugin) + raise "Required vagrant plugin #{plugin} not found. Please run `vagrant plugin install #{plugin}`" + end +end + +# Read configuration file(s) +instances_config = YAML.load_file(File.join(File.dirname(__FILE__), CONFIGURATION_FILE)) +hosts_config = File.open(HOSTS_FILE, 'w+') +# TODO: (muawiakh) Add support for Docker, AWS, Azure +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + instances_config["bdb_hosts"].each do |instance| + # Workaround till canonical fixes https://bugs.launchpad.net/cloud-images/+bug/1569237 + # using -u ubuntu as remote user, conventionally vagrant boxes use `vagrant` user + if instance["box"]["name"] == "ubuntu/xenial64" + hosts_config.puts("#{instance["name"]} ansible_user=ubuntu") + else + hosts_config.puts("#{instance["name"]} ansible_user=vagrant") + end + config.vm.define instance['name'] do |bdb| + # Workaround until vagrant cachier plugin supports dnf + if !(instance["box"]["name"].include? "fedora") + if Vagrant.has_plugin?("vagrant-cachier") + bdb.cache.scope = :box + end + elsif instance["box"]["name"] == "ubuntu/xenial64" + if Vagrant.has_plugin?("vagrant-vbguest") + bdb.vbguest.auto_update = false + bdb.vbguest.no_install = true + bdb.vbguest.no_remote = true + end + end + bdb.vm.hostname = instance["name"] + if instance["network"]["type"] == "private_network" + bdb.vm.network instance["network"]["type"], ip: instance["network"]["ip"] + elsif instance["network"]["type"] == "public_network" + bdb.vm.network instance["network"]["type"], use_dhcp_assigned_default_route: true, bridge: instance["network"]["bridge"] + else + raise "Invalid network type: Please specify one of the following: [private_network, public_network]" + end + bdb.vm.provision :hosts, :sync_hosts => true + bdb.vm.box = instance["box"]["name"] + bdb.vm.synced_folder ".", "/bigchaindb" + File.open("#{HOST_VARS_PATH}/#{instance["name"]}", "w+") {|f| \ + f.write("ansible_ssh_private_key_file: /bigchaindb/.vagrant/machines/#{instance["name"]}/virtualbox/private_key") } + bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" + bdb.vm.provider 'vmware_fusion' do |vmwf, override| + vmwf.vmx['memsize'] = instance["ram"] + vmwf.vmx['numvcpus'] = instance['vcpus'] + end + + bdb.vm.provider 'virtualbox' do |vb, override| + vb.memory = instance["ram"] + vb.cpus = instance['vcpus'] + end + end + end + hosts_config.close + config.vm.define "config-node" do |bdb| + bdb.vm.box = "ubuntu/xenial64" + bdb.vm.hostname = "config-node" + bdb.vm.provision :hosts, :sync_hosts => true + bdb.vm.synced_folder ".", "/bigchaindb" + bdb.vm.network "private_network", ip: "192.168.100.200" + bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" + bdb.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /bigchaindb/configuration/bdb-deploy.yml \ + -i /bigchaindb/configuration/hosts/all" + bdb.vm.provider "virtualbox" do |vb| + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + vb.memory = 2048 + vb.cpus = 2 + end + bdb.vm.provider 'vmware_fusion' do |vmwf| + vmwf.vmx['memsize'] = 2048 + vmwf.vmx['numvcpus'] = 2 + end + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + config.vbguest.no_install = true + config.vbguest.no_remote = true + end + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + end +end diff --git a/pkg/configuration/bdb-deploy.yml b/pkg/configuration/bdb-deploy.yml new file mode 100644 index 00000000..8dd8c7ea --- /dev/null +++ b/pkg/configuration/bdb-deploy.yml @@ -0,0 +1,12 @@ +- import_playbook: pre_req.yml + +- hosts: all + vars_files: + - vars/bdb-config.yml + serial: 1 + roles: + - bigchaindb + - bigchaindb-driver + +- import_playbook: multi_node.yml + when: (bdb_hosts|length > 1) or docker_cluster_size|int > 1 diff --git a/pkg/configuration/group_vars/all b/pkg/configuration/group_vars/all new file mode 100644 index 00000000..f530443e --- /dev/null +++ b/pkg/configuration/group_vars/all @@ -0,0 +1,5 @@ +--- +ansible_connection: ssh +ansible_ssh_port: 22 +ansible_become: yes +ansible_ssh_common_args: '-o StrictHostKeyChecking=no' \ No newline at end of file diff --git a/pkg/configuration/host_vars/bdb-node-01 b/pkg/configuration/host_vars/bdb-node-01 new file mode 100644 index 00000000..fef41e4a --- /dev/null +++ b/pkg/configuration/host_vars/bdb-node-01 @@ -0,0 +1,5 @@ +# Place holder file for users, running Ansible playbooks manually. Otherwise Vagrant +# populates this dynamically. + +# Only needed for logging into remote hosts and adding host specific variables e.g. +#ansible_ssh_private_key_file: "/path/to/private/key" diff --git a/pkg/configuration/hosts/all b/pkg/configuration/hosts/all new file mode 100644 index 00000000..f84cc374 --- /dev/null +++ b/pkg/configuration/hosts/all @@ -0,0 +1,8 @@ +# Place holder file for users, running Ansible playbooks manually. Otherwise Vagrant +# populates this dynamically. + +# For local host +# ansible_connection=local + +# For remote host(s) +# ansible_ssh_user= ansible_sudo_pass= diff --git a/pkg/configuration/multi_node.yml b/pkg/configuration/multi_node.yml new file mode 100644 index 00000000..aa293ad9 --- /dev/null +++ b/pkg/configuration/multi_node.yml @@ -0,0 +1,5 @@ +- hosts: all + vars_files: + - vars/bdb-config.yml + roles: + - key-exchange \ No newline at end of file diff --git a/pkg/configuration/pre_req.yml b/pkg/configuration/pre_req.yml new file mode 100644 index 00000000..b7e54581 --- /dev/null +++ b/pkg/configuration/pre_req.yml @@ -0,0 +1,8 @@ +- hosts: all + vars_files: + - vars/bdb-config.yml + serial: 1 + roles: + - { role: docker, when: deploy_docker|bool } + - { role: docker-compose, when: deploy_docker|bool } + - mongodb \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml b/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml new file mode 100644 index 00000000..28c99323 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml @@ -0,0 +1,31 @@ +--- +dependencies_deb: + - python3-dev + - libffi-dev + - libssl-dev + - python3-pip + +dependencies_yum: + - gcc-c++ + - "@Development Tools" + - python35u-devel + - libffi-devel + - openssl-devel + - python35u-setuptools + +dependencies_dnf: + - gcc-c++ + - redhat-rpm-config + - "@Development Tools" + - python3-devel + - libffi-devel + - openssl-devel + - python3-pip + +python_pip_upgrade: true +python_setuptools_upgrade: true + +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml new file mode 100644 index 00000000..0eb69356 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml @@ -0,0 +1,12 @@ +--- +- name: Install dependencies + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ dependencies_yum }}" + tags: [bigchaindb-driver] + +- name: Install pip + shell: "easy_install-3.5 pip" + tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml new file mode 100644 index 00000000..55641bb4 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml @@ -0,0 +1,14 @@ +--- +- name: Upgrade pip + shell: "pip3 install --upgrade pip" + when: python_pip_upgrade + tags: [bigchaindb-driver] + +- name: Upgade setuptools + shell: "pip3 install --upgrade setuptools" + when: python_setuptools_upgrade + tags: [bigchaindb-driver] + +- name: Install BigchainDB Driver + shell: "pip3 install bigchaindb-driver" + tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml new file mode 100644 index 00000000..e00a7061 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml @@ -0,0 +1,8 @@ +--- +- name: Install dependencies + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ dependencies_deb }}" + tags: [bigchaindb] diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml new file mode 100644 index 00000000..bd8b701a --- /dev/null +++ b/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml @@ -0,0 +1,7 @@ +--- +- name: Install dependencies + dnf: + name: "{{ item }}" + state: present + with_items: "{{ dependencies_dnf }}" + tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml new file mode 100644 index 00000000..f70ef304 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml @@ -0,0 +1,12 @@ +--- + +- import_tasks: debian.yml + when: distribution_name == "debian" or distribution_name == "ubuntu" + +- import_tasks: centos.yml + when: distribution_name == "centos" or distribution_name == "red hat enterprise linux" + +- import_tasks: fedora.yml + when: distribution_name == "fedora" + +- import_tasks: common.yml \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/defaults/main.yml b/pkg/configuration/roles/bigchaindb/defaults/main.yml new file mode 100644 index 00000000..594deb20 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/defaults/main.yml @@ -0,0 +1,51 @@ +--- +dependencies_deb: + - g++ + - python3-dev + - libffi-dev + - build-essential + - libssl-dev + - python3-pip + +dependencies_yum: + - gcc-c++ + - "@Development Tools" + - python35u-devel + - libffi-devel + - openssl-devel + - python35u-setuptools + +dependencies_dnf: + - gcc-c++ + - redhat-rpm-config + - "@Development Tools" + - python3-devel + - libffi-devel + - openssl-devel + - python3-pip + +python_pip_upgrade: true +python_setuptools_upgrade: true + +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" + +directories: + - /data + +backend_db: mongodb #[mongodb] +bigchaindb_config_path: /data/.bigchaindb + +bigchaindb_server_bind: "0.0.0.0:9984" +bigchaindb_log_file: "{{ ansible_env.HOME }}/bigchaindb.log" + +# Docker configuration +bigchaindb_image_name: "bigchaindb/bigchaindb" +bigchaindb_docker_name: "bigchaindb" +bigchaindb_default_port: 9984 +bigchandb_host_port: 59984 +bigchaindb_host_mount_dir: "{{ ansible_env.HOME }}/bigchaindb_docker" +# Default IP of docker0 bridge +bigchaindb_default_host: "172.17.0.1" diff --git a/pkg/configuration/roles/bigchaindb/tasks/centos.yml b/pkg/configuration/roles/bigchaindb/tasks/centos.yml new file mode 100644 index 00000000..912d0f74 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/centos.yml @@ -0,0 +1,20 @@ +--- +- name: Creating directories | CentOS + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + tags: [bigchaindb] + +- name: Install dependencies | CentOS + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ dependencies_yum }}" + tags: [bigchaindb] + +- name: Install pip + shell: "easy_install-3.5 pip" + tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/common.yml b/pkg/configuration/roles/bigchaindb/tasks/common.yml new file mode 100644 index 00000000..d29cc3ef --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/common.yml @@ -0,0 +1,59 @@ +--- +- name: Upgrade pip + shell: "pip3 install --upgrade pip" + when: python_pip_upgrade + tags: [bigchaindb] + +- name: Upgade setuptools + shell: "pip3 install --upgrade setuptools" + when: python_setuptools_upgrade + tags: [bigchaindb] + +- name: Install BigchainDB + shell: "pip3 install bigchaindb" + tags: [bigchaindb] + +- name: Check if BigchainDB node is already configured + stat: + path: "{{ bigchaindb_config_path }}" + register: stat_result + +- name: Configure BigchainDB + shell: "bigchaindb -y configure {{ backend_db }}" + environment: + BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" + BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path }}" + BIGCHAINDB_DATABASE_HOST: "{{ ansible_hostname }}" + when: stat_result.stat.exists == False + tags: [bigchaindb] + +- name: MongoDB Process Check + shell: pgrep mongod | wc -l + register: mdb_pchk + tags: [bigchaindb] + +- name: BigchainDB Process Check + shell: pgrep bigchaindb | wc -l + register: bdb_pchk + tags: [bigchaindb] + +- name: Start BigchainDB + shell: "bigchaindb start > {{ bigchaindb_log_file }} 2>&1 &" + environment: + BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path }}" + when: mdb_pchk.stdout| int >= 1 and bdb_pchk.stdout| int == 0 + async: 10 + poll: 0 + tags: [bigchaindb] + +- name: Get BigchainDB node public key + shell: "cat {{ bigchaindb_config_path }}" + register: bdb_node_config + tags: [bigchaindb] + +- name: Set Facts BigchainDB + set_fact: + pub_key="{{ ( bdb_node_config.stdout|from_json).keypair.public }}" + hostname="{{ ansible_hostname }}" + bdb_config="{{ bigchaindb_config_path }}" + tags: [bigchaindb] diff --git a/pkg/configuration/roles/bigchaindb/tasks/debian.yml b/pkg/configuration/roles/bigchaindb/tasks/debian.yml new file mode 100644 index 00000000..0686848a --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/debian.yml @@ -0,0 +1,8 @@ +--- +- name: Install dependencies + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ dependencies_deb }}" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml new file mode 100644 index 00000000..98fd90d5 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml @@ -0,0 +1,48 @@ +--- +- name: Check if BigchainDB Dockers are already configured + stat: + path: "{{ bigchaindb_host_mount_dir }}{{ item|string }}/.bigchaindb" + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + register: stat_result + tags: [bigchaindb] + +- name: Configuring BigchainDB Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + hostname: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}" + volumes: + - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" + env: + BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" + BIGCHAINDB_DATABASE_HOST: "{{ hostvars[ansible_hostname]['mongodb' + item|string] }}" + entrypoint: "bigchaindb -y configure mongodb" + when: stat_result.results[item|int].stat.exists == False + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] + +- name: Start BigchainDB Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}" + detach: true + published_ports: + - "{{ bigchandb_host_port|int + item|int }}:{{ bigchaindb_default_port }}" + restart_policy: always + volumes: + - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" + state: started + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] + +- name: Get BigchainDB node public key + shell: "cat {{ bigchaindb_host_mount_dir + item|string }}/.bigchaindb" + register: bdb_node_config + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] + +- name: Set facts for BigchainDB containers + set_fact: + pub_key_{{ bigchaindb_docker_name }}{{ item }}="{{ (bdb_node_config.results[item|int].stdout|from_json).keypair.public }}" + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [bigchaindb] diff --git a/pkg/configuration/roles/bigchaindb/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml new file mode 100644 index 00000000..4f542d3c --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml @@ -0,0 +1,7 @@ +--- +- name: Install dependencies + dnf: + name: "{{ item }}" + state: present + with_items: "{{ dependencies_dnf }}" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/main.yml b/pkg/configuration/roles/bigchaindb/tasks/main.yml new file mode 100644 index 00000000..3b1de267 --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- import_tasks: deploy_docker.yml + when: deploy_docker|bool + tags: [bigchaindb] + +- import_tasks: debian.yml + when: not deploy_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [bigchaindb] + +- import_tasks: centos.yml + when: not deploy_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [bigchaindb] + +- import_tasks: fedora.yml + when: not deploy_docker|bool and (distribution_name == "fedora") + tags: [bigchaindb] + +- import_tasks: common.yml + when: not deploy_docker|bool + tags: [bigchaindb] diff --git a/pkg/configuration/roles/docker-compose/defaults/main.yml b/pkg/configuration/roles/docker-compose/defaults/main.yml new file mode 100644 index 00000000..e50ee45c --- /dev/null +++ b/pkg/configuration/roles/docker-compose/defaults/main.yml @@ -0,0 +1,7 @@ +--- +# TODO: (muawiakh) Install docker-compose using pip +# docker_compose_pip_install: false #[true, false] +docker_compose_install: true #[true, false] +docker_compose_version: "1.15.0" +docker_compose_binary: /usr/local/bin/docker-compose +docker_compose_base_url: "https://github.com/docker/compose/releases/download/" \ No newline at end of file diff --git a/pkg/configuration/roles/docker-compose/tasks/main.yml b/pkg/configuration/roles/docker-compose/tasks/main.yml new file mode 100644 index 00000000..2862061e --- /dev/null +++ b/pkg/configuration/roles/docker-compose/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Get docker-compose + become: yes + get_url: + url: "{{ docker_compose_base_url }}{{ docker_compose_version }}/docker-compose-{{ ansible_system }}-{{ ansible_machine}}" + dest: "{{docker_compose_binary}}" + mode: 0755 + when: docker_compose_install + tags: [docker-compose] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/defaults/main.yml b/pkg/configuration/roles/docker/defaults/main.yml new file mode 100644 index 00000000..b1eda858 --- /dev/null +++ b/pkg/configuration/roles/docker/defaults/main.yml @@ -0,0 +1,18 @@ +--- +uninstall_old_version: false #[true, false] +docker_edition: 'ce' #[ce, ee] Currently, onlt CE is supported +docker_pkg: "docker-{{ docker_edition }}" #[docker-ce, docker-ee] +docker_update_channel: "stable" #[stable, edge] + +# Host configuration +distribution_name: "{{ ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" +server_arch: "amd64" #[amd64, armhf, s390x] + +# Docker Repositories +docker_apt_repo: "deb [arch={{ server_arch }}] https://download.docker.com/linux/{{ distribution_name }} {{ distribution_codename }} {{ docker_update_channel }}" +apt_key_fingerprint: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88" +apt_key_url: "https://download.docker.com/linux/{{ distribution_name }}/gpg" +docker_yum_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ distribution_major }}/$basearch/{{ docker_update_channel }}" +docker_dnf_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ docker_pkg }}.repo" \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/centos.yml b/pkg/configuration/roles/docker/tasks/centos.yml new file mode 100644 index 00000000..62d40d53 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/centos.yml @@ -0,0 +1,42 @@ +--- +- name: Uninstall older versions of Docker | CentOS + yum: + name: "{{ item }}" + state: absent + with_items: + - docker + - docker-common + - docker-engine + - docker-selinux + when: uninstall_old_version + tags: [docker] + +- name: Setup Pre-reqs | CentOS + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - yum-utils + - device-mapper-persistent-data + - lvm2 + - python-pip + tags: [docker] + +- name: Add Docker Repo | CentOS + yum_repository: + name: "{{ docker_pkg }}" + gpgcheck: yes + gpgkey: https://download.docker.com/linux/centos/gpg + baseurl: "{{ docker_yum_repo }}" + file: "{{ docker_pkg }}" + description: "Docker Repo" + enabled: yes + tags: [docker] + +- name: Install Docker | CentOS + yum: + name: "{{ docker_pkg }}" + state: present + update_cache: yes + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/debian.yml b/pkg/configuration/roles/docker/tasks/debian.yml new file mode 100644 index 00000000..483b7ac4 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/debian.yml @@ -0,0 +1,52 @@ +--- +- name: Uninstall older versions of Docker | Debian + apt: + name: "{{ item }}" + state: absent + with_items: + - docker + - docker-engine + - docker.io + when: uninstall_old_version + tags: [docker] + +- name: Install dependencies | Debian + apt: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + - curl + - software-properties-common + - python-pip + tags: [docker] + +- name: Add APT Key | Debian + apt_key: + url: "{{ apt_key_url }}" + id: "{{ apt_key_fingerprint }}" + state: present + register: add_repository_key + ignore_errors: true + tags: [docker] + +- name: Use curl if apt_key fails | Debian + shell: "curl -sSl {{ apt_key_url }} | sudo apt-key add -" + args: + warn: no + when: add_repository_key|failed + tags: [docker] + +- name: Add Docker repo and update cache | Debian + apt_repository: + repo: "{{ docker_apt_repo }}" + update_cache: yes + state: present + tags: [docker] + +- name: Install Docker | Debian + apt: + name: "{{ docker_pkg }}" + state: present + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/fedora.yml b/pkg/configuration/roles/docker/tasks/fedora.yml new file mode 100644 index 00000000..93d82b20 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/fedora.yml @@ -0,0 +1,34 @@ +--- +- name: Uninstall older versions of Docker | Fedora + dnf: + name: "{{ item }}" + state: absent + with_items: + - docker-engine-selinux + - docker-common + - docker-engine + - docker-selinux + when: uninstall_old_version + tags: [docker] + +- name: Setup Pre-reqs | Fedora + dnf: + name: "{{ item }}" + state: present + with_items: + - dnf-plugins-core + tags: [docker] + +- name: Add Docker repo | Fedora + shell: "dnf config-manager --add-repo {{ docker_dnf_repo }}" + tags: [docker] + +- name: Update Cache | Fedora + shell: "dnf makecache fast" + tags: [docker] + +- name: Install Docker | Fedora + dnf: + name: "{{ docker_pkg }}" + state: present + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/main.yml b/pkg/configuration/roles/docker/tasks/main.yml new file mode 100644 index 00000000..5676e153 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- import_tasks: debian.yml + when: distribution_name == "debian" or distribution_name == "ubuntu" + +- import_tasks: centos.yml + when: distribution_name == "centos" or distribution_name == "red hat enterprise linux" + +- import_tasks: fedora.yml + when: distribution_name == "fedora" + +- name: Create Docker group + group: + name: docker + state: present + register: group_result + tags: [docker] + +- name: Add USER to docker group + user: + append: yes + name: "{{ item }}" + state: present + group: docker + with_items: + - vagrant + - "{{ distribution_name }}" + tags: [docker] + +- name: Start docker service + systemd: + name: docker + enabled: yes + state: started + tags: [docker] + +- name: Install docker-py + pip: + name: docker-py + state: present + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/defaults/main.yml b/pkg/configuration/roles/key-exchange/defaults/main.yml new file mode 100644 index 00000000..6bfe9035 --- /dev/null +++ b/pkg/configuration/roles/key-exchange/defaults/main.yml @@ -0,0 +1,12 @@ +keyring_script_host: /tmp/keyring.py +bigchaindb_log_file_host: "{{ ansible_env.HOME }}/bigchaindb.log" +bigchaindb_config_path_host: /data/.bigchaindb + +# Docker configuration +keyring_script_docker: "{{ ansible_env.HOME }}/config/keyring.py" +bigchaindb_config_path_docker: "{{ ansible_env.HOME }}/bigchaindb_docker" +bigchaindb_docker_name: bigchaindb +bigchaindb_default_port: 9984 +bigchandb_host_port: 59984 +bigchaindb_host_mount_dir: "{{ ansible_env.HOME }}/bigchaindb_docker" +bigchaindb_image_name: "bigchaindb/bigchaindb" diff --git a/pkg/configuration/roles/key-exchange/tasks/main.yml b/pkg/configuration/roles/key-exchange/tasks/main.yml new file mode 100644 index 00000000..031bf16b --- /dev/null +++ b/pkg/configuration/roles/key-exchange/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- include_tasks: pub_key_exchange_host.yml + when: not deploy_docker|bool + tags: [bigchaindb] + +- include_tasks: pub_key_exchange_docker.yml + when: deploy_docker|bool + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml new file mode 100644 index 00000000..eaf0eb14 --- /dev/null +++ b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml @@ -0,0 +1,29 @@ +--- +- name: Creating files for key exchange + template: src=exchange_keyring_docker.j2 dest="{{ keyring_script_docker }}" + tags: [keyex] + +- name: Setting permissions + file: + path: "{{ keyring_script_docker }}" + mode: "0777" + tags: [keyex] + +- name: Update Keyring Configuration + shell: "python {{ keyring_script_docker }}" + tags: [keyex] + +- name: Restart BigchainDB Docker after keyring update + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}" + detach: true + published_ports: + - "{{ bigchandb_host_port|int + item|int }}:{{ bigchaindb_default_port }}" + restart_policy: always + volumes: + - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" + state: started + restart: true + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [keyex] diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml new file mode 100644 index 00000000..f3d5c4d6 --- /dev/null +++ b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml @@ -0,0 +1,28 @@ +--- +- name: Creating files for key exchange + template: src=exchange_keyring_host.j2 dest="{{ keyring_script_host }}" + tags: [keyex] + +- name: Setting permissions + file: + path: "{{ keyring_script_host }}" + mode: "0777" + tags: [keyex] + +- name: Update Keyring Configuration + shell: "python {{ keyring_script_host }}" + tags: [keyex] + +- name: Stop BigchainDB + shell: pkill bigchaindb + register: bdb_stop + tags: [keyex] + +- name: Start BigchainDB + shell: "bigchaindb start > {{ bigchaindb_log_file_host }} 2>&1 &" + environment: + BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path_host }}" + async: 10 + poll: 0 + when: bdb_stop|succeeded + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 new file mode 100644 index 00000000..31aed4fb --- /dev/null +++ b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 @@ -0,0 +1,18 @@ +#!/usr/bin/python +import json +{% set keyring = {} %} +{% for docker in range(0, docker_cluster_size|int, 1) %} + {{- keyring.update({'pub_key_' + bigchaindb_docker_name + docker|string: hostvars[ansible_hostname]['pub_key_' + bigchaindb_docker_name + docker|string]}) -}} +{%- endfor -%} +{% for docker in range(0, docker_cluster_size|int, 1) %} +keyring = {{ keyring }} +keyring.pop('{{ 'pub_key_' + bigchaindb_docker_name + docker|string }}', None) +with open('{{ bigchaindb_config_path_docker + docker|string }}/.bigchaindb', 'r+') as f: + data = json.load(f) + del data['keyring'][:] + for key, value in keyring.iteritems(): + data['keyring'].append(value) + f.seek(0) + json.dump(data, f, indent=4) + f.truncate() +{% endfor %} \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 new file mode 100644 index 00000000..f02a208c --- /dev/null +++ b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 @@ -0,0 +1,21 @@ +{%- set keyring = [] -%} +{%- set bdb_config_path = {'path': ''} -%} +{%- for host in bdb_hosts -%} + {%- if host["name"] != ansible_hostname -%} + {{- keyring.append(hostvars[host["name"]]["pub_key"]) -}} + {%- else -%} + {%- if bdb_config_path.update({'path': hostvars[host["name"]]["bdb_config"]}) -%} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if keyring|length != 0 -%} +#!/usr/bin/python +import json +with open('{{ bdb_config_path['path'] }}', 'r+') as f: + data = json.load(f) + del data['keyring'][:] + data['keyring'] = {{ keyring }} + f.seek(0) + json.dump(data, f, indent=4) + f.truncate() +{%- endif -%} \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/defaults/main.yml b/pkg/configuration/roles/mongodb/defaults/main.yml new file mode 100644 index 00000000..7088a61b --- /dev/null +++ b/pkg/configuration/roles/mongodb/defaults/main.yml @@ -0,0 +1,39 @@ +--- +mongodb_version: "3.4" +mongodb_package: "mongodb-org" +apt_key_fingerprint: "0C49F3730359A14518585931BC711F9BA15703C6" +apt_keyserver: "keyserver.ubuntu.com" +distribution_name: "{{ansible_distribution|lower }}" +distribution_codename: "{{ ansible_distribution_release|lower }}" +distribution_major: "{{ ansible_distribution_major_version }}" +server_arch: "amd64,arm64" + +# MongoDB Repos +mongodb_apt_repo: "deb [arch={{ server_arch }}] http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongodb_version }} {{'main' if ansible_distribution == 'debian' else 'multiverse'}}" +mongodb_yum_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/$releasever/{{ mongodb_package }}/{{ mongodb_version }}/{{ ansible_architecture }}" +mongodb_dnf_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/7/{{ mongodb_package }}/{{ mongodb_version }}/{{ ansible_architecture }}" + +# MongoDB running config +mongodb_storage_path: /data/db/main +mongodb_log_path: /var/log/mongodb +mongodb_config_path: /data/configdb +directories: + - "{{ mongodb_storage_path }}" + - "{{ mongodb_log_path }}" + - "{{ mongodb_config_path }}" + +mongodb_conf_file: /etc/mongod.conf +mongodb_conf_files: [ + { src: "mongod.conf", dest: "{{ mongodb_conf_file }}"} +] +mongodb_port: 27017 +mongodb_admin_user: "adminUser" +mongodb_admin_password: "superstrongpassword" + +replica_set_name: bigchain-rs + +# Docker configuration +mongodb_docker_image: "mongo:3.4.9" +mongodb_docker_name: "mongodb" +mongodb_host_mount_dir: "{{ ansible_env.HOME }}/mongodb_docker" +mongodb_host_config: "{{ ansible_env.HOME }}/config" diff --git a/pkg/configuration/roles/mongodb/files/mongod.conf b/pkg/configuration/roles/mongodb/files/mongod.conf new file mode 100644 index 00000000..ed961801 --- /dev/null +++ b/pkg/configuration/roles/mongodb/files/mongod.conf @@ -0,0 +1,101 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# where to write logging data. +systemLog: + verbosity: 0 + # traceAllExceptions: true + timeStampFormat: iso8601-utc + component: + accessControl: + verbosity: 0 + command: + verbosity: 0 + control: + verbosity: 0 + ftdc: + verbosity: 0 + geo: + verbosity: 0 + index: + verbosity: 0 + network: + verbosity: 0 + query: + verbosity: 0 + replication: + verbosity: 0 + sharding: + verbosity: 0 + storage: + verbosity: 0 + journal: + verbosity: 0 + write: + verbosity: 0 + +processManagement: + fork: false + pidFilePath: /tmp/mongod.pid + +net: + port: 27017 + bindIp: 0.0.0.0 + maxIncomingConnections: 8192 + wireObjectCheck: false + unixDomainSocket: + enabled: false + pathPrefix: /tmp + filePermissions: 0700 + http: + enabled: false + compression: + compressors: snappy +# ssl: +# mode: requireSSL +# PEMKeyFile: MONGODB_KEY_FILE_PATH +# PEMKeyPassword: MONGODB_KEY_FILE_PASSWORD +# CAFile: MONGODB_CA_FILE_PATH +# CRLFile: MONGODB_CRL_FILE_PATH + +# allowConnectionsWithoutCertificates: false +# allowInvalidHostnames: false +# weakCertificateValidation: false +# allowInvalidCertificates: false + +#security: +# authorization: enabled +# clusterAuthMode: x509 + +#setParameter: +# enableLocalhostAuthBypass: true +# #notablescan: 1 +# logUserIds: 1 +# authenticationMechanisms: MONGODB-X509,SCRAM-SHA-1 + +storage: + dbPath: /data/db/main + indexBuildRetry: true + journal: + enabled: true + commitIntervalMs: 100 + directoryPerDB: true + engine: wiredTiger + wiredTiger: + engineConfig: + journalCompressor: snappy +# configString: cache_size=STORAGE_ENGINE_CACHE_SIZE + collectionConfig: + blockCompressor: snappy + indexConfig: + prefixCompression: true # TODO false may affect performance? + +operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + +replication: + replSetName: bigchain-rs + enableMajorityReadConcern: true diff --git a/pkg/configuration/roles/mongodb/tasks/centos.yml b/pkg/configuration/roles/mongodb/tasks/centos.yml new file mode 100644 index 00000000..62e8faa0 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/centos.yml @@ -0,0 +1,25 @@ +--- +- name: Add MongoDB Repo | CentOS + yum_repository: + name: "{{ mongodb_package }}" + gpgcheck: yes + gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongodb_version }}.asc + baseurl: "{{ mongodb_yum_base_url }}" + file: "{{ mongodb_package }}" + description: "MongoDB Repo" + enabled: yes + tags: [mongodb] + +- name: Install MongoDB | CentOS + yum: + name: "{{ mongodb_package }}" + state: present + update_cache: yes + tags: [mongodb] + +- name: Install pip | CentOS + yum: + name: python-pip + state: present + update_cache: yes + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/common.yml b/pkg/configuration/roles/mongodb/tasks/common.yml new file mode 100644 index 00000000..40369191 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/common.yml @@ -0,0 +1,25 @@ +--- +- name: MongoDB config files are copied + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: "{{ mongodb_conf_files }}" + tags: [mongodb] + +- name: MongoDB Process Check + shell: pgrep mongod | wc -l + register: command_result + tags: [mongodb] + +- name: Install pymongo + pip: + name: pymongo + state: present + tags: [mongodb] + +- name: Run MongoDB + shell: "mongod --config {{ mongodb_conf_file }} 2>&1 &" + when: command_result.stdout| int != 1 + async: 5 + poll: 0 + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/debian.yml b/pkg/configuration/roles/mongodb/tasks/debian.yml new file mode 100644 index 00000000..aac606fa --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/debian.yml @@ -0,0 +1,22 @@ +--- +- name: Add APT Key | Debian + apt_key: + keyserver: "{{ apt_keyserver }}" + id: "{{ apt_key_fingerprint }}" + tags: [mongodb] + +- name: Add MongoDB repo and update cache | Debian + apt_repository: + repo: "{{ mongodb_apt_repo }}" + update_cache: yes + state: present + tags: [mongodb] + +- name: Install MongoDB | Debian + apt: + name: "{{ item }}" + state: present + with_items: + - "{{ mongodb_package }}" + - python-pip + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml new file mode 100644 index 00000000..ecadea49 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml @@ -0,0 +1,33 @@ +--- +- name: Check Docker Service + systemd: + name: docker + enabled: yes + state: started + tags: [mongodb] + +- name: Running MongoDB Docker + docker_container: + name: "{{ mongodb_docker_name }}{{ item }}" + hostname: "{{ mongodb_docker_name }}{{ item }}" + image: "{{ mongodb_docker_image }}" + detach: true + published_ports: + - "{{ (mongodb_port|int + item|int)|string }}:{{ mongodb_port }}" + restart_policy: always + volumes: + - "{{ mongodb_host_mount_dir }}{{ item|string }}/db:{{ mongodb_storage_path }}" + - "{{ mongodb_host_mount_dir }}{{ item|string }}/configdb:{{ mongodb_config_path }}" + - "{{ mongodb_host_config }}:/bdb_config" + state: started + keep_volumes: true + entrypoint: /entrypoint.sh --replSet=bigchain-rs + register: mongo_container_info + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [mongodb] + +- name: Set facts for MongoDB containers + set_fact: + mongodb{{ item }}={{ mongo_container_info.results[item|int].ansible_facts.docker_container.NetworkSettings.IPAddress }} + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 + tags: [mongodb] diff --git a/pkg/configuration/roles/mongodb/tasks/fedora.yml b/pkg/configuration/roles/mongodb/tasks/fedora.yml new file mode 100644 index 00000000..c2f61110 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/fedora.yml @@ -0,0 +1,19 @@ +--- +- name: Add MongoDB Repo | Fedora + yum_repository: + name: "{{ mongodb_package }}" + gpgcheck: yes + gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongodb_version }}.asc + baseurl: "{{ mongodb_dnf_base_url }}" + file: "{{ mongodb_package }}" + description: "MongoDB Repo" + enabled: yes + tags: [mongodb] + +- name: Install MongoDB | Fedora + dnf: + name: "{{ item }}" + state: present + with_items: + - "{{ mongodb_package }}" + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml new file mode 100644 index 00000000..ecbdc625 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml @@ -0,0 +1,6 @@ +--- +- import_tasks: initiate_repl_set_host.yml + when: (ansible_hostname == bdb_hosts[bdb_hosts|length-1]['name']) and not deploy_docker|bool + +- import_tasks: initiate_repl_set_docker.yml + when: deploy_docker|bool and docker_cluster_size|int > 1 diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml new file mode 100644 index 00000000..5cd341fa --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml @@ -0,0 +1,13 @@ +--- +- name: Creating files to initialize MongoDB Replica Set | Docker + template: src=replSet_init_docker.j2 dest="{{ mongodb_host_config }}/replSet_init.js" + tags: [mongodb] + +- name: Initializing Replica Set and Adding AdminUser | Docker + run_once: true + shell: + cmd: + "docker exec {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} bash -l -c + '/usr/bin/mongo --host {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} + --port {{ mongodb_port }} < /bdb_config/replSet_init.js'" + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml new file mode 100644 index 00000000..8d398bcb --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml @@ -0,0 +1,20 @@ +--- +- name: Creating files to initialize MongoDB Replica Set + template: src=replSet_init_host.j2 dest=/tmp/replSet_init.js + tags: [mongodb] + +- name: Initializing Replica Set + shell: "/usr/bin/mongo --host {{ ansible_hostname }} --port {{ mongodb_port }} < /tmp/replSet_init.js" + tags: [mongodb] + +- name: Adding AdminUser to MongoDB + run_once: true + mongodb_user: + database: admin + login_host: "{{ ansible_hostname }}" + login_port: "{{ mongodb_port }}" + name: "{{ mongodb_admin_user }}" + password: "{{ mongodb_admin_password }}" + roles: readWriteAnyDatabase,clusterManager + state: present + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/main.yml b/pkg/configuration/roles/mongodb/tasks/main.yml new file mode 100644 index 00000000..8daeffc6 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Creating directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + when: not deploy_docker|bool + tags: [mongodb] + +- import_tasks: deploy_docker.yml + when: deploy_docker|bool + tags: [mongodb] + +- import_tasks: debian.yml + when: not deploy_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [mongodb] + +- import_tasks: centos.yml + when: not deploy_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [mongodb] + +- import_tasks: fedora.yml + when: not deploy_docker|bool and (distribution_name == "fedora") + tags: [mongodb] + +- import_tasks: common.yml + when: not deploy_docker|bool + tags: [mongodb] + +- import_tasks: initiate_repl_set.yml diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 new file mode 100644 index 00000000..ed8c8fa4 --- /dev/null +++ b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 @@ -0,0 +1,30 @@ +rs.initiate({ + "_id": "{{ replica_set_name }}", + "members": [ + { + "_id": 0, + "host": "{{ hostvars[ansible_hostname][mongodb_docker_name + (docker_cluster_size|int - 1)|string] }}:{{ mongodb_port }}" + } + ] +}); +sleep(5000); +{% for docker in range(0, docker_cluster_size|int, 1) %} +{%- if docker != (docker_cluster_size|int - 1) -%} +rs.add("{{ hostvars[ansible_hostname][mongodb_docker_name + docker|string] }}:{{ mongodb_port }}"); +{% endif %} +{%- endfor -%} +use admin; +db.createUser(db.createUser({ + "user": "{{ mongodb_admin_user }}", + "pwd": "{{ mongodb_admin_password }}", + "roles": [ + { + "role": "userAdminAnyDatabase", + "db": "admin" + }, + { + "role": "clusterManager", + "db": "admin" + } + ] +}); diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 new file mode 100644 index 00000000..42bca2a3 --- /dev/null +++ b/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 @@ -0,0 +1,7 @@ +rs.initiate( { _id : "{{ replica_set_name }}", members: [ { _id : 0, host :"{{ bdb_hosts[bdb_hosts|length-1]['name'] }}:{{ mongodb_port }}" } ] } ) +sleep(5000); +{% for host in bdb_hosts %} +{%- if ansible_hostname != host["name"] -%} +rs.add("{{ host["name"] }}:{{ mongodb_port }}"); +{% endif %} +{%- endfor -%} \ No newline at end of file diff --git a/pkg/configuration/vars/bdb-config.yml b/pkg/configuration/vars/bdb-config.yml new file mode 100644 index 00000000..f34cd7c7 --- /dev/null +++ b/pkg/configuration/vars/bdb-config.yml @@ -0,0 +1,13 @@ +--- +deploy_docker: false #[true, false] +docker_cluster_size: 1 +upstart: "/bigchaindb/scripts/bootstrap.sh" +bdb_hosts: + - name: "bdb-node-01" + box: + name: "ubuntu/xenial64" + ram: "2048" + vcpus: "2" + network: + ip: "10.20.30.20" + type: "private_network" diff --git a/pkg/scripts/bootstrap.sh b/pkg/scripts/bootstrap.sh new file mode 100755 index 00000000..9fcf5612 --- /dev/null +++ b/pkg/scripts/bootstrap.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +. ./bootstrap_constants.sh +. ./bootstrap_helper.sh + +# OS ID(ubuntu, centos, fedora) +OS="" +# OS Version(16.04, 7, 24) +VER="" + +# Parsing arguments +while [[ $# -gt 1 ]]; do + arg="$1" + case $arg in + --os) + OS="$2" + shift + ;; + --os-version) + VER="$2" + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift +done + +validate_os_configuration(){ + valid_os=1 + if [ -f $1 ]; then + . $1 + OS=$ID + VER=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + OS=$(lsb_release -si) + VER=$(lsb_release -sr) + else + echo "Cannot find $OS_CONF. Pass arguments to your OS configurations: NAME, VERSION_ID. + Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + exit 1 + fi + for os in "${SUPPORTED_OS[@]}"; do + if [[ $os = $2 ]]; then + valid_os=true + break + fi + done +} + +validate_os_configuration $OS_CONF $OS $VER +echo "Operation Sytem: $OS" +echo "Version: $VER" +install_deps=$(validate_os_version_and_deps true $OS $VER) +if [[ $install_deps -eq 1 ]]; then + install_dependencies $OS +else + echo "Dependencies already installed:[ ${OS_DEPENDENCIES[*]} ]" +fi \ No newline at end of file diff --git a/pkg/scripts/bootstrap_constants.sh b/pkg/scripts/bootstrap_constants.sh new file mode 100755 index 00000000..67a6d7ef --- /dev/null +++ b/pkg/scripts/bootstrap_constants.sh @@ -0,0 +1,7 @@ +#!/bin/bash +OS_CONF=/etc/os-release +declare -a SUPPORTED_OS=('ubuntu' 'centos' 'fedora') +declare -a OS_DEPENDENCIES=('ansible') +MINIMUM_UBUNTU_VERSION=16.04 +MINIUMUM_CENTOS_VERSION=7 +MINIMIUM_FEDORA_VERSION=24 \ No newline at end of file diff --git a/pkg/scripts/bootstrap_helper.sh b/pkg/scripts/bootstrap_helper.sh new file mode 100755 index 00000000..8afb4d7e --- /dev/null +++ b/pkg/scripts/bootstrap_helper.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +. ./bootstrap_constants.sh + +validate_os_version_and_deps(){ + if $1; then + case $2 in + ubuntu) + apt-get install bc -y > /dev/null 2>&1 + if [[ ($(echo $3 | bc) > $MINIMUM_UBUNTU_VERSION) + || ($(echo $3 | bc) == $MINIMUM_UBUNTU_VERSION)]]; then + dpkg -s "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo "Supported $2 Versions: >= $MINIMUM_UBUNTU_VERSION" + exit 1 + fi + ;; + centos) + yum install bc -y > /dev/null 2>&1 + if [[ ($(echo $3 | bc) > $MINIMUM_CENTOS_VERSION) + || ($(echo $3 | bc) == $MINIMUM_CENTOS_VERSION) ]]; then + rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo "Supported $2 Versions: >= $MINIMUM_CENTOS_VERSION" + exit 1 + fi + ;; + fedora) + dnf install bc python2-dnf libselinux-python -y > /dev/null 2>&1 + if [[ ($(echo $3 | bc) > $MINIMUM_FEDORA_VERSION) + || ($(echo $3 | bc) == $MINIMUM_FEDORA_VERSION) ]]; then + rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1 + echo $? + else + echo "Supported $2 Versions: >= $MINIMUM_FEDORA_VERSION" + exit 1 + fi + ;; + *) + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + exit 1 + ;; + esac + else + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + exit 1 + fi +} + +install_dependencies() { + case $1 in + ubuntu) + install_deps_deb + ;; + centos) + install_deps_centos + ;; + fedora) + install_deps_fedora + ;; + *) + echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]." + exit 1 + ;; + esac +} + +#TODO: muawiakh(Currently only ansible is required. Make it generic for +# multiple dependencies) +install_deps_deb() { + echo "Installing Dependencies..." + apt-get update -y + apt-get install -y software-properties-common + apt-add-repository ppa:ansible/ansible + apt-get update -y + apt-get install -y "${OS_DEPENDENCIES[@]}" +} +install_deps_centos() { + echo "Installing Dependencies..." + yum install epel-release -y + yum install -y https://centos7.iuscommunity.org/ius-release.rpm + yum install "${OS_DEPENDENCIES[@]}" -y +} +install_deps_fedora() { + echo "Installing Dependencies..." + export LC_ALL=C + dnf makecache + echo "${OS_DEPENDENCIES[@]}" + dnf -y install "${OS_DEPENDENCIES[@]}" +} diff --git a/tests/README.md b/tests/README.md index 252fcda4..146d3bc6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -105,17 +105,17 @@ $ docker-compose build First, start `RethinkDB` in the background: ```text -$ docker-compose up -d rdb +$ docker-compose -f docker-compose.rdb.yml up -d rdb ``` then run the tests using: ```text -$ docker-compose run --rm bdb-rdb py.test -v +$ docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb py.test -v ``` to rebuild all the images (usually you only need to rebuild the `bdb` and - `bdb-rdb` images). + `bdb-rdb` images). If that fails, then do `make clean-pyc` and try again. ## Automated Testing of All Pull Requests diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index 0fd7229a..113f4249 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -299,12 +299,13 @@ def test_count_blocks(signed_create_tx): from bigchaindb.models import Block conn = connect() + assert query.count_blocks(conn) == 0 + # create and insert some blocks block = Block(transactions=[signed_create_tx]) conn.db.bigchain.insert_one(block.to_dict()) - conn.db.bigchain.insert_one(block.to_dict()) - assert query.count_blocks(conn) == 2 + assert query.count_blocks(conn) == 1 def test_count_backlog(signed_create_tx): @@ -352,6 +353,7 @@ def test_get_genesis_block(genesis_block): conn = connect() assets, genesis_block_dict = genesis_block.decouple_assets() + metadata, genesis_block_dict = genesis_block.decouple_metadata(genesis_block_dict) assert query.get_genesis_block(conn) == genesis_block_dict @@ -419,7 +421,8 @@ def test_get_new_blocks_feed(b, create_tx): ts = str(random.random()) block = Block(transactions=[create_tx], timestamp=ts) b.write_block(block) - return block.decouple_assets()[1] + block_dict = block.decouple_assets()[1] + return block.decouple_metadata(block_dict)[1] create_block() b1 = create_block() @@ -526,13 +529,14 @@ def test_get_assets(): assert list(cursor.sort('id', pymongo.ASCENDING)) == assets[::2] -def test_text_search(): +@pytest.mark.parametrize("table", ['assets', 'metadata']) +def test_text_search(table): from bigchaindb.backend import connect, query conn = connect() # Example data and tests cases taken from the mongodb documentation # https://docs.mongodb.com/manual/reference/operator/query/text/ - assets = [ + objects = [ {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, {'id': 3, 'subject': 'Baking a cake', 'author': 'abc', 'views': 90}, @@ -544,17 +548,17 @@ def test_text_search(): ] # insert the assets - conn.db.assets.insert_many(deepcopy(assets), ordered=False) + conn.db[table].insert_many(deepcopy(objects), ordered=False) # test search single word - assert list(query.text_search(conn, 'coffee')) == [ + assert list(query.text_search(conn, 'coffee', table=table)) == [ {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, ] # match any of the search terms - assert list(query.text_search(conn, 'bake coffee cake')) == [ + assert list(query.text_search(conn, 'bake coffee cake', table=table)) == [ {'author': 'abc', 'id': 3, 'subject': 'Baking a cake', 'views': 90}, {'author': 'xyz', 'id': 1, 'subject': 'coffee', 'views': 50}, {'author': 'xyz', 'id': 4, 'subject': 'baking', 'views': 100}, @@ -563,48 +567,48 @@ def test_text_search(): ] # search for a phrase - assert list(query.text_search(conn, '\"coffee shop\"')) == [ + assert list(query.text_search(conn, '\"coffee shop\"', table=table)) == [ {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, ] # exclude documents that contain a term - assert list(query.text_search(conn, 'coffee -shop')) == [ + assert list(query.text_search(conn, 'coffee -shop', table=table)) == [ {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10}, ] # search different language - assert list(query.text_search(conn, 'leche', language='es')) == [ + assert list(query.text_search(conn, 'leche', language='es', table=table)) == [ {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} ] # case and diacritic insensitive search - assert list(query.text_search(conn, 'сы́рники CAFÉS')) == [ + assert list(query.text_search(conn, 'сы́рники CAFÉS', table=table)) == [ {'id': 6, 'subject': 'Сырники', 'author': 'jkl', 'views': 80}, {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, {'id': 8, 'subject': 'Cafe con Leche', 'author': 'xyz', 'views': 10} ] # case sensitive search - assert list(query.text_search(conn, 'Coffee', case_sensitive=True)) == [ + assert list(query.text_search(conn, 'Coffee', case_sensitive=True, table=table)) == [ {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, ] # diacritic sensitive search - assert list(query.text_search(conn, 'CAFÉ', diacritic_sensitive=True)) == [ + assert list(query.text_search(conn, 'CAFÉ', diacritic_sensitive=True, table=table)) == [ {'id': 5, 'subject': 'Café Con Leche', 'author': 'abc', 'views': 200}, ] # return text score - assert list(query.text_search(conn, 'coffee', text_score=True)) == [ + assert list(query.text_search(conn, 'coffee', text_score=True, table=table)) == [ {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50, 'score': 1.0}, {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5, 'score': 0.75}, {'id': 7, 'subject': 'coffee and cream', 'author': 'efg', 'views': 10, 'score': 0.75}, ] # limit search result - assert list(query.text_search(conn, 'coffee', limit=2)) == [ + assert list(query.text_search(conn, 'coffee', limit=2, table=table)) == [ {'id': 1, 'subject': 'coffee', 'author': 'xyz', 'views': 50}, {'id': 2, 'subject': 'Coffee Shopping', 'author': 'efg', 'views': 5}, ] diff --git a/tests/backend/mongodb/test_schema.py b/tests/backend/mongodb/test_schema.py index e11dbfe8..f1a2f428 100644 --- a/tests/backend/mongodb/test_schema.py +++ b/tests/backend/mongodb/test_schema.py @@ -19,11 +19,11 @@ def test_init_creates_db_tables_and_indexes(): collection_names = conn.conn[dbname].collection_names() assert sorted(collection_names) == ['assets', 'backlog', 'bigchain', - 'votes'] + 'metadata', 'votes'] indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', - 'outputs', 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_id', 'block_timestamp', + 'inputs', 'outputs', 'transaction_id'] indexes = conn.conn[dbname]['backlog'].index_information().keys() assert sorted(indexes) == ['_id_', 'assignee__transaction_timestamp', @@ -35,6 +35,9 @@ def test_init_creates_db_tables_and_indexes(): indexes = conn.conn[dbname]['assets'].index_information().keys() assert sorted(indexes) == ['_id_', 'asset_id', 'text'] + indexes = conn.conn[dbname]['metadata'].index_information().keys() + assert sorted(indexes) == ['_id_', 'text', 'transaction_id'] + def test_init_database_fails_if_db_exists(): import bigchaindb @@ -67,7 +70,7 @@ def test_create_tables(): collection_names = conn.conn[dbname].collection_names() assert sorted(collection_names) == ['assets', 'backlog', 'bigchain', - 'votes'] + 'metadata', 'votes'] def test_create_secondary_indexes(): @@ -86,8 +89,8 @@ def test_create_secondary_indexes(): # Bigchain table indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', - 'outputs', 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_id', 'block_timestamp', + 'inputs', 'outputs', 'transaction_id'] # Backlog table indexes = conn.conn[dbname]['backlog'].index_information().keys() diff --git a/tests/backend/rethinkdb/test_schema.py b/tests/backend/rethinkdb/test_schema.py index 6f77b672..e56dfd21 100644 --- a/tests/backend/rethinkdb/test_schema.py +++ b/tests/backend/rethinkdb/test_schema.py @@ -64,7 +64,8 @@ def test_create_tables(): assert conn.run(r.db(dbname).table_list().contains('backlog')) is True assert conn.run(r.db(dbname).table_list().contains('votes')) is True assert conn.run(r.db(dbname).table_list().contains('assets')) is True - assert len(conn.run(r.db(dbname).table_list())) == 4 + assert conn.run(r.db(dbname).table_list().contains('metadata')) is True + assert len(conn.run(r.db(dbname).table_list())) == 5 @pytest.mark.bdb diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 18d4df3e..01f28675 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -40,6 +40,8 @@ def test_schema(schema_func_name, args_qty): ('get_spending_transactions', 1), ('write_assets', 1), ('get_assets', 1), + ('write_metadata', 1), + ('get_metadata', 1), )) def test_query(query_func_name, args_qty): from bigchaindb.backend import query diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index 4a60c0cc..46f8a8f6 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -49,6 +49,7 @@ def run_start_args(request): config=param.get('config'), start_rethinkdb=param.get('start_rethinkdb', False), allow_temp_keypair=param.get('allow_temp_keypair', False), + skip_initialize_database=param.get('skip_initialize_database', False), ) diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index e40b3ff2..c8990582 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -13,7 +13,8 @@ def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, mocked_setup_logging): from bigchaindb import config from bigchaindb.commands.bigchaindb import run_start - args = Namespace(start_rethinkdb=True, allow_temp_keypair=False, config=None, yes=True) + args = Namespace(start_rethinkdb=True, allow_temp_keypair=False, config=None, yes=True, + skip_initialize_database=False) run_start(args) mock_start_rethinkdb.assert_called_with() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index b50a2a67..423e4614 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -39,7 +39,8 @@ def test_bigchain_run_start(mock_run_configure, mocked_setup_logging): from bigchaindb import config from bigchaindb.commands.bigchaindb import run_start - args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True) + args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True, + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with(user_log_config=config['log']) @@ -288,7 +289,8 @@ def test_allow_temp_keypair_generates_one_on_the_fly( bigchaindb.config['keypair'] = {'private': None, 'public': None} - args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True) + args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True, + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with( @@ -314,7 +316,8 @@ def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, assert isinstance(original_public_key, str) assert isinstance(original_private_key, str) - args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True) + args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True, + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with( diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py index e80ad0e2..8c064696 100644 --- a/tests/common/test_schema.py +++ b/tests/common/test_schema.py @@ -11,7 +11,7 @@ from pytest import raises from bigchaindb.common.exceptions import SchemaValidationError from bigchaindb.common.schema import ( - TX_SCHEMA_COMMON, VOTE_SCHEMA, drop_schema_descriptions, + TX_SCHEMA_COMMON, VOTE_SCHEMA, validate_transaction_schema, validate_vote_schema) SUPPORTED_CRYPTOCONDITION_TYPES = ('threshold-sha-256', 'ed25519-sha-256') @@ -46,49 +46,6 @@ def test_vote_schema_additionalproperties(): _test_additionalproperties(VOTE_SCHEMA) -def test_drop_descriptions(): - node = { - 'description': 'abc', - 'properties': { - 'description': { - 'description': ('The property named "description" should stay' - 'but description meta field goes'), - }, - 'properties': { - 'description': 'this must go' - }, - 'any': { - 'anyOf': [ - { - 'description': 'must go' - } - ] - } - }, - 'definitions': { - 'wat': { - 'description': 'go' - } - } - } - expected = { - 'properties': { - 'description': {}, - 'properties': {}, - 'any': { - 'anyOf': [ - {} - ] - } - }, - 'definitions': { - 'wat': {}, - } - } - drop_schema_descriptions(node) - assert node == expected - - ################################################################################ # Test call transaction schema diff --git a/tests/conftest.py b/tests/conftest.py index bf28b9b4..7d116b83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,15 @@ USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie' USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' +def pytest_runtest_setup(item): + if isinstance(item, item.Function): + if item.get_marker('skip_travis_rdb'): + if (os.getenv('TRAVIS_CI') == 'true' and + os.getenv('BIGCHAINDB_DATABASE_BACKEND') == 'rethinkdb'): + pytest.skip( + 'Skip test during Travis CI build when using rethinkdb') + + def pytest_addoption(parser): from bigchaindb.backend.connection import BACKENDS diff --git a/tests/integration/test_federation.py b/tests/integration/test_federation.py index 598412ff..05d8f32e 100644 --- a/tests/integration/test_federation.py +++ b/tests/integration/test_federation.py @@ -97,6 +97,7 @@ def process_vote(steps, result=None): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_valid(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) @@ -116,6 +117,7 @@ def test_elect_valid(federation_3): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_invalid(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) @@ -135,6 +137,7 @@ def test_elect_invalid(federation_3): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_sybill(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 2bf0ebcd..0c67b783 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -5,6 +5,7 @@ import pytest pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('processes')] +@pytest.mark.serial def test_double_create(b, user_pk): from bigchaindb.models import Transaction from bigchaindb.backend.query import count_blocks @@ -12,9 +13,9 @@ def test_double_create(b, user_pk): metadata={'test': 'test'}).sign([b.me_private]) b.write_transaction(tx) - time.sleep(2) + time.sleep(5) b.write_transaction(tx) - time.sleep(2) + time.sleep(5) tx_returned = b.get_transaction(tx.id) # test that the tx can be queried @@ -25,6 +26,7 @@ def test_double_create(b, user_pk): assert count_blocks(b.connection) == 2 +@pytest.mark.dspend @pytest.mark.usefixtures('inputs') def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): """ Test for #633 https://github.com/bigchaindb/bigchaindb/issues/633 """ @@ -39,7 +41,7 @@ def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): # write the valid tx and wait for voting/block to catch up b.write_transaction(tx_valid) - time.sleep(2) + time.sleep(5) # doesn't throw an exception b.get_owned_ids(user_pk) diff --git a/tests/log/test_loggers.py b/tests/log/test_loggers.py index 795de046..83a32252 100644 --- a/tests/log/test_loggers.py +++ b/tests/log/test_loggers.py @@ -7,9 +7,9 @@ class TestHttpServerLogger: from bigchaindb.log.configs import ( DEFAULT_SOCKET_LOGGING_ADDR as expected_socket_address) from bigchaindb.log.loggers import HttpServerLogger - mocked_config = mocker.patch( - 'gunicorn.config.Config', autospec=True, spec_set=True) - logger = HttpServerLogger(mocked_config.return_value) + from gunicorn import config + logger_config = config.Config() + logger = HttpServerLogger(logger_config) assert len(logger.access_log.handlers) == 1 assert len(logger.error_log.handlers) == 1 assert isinstance(logger.access_log.handlers[0], SocketHandler) diff --git a/tests/log/test_setup.py b/tests/log/test_setup.py index 0e608d26..a02bd4c2 100644 --- a/tests/log/test_setup.py +++ b/tests/log/test_setup.py @@ -70,7 +70,7 @@ def log_record_bytes(log_record_dict): def test_setup_logging(mocked_setup_pub_logger, mocked_setup_sub_logger): from bigchaindb.log.setup import setup_logging setup_logging() - mocked_setup_pub_logger.assert_called_once_with() + mocked_setup_pub_logger.assert_called_once_with(logging_port=None) mocked_setup_sub_logger.assert_called_once_with(user_log_config=None) @@ -112,11 +112,12 @@ def test_setup_sub_logger_with_config(mocked_socket_server, mocked_process): 'granular_levels': { 'bigchaindb.core': 'debug', }, + 'port': 9020 } root_logger = getLogger() setup_sub_logger(user_log_config=user_log_config) assert root_logger.level == logging.NOTSET - mocked_socket_server.assert_called_once_with() + mocked_socket_server.assert_called_once_with(port=9020) mocked_process.assert_called_once_with( target=mocked_socket_server.return_value.serve_forever, kwargs={'log_config': user_log_config}, diff --git a/tests/pipelines/test_vote.py b/tests/pipelines/test_vote.py index 21bbc0bf..701434fd 100644 --- a/tests/pipelines/test_vote.py +++ b/tests/pipelines/test_vote.py @@ -29,6 +29,16 @@ def decouple_assets(b, block): return block_dict +def decouple_metadata(b, block, block_dict): + # the block comming from the database does not contain the metadata + # so we need to pass the block without the metadata and store the metadata + # so that the voting pipeline can reconstruct it + metadata, block_dict = block.decouple_metadata(block_dict) + if metadata: + b.write_metadata(metadata) + return block_dict + + DUMMY_SHA3 = '0123456789abcdef' * 4 @@ -89,6 +99,7 @@ def test_vote_validate_block(b): tx = dummy_tx(b) block = b.create_block([tx]) block_dict = decouple_assets(b, block) + block_dict = decouple_metadata(b, block, block_dict) vote_obj = vote.Vote() validation = vote_obj.validate_block(block_dict) @@ -230,6 +241,7 @@ def test_valid_block_voting_multiprocessing(b, genesis_block, monkeypatch): block = dummy_block(b) block_dict = decouple_assets(b, block) + block_dict = decouple_metadata(b, block, block_dict) inpipe.put(block_dict) vote_pipeline.start() @@ -268,6 +280,7 @@ def test_valid_block_voting_with_create_transaction(b, monkeypatch.setattr('time.time', lambda: 1111111111) block = b.create_block([tx]) block_dict = decouple_assets(b, block) + block_dict = decouple_metadata(b, block, block_dict) inpipe = Pipe() outpipe = Pipe() diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index cba1f77e..1f16cb6b 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -302,6 +302,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request, certs_d 'fmt_console': log_config['formatters']['console']['format'], 'fmt_logfile': log_config['formatters']['file']['format'], 'granular_levels': {}, + 'port': 9020 }, 'graphite': {'host': 'localhost'}, 'consensus_plugin': None diff --git a/tests/test_utils.py b/tests/test_utils.py index fbf5d65d..9620d0f9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,9 +142,9 @@ def test_is_genesis_block_returns_true_if_genesis(b): def test_lazy_execution(): from bigchaindb.utils import Lazy - l = Lazy() - l.split(',')[1].split(' ').pop(1).strip() - result = l.run('Like humans, cats tend to favor one paw over another') + lz = Lazy() + lz.split(',')[1].split(' ').pop(1).strip() + result = lz.run('Like humans, cats tend to favor one paw over another') assert result == 'cats' class Cat: @@ -153,7 +153,7 @@ def test_lazy_execution(): cat = Cat('Shmui') - l = Lazy() - l.name.upper() - result = l.run(cat) + lz = Lazy() + lz.name.upper() + result = lz.run(cat) assert result == 'SHMUI' diff --git a/tests/utils.py b/tests/utils.py index 9af72a5d..5b3b5242 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,6 +33,7 @@ def flush_rethink_db(connection, dbname): connection.run(r.db(dbname).table('backlog').delete()) connection.run(r.db(dbname).table('votes').delete()) connection.run(r.db(dbname).table('assets').delete()) + connection.run(r.db(dbname).table('metadata').delete()) except r.ReqlOpFailedError: pass @@ -43,6 +44,7 @@ def flush_mongo_db(connection, dbname): connection.conn[dbname].backlog.delete_many({}) connection.conn[dbname].votes.delete_many({}) connection.conn[dbname].assets.delete_many({}) + connection.conn[dbname].metadata.delete_many({}) @singledispatch diff --git a/tests/web/test_metadata.py b/tests/web/test_metadata.py new file mode 100644 index 00000000..22c025ae --- /dev/null +++ b/tests/web/test_metadata.py @@ -0,0 +1,87 @@ +import pytest + +METADATA_ENDPOINT = '/api/v1/metadata/' + + +def test_get_metadata_with_empty_text_search(client): + res = client.get(METADATA_ENDPOINT + '?search=') + assert res.json == {'status': 400, + 'message': 'text_search cannot be empty'} + assert res.status_code == 400 + + +def test_get_metadata_with_missing_text_search(client): + res = client.get(METADATA_ENDPOINT) + assert res.status_code == 400 + + +@pytest.mark.genesis +def test_get_metadata(client, b): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + if isinstance(b.connection, MongoDBConnection): + # test returns empty list when no assets are found + res = client.get(METADATA_ENDPOINT + '?search=abc') + assert res.json == [] + assert res.status_code == 200 + + # create asset + asset = {'msg': 'abc'} + metadata = {'key': 'my_meta'} + tx = Transaction.create([b.me], [([b.me], 1)], metadata=metadata, + asset=asset).sign([b.me_private]) + # create block + block = b.create_block([tx]) + b.write_block(block) + # vote valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + + # test that metadata is returned + res = client.get(METADATA_ENDPOINT + '?search=my_meta') + assert res.status_code == 200 + assert len(res.json) == 1 + assert res.json[0] == { + 'metadata': {'key': 'my_meta'}, + 'id': tx.id + } + else: + # test that the correct error is returned if not running MongoDB + res = client.get(METADATA_ENDPOINT + '?search=my_meta') + assert res.status_code == 400 + assert res.json['message'].startswith('(OperationError)') + + +@pytest.mark.genesis +def test_get_metadata_limit(client, b): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + if isinstance(b.connection, MongoDBConnection): + # create two assets + asset1 = {'msg': 'abc 1'} + meta1 = {'key': 'meta 1'} + tx1 = Transaction.create([b.me], [([b.me], 1)], metadata=meta1, + asset=asset1).sign([b.me_private]) + + asset2 = {'msg': 'abc 2'} + meta2 = {'key': 'meta 2'} + tx2 = Transaction.create([b.me], [([b.me], 1)], metadata=meta2, + asset=asset2).sign([b.me_private]) + # create block + block = b.create_block([tx1, tx2]) + b.write_block(block) + # vote valid + vote = b.vote(block.id, b.get_last_voted_block().id, True) + b.write_vote(vote) + + # test that both assets are returned without limit + res = client.get(METADATA_ENDPOINT + '?search=meta') + assert res.status_code == 200 + assert len(res.json) == 2 + + # test that only one asset is returned when using limit=1 + res = client.get(METADATA_ENDPOINT + '?search=meta&limit=1') + assert res.status_code == 200 + assert len(res.json) == 1 diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index acea8c2c..ab01357a 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,6 +47,82 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub +@pytest.mark.parametrize("nested", [False, True]) +@pytest.mark.parametrize("language,expected_status_code", [ + ('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202), + ('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202), + ('norwegian', 202), ('portuguese', 202), ('romanian', 202), ('none', 202), + ('russian', 202), ('spanish', 202), ('swedish', 202), ('turkish', 202), + ('da', 202), ('nl', 202), ('en', 202), ('fi', 202), ('fr', 202), + ('de', 202), ('hu', 202), ('it', 202), ('nb', 202), ('pt', 202), + ('ro', 202), ('ru', 202), ('es', 202), ('sv', 202), ('tr', 202), + ('any', 400) +]) +@pytest.mark.language +@pytest.mark.bdb +def test_post_create_transaction_with_language(b, client, nested, language, + expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + if isinstance(b.connection, MongoDBConnection): + user_priv, user_pub = crypto.generate_key_pair() + lang_obj = {'language': language} + + if nested: + asset = {'root': lang_obj} + else: + asset = lang_obj + + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset=asset) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + 'Invalid transaction (ValidationError): MongoDB does not support ' + 'text search for the language "{}". If you do not understand this ' + 'error message then please rename key/field "language" to something ' + 'else like "lang".').format(language) + assert res.json['message'] == expected_error_message + + +@pytest.mark.parametrize("field", ['asset', 'metadata']) +@pytest.mark.parametrize("value,err_key,expected_status_code", [ + ({'bad.key': 'v'}, 'bad.key', 400), + ({'$bad.key': 'v'}, '$bad.key', 400), + ({'$badkey': 'v'}, '$badkey', 400), + ({'bad\x00key': 'v'}, 'bad\x00key', 400), + ({'good_key': {'bad.key': 'v'}}, 'bad.key', 400), + ({'good_key': 'v'}, 'good_key', 202) +]) +@pytest.mark.bdb +def test_post_create_transaction_with_invalid_key(b, client, field, value, + err_key, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + user_priv, user_pub = crypto.generate_key_pair() + + if isinstance(b.connection, MongoDBConnection): + if field == 'asset': + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset=value) + elif field == 'metadata': + tx = Transaction.create([user_pub], [([user_pub], 1)], + metadata=value) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + 'Invalid transaction (ValidationError): Invalid key name "{}" ' + 'in {} object. The key name cannot contain characters ' + '".", "$" or null characters').format(err_key, field) + assert res.json['message'] == expected_error_message + + @patch('bigchaindb.web.views.base.logger') def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash