From fba6e1b30bc98065b4921264ff4164e9de28f437 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 18 Oct 2017 15:38:46 +0200 Subject: [PATCH 01/24] New root docs page about permissions in BigchainDB --- docs/root/source/conf.py | 4 +- docs/root/source/index.rst | 1 + docs/root/source/permissions.rst | 74 ++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 docs/root/source/permissions.rst diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py index 0d799fed..990fa3ce 100644 --- a/docs/root/source/conf.py +++ b/docs/root/source/conf.py @@ -34,7 +34,9 @@ from recommonmark.parser import CommonMarkParser # ones. import sphinx_rtd_theme -extensions = [] +extensions = [ + 'sphinx.ext.autosectionlabel', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst index 455cae0c..71fdd022 100644 --- a/docs/root/source/index.rst +++ b/docs/root/source/index.rst @@ -88,5 +88,6 @@ More About BigchainDB assets smart-contracts transaction-concepts + permissions timestamps Data Models diff --git a/docs/root/source/permissions.rst b/docs/root/source/permissions.rst new file mode 100644 index 00000000..d5972ac0 --- /dev/null +++ b/docs/root/source/permissions.rst @@ -0,0 +1,74 @@ +Permissions in BigchainDB +------------------------- + +BigchainDB lets users control what other users can do, to some extent. That ability resembles "permissions" in the \*nix world, "privileges" in the SQL world, and "access control" in the security world. + + +Permission to Spend/Transfer an Output +====================================== + +In BigchainDB, every output has an associated condition (crypto-condition). + +To spend/transfer an unspent output, a user (or group of users) must fulfill the condition. Another way to say that is that only certain users have permission to spend the output. The simplest condition is of the form, "Only someone with the private key corresponding to this public key can spend this output." Much more elaborate conditions are possible, e.g. "To spend this output, …" + +- "…anyone in the Accounting Group can sign." +- "…three of these four people must sign." +- "…either Bob must sign, or both Tom and Sylvia must sign." + +For details, see `the documentation about conditions in BigchainDB `_. + +Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output. + + +Write Permissions +================= + +When someone builds a TRANSFER transaction, they can put an arbitrary JSON object in the ``metadata`` field (within reason; real BigchainDB networks put a limit on the size of transactions). That is, they can write just about anything they want in a TRANSFER transaction. + +Does that mean there are no "write permissions" in BigchainDB? Not at all! + +A TRANSFER transaction will only be valid (allowed) if its inputs fulfill some previous outputs. The conditions on those outputs will control who can build valid TRANSFER transactions. In other words, one can interpret the condition on an output as giving "write permissions" to certain users to write something into the history of the associated asset. + +As a concrete example, you could use BigchainDB to write a public journal where only you have write permissions. Here's how: First you'd build a CREATE transaction with the ``asset.data`` being something like ``{"title": "The Journal of John Doe"}``, with one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. +Each time you want to append something to your journal, you'd build a new TRANSFER transaction with your latest entry in the ``metadata`` field, e.g. + +.. code-block:: json + + {"timestamp": "1508319582", + "entry": "I visited Marmot Lake with Jane."} + +The TRANSFER transaction would have one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. And so on. Only you would be able to append to the history of that asset (your journal). + +The same technique could be used for scientific notebooks, supply-chain records, government meeting minutes, and so on. + +You could do more elaborate things too. As one example, each time someone writes a TRANSFER transaction, they give *someone else* permission to spend it, setting up a sort of writers-relay or chain letter. + +.. note:: + + Anyone can write any JSON (again, within reason) in the ``asset.data`` field of a CREATE transaction. They don't need permission. + + +Read Permissions +================ + +All the data stored in a BigchainDB network can be read by anyone with access to that network. One *can* store encrypted data, but if the decryption key ever leaks out, then the encrypted data can be read, decrypted, and leak out too. (Deleting the encrypted data is :doc:`not an option `.) + +The permission to read some specific information (e.g. a music file) can be thought of as an *asset*. (In many countries, that permission or "right" is a kind of intellectual property.) +BigchainDB can be used to register that asset and transfer it from owner to owner. +Today, BigchainDB does not have a way to restrict read access of data stored in a BigchainDB network, but many third-party services do offer that (e.g. Google Docs, Dropbox). +In principle, a third party service could ask a BigchainDB network to determine if a particular user has permission to read some particular data. Indeed they could use BigchainDB to keep track of *all* the rights a user has for some data (not just the right to read it). +That third party could also use BigchainDB to store audit logs, i.e. records of every read, write or other operation on stored data. + +BigchainDB can be used in other ways to help parties exchange private data: + +- It can be used to publicly disclose the *availability* of some private data (stored elsewhere). For example, there might be a description of the data and a price. +- It can be used to record the TLS handshakes which two parties sent to each other to establish an encrypted and authenticated TLS connection, which they could use to exchange private data with each other. (The stored handshake information wouldn't be enough, by itself, to decrypt the data.) It would be a "proof of TLS handshake." +- See the BigchainDB `Privacy Protocols repository `_ for more techniques. + + +Role-Based Access Control (RBAC) +================================ + +In September 2017, we published a `blog post about how one can define an RBAC sub-system on top of BigchainDB `_. +At the time of writing (October 2017), doing so required the use of a plugin, so it's not possible using standard BigchainDB (which is what's available on `IPDB `_). That may change in the future. +If you're interested, `contact BigchainDB `_. From ddfce61b79895c63c84ace2e8861e1a59c63ed67 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 27 Oct 2017 15:05:43 +0530 Subject: [PATCH 02/24] Added secondary index for "id" in bigchain collection. --- bigchaindb/backend/mongodb/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index e398560f..00616dcb 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -50,6 +50,11 @@ def drop_database(conn, dbname): def create_bigchain_secondary_index(conn, dbname): logger.info('Create `bigchain` secondary index.') + # secondary index on block id which is should be unique + conn.conn[dbname]['bigchain'].create_index('id', + name='block_id', + unique=True) + # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', ASCENDING)], From 421c67c62149b28b8de137c3ed46659c743df4dd Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 27 Oct 2017 15:31:44 +0530 Subject: [PATCH 03/24] Fixed mongodb tests --- bigchaindb/backend/mongodb/schema.py | 3 +-- tests/backend/mongodb/test_schema.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 00616dcb..01eac29c 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -52,8 +52,7 @@ def create_bigchain_secondary_index(conn, dbname): # secondary index on block id which is should be unique conn.conn[dbname]['bigchain'].create_index('id', - name='block_id', - unique=True) + name='block_id') # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', diff --git a/tests/backend/mongodb/test_schema.py b/tests/backend/mongodb/test_schema.py index e11dbfe8..1a244b1b 100644 --- a/tests/backend/mongodb/test_schema.py +++ b/tests/backend/mongodb/test_schema.py @@ -22,8 +22,8 @@ def test_init_creates_db_tables_and_indexes(): 'votes'] indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', - 'outputs', 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_id', 'block_timestamp', + 'inputs', 'outputs', 'transaction_id'] indexes = conn.conn[dbname]['backlog'].index_information().keys() assert sorted(indexes) == ['_id_', 'assignee__transaction_timestamp', @@ -86,8 +86,8 @@ def test_create_secondary_indexes(): # Bigchain table indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', - 'outputs', 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_id', 'block_timestamp', + 'inputs', 'outputs', 'transaction_id'] # Backlog table indexes = conn.conn[dbname]['backlog'].index_information().keys() From 722658421580c36ac32d6a7bd9e3537ed96a9bb6 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 10:12:16 +0530 Subject: [PATCH 04/24] Api fix for asset language --- bigchaindb/backend/schema.py | 22 ++++++++++++++++++ bigchaindb/models.py | 2 ++ tests/web/test_transactions.py | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index f6ce466f..8b85c068 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -16,10 +16,14 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect +from bigchaindb.common.exceptions import ValidationError logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') +VALID_LANGUAGES = ('danish' 'dutch' 'english' 'finnish' 'french' 'german' + 'hungarian' 'italian' 'norwegian' 'portuguese' 'romanian' + 'russian' 'spanish' 'swedish' 'turkish') @singledispatch @@ -99,3 +103,21 @@ def init_database(connection=None, dbname=None): create_database(connection, dbname) create_tables(connection, dbname) create_indexes(connection, dbname) + + +def validate_if_exists_asset_language(tx_body): + data = tx_body['asset'].get('data', {}) + + if data and 'language' in data: + + language = data.get('language') + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb' and language not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(language) + raise ValidationError(error_str) from ValueError() + + return diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c8ad9dd3..d88d239e 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -10,6 +10,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import gen_timestamp, serialize from bigchaindb.common.schema import validate_transaction_schema +from bigchaindb.backend.schema import validate_if_exists_asset_language class Transaction(Transaction): @@ -84,6 +85,7 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) + validate_if_exists_asset_language(tx_body) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index acea8c2c..8b4d31bf 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,6 +47,47 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub +@pytest.mark.parametrize("language,expected_status_code", [ + ('danish', 202), + ('dutch', 202), + ('english', 202), + ('finnish', 202), + ('french', 202), + ('german', 202), + ('hungarian', 202), + ('italian', 202), + ('norwegian', 202), + ('portuguese', 202), + ('romanian', 202), + ('russian', 202), + ('spanish', 202), + ('swedish', 202), + ('turkish', 202), + ('any', 400), +]) +@pytest.mark.language +@pytest.mark.bdb +def test_post_create_transaction_with_language(b, client, language, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + if isinstance(b.connection, MongoDBConnection): + user_priv, user_pub = crypto.generate_key_pair() + + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset={'language': language}) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + "Invalid transaction (ValidationError): MongoDB does not support " + "text search for the language \"{}\". If you do not understand this " + "error message then please rename key/field \"language\" to something " + "else like \"lang\".").format(language) + assert res.json['message'] == expected_error_message + + @patch('bigchaindb.web.views.base.logger') def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash From 1de5375962306b1ce04c553eb38ad5237839959f Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 15:16:59 +0530 Subject: [PATCH 05/24] Validate asset data keys --- bigchaindb/common/utils.py | 19 ++++++++++++++++++- bigchaindb/models.py | 4 +++- tests/web/test_transactions.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index f6f671db..fd9386e1 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -1,7 +1,10 @@ import time - +import re import rapidjson +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + def gen_timestamp(): """The Unix time, rounded to the nearest second. @@ -46,3 +49,17 @@ def deserialize(data): string. """ return rapidjson.loads(data) + + +def validate_asset_data_keys(tx_body): + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = tx_body['asset'].get('data', {}) + keys = data.keys() if data else [] + for key in keys: + if re.search(r'^[$]|\.', key): + error_str = ('Invalid key name "{}" in asset object. The ' + 'key name cannot contain characters ' + '"." and "$"').format(key) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c8ad9dd3..9704372d 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -8,7 +8,8 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, SybilError, DuplicateTransaction) from bigchaindb.common.transaction import Transaction -from bigchaindb.common.utils import gen_timestamp, serialize +from bigchaindb.common.utils import (gen_timestamp, serialize, + validate_asset_data_keys) from bigchaindb.common.schema import validate_transaction_schema @@ -84,6 +85,7 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) + validate_asset_data_keys(tx_body) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index acea8c2c..4c2a6ed9 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,6 +47,34 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub +@pytest.mark.parametrize("key,expected_status_code", [ + ('bad.key', 400), + ('$bad.key', 400), + ('$badkey', 400), + ('good_key', 202) +]) +@pytest.mark.assetkey +@pytest.mark.bdb +def test_post_create_transaction_with_invalid_asset_key(b, client, key, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + user_priv, user_pub = crypto.generate_key_pair() + + if isinstance(b.connection, MongoDBConnection): + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset={key: 'random_value'}) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + 'Invalid transaction (ValidationError): Invalid key name "{}" ' + 'in asset object. The key name cannot contain characters ' + '"." and "$"').format(key) + assert res.json['message'] == expected_error_message + + @patch('bigchaindb.web.views.base.logger') def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash From cd636101a7c03e433c27a26d1749e87a37dbec55 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 16:54:47 +0530 Subject: [PATCH 06/24] Support abbreviated values for "language" --- bigchaindb/backend/schema.py | 12 +++++++----- tests/web/test_transactions.py | 24 ++++++++---------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 8b85c068..3b7413a2 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -21,9 +21,11 @@ from bigchaindb.common.exceptions import ValidationError logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') -VALID_LANGUAGES = ('danish' 'dutch' 'english' 'finnish' 'french' 'german' - 'hungarian' 'italian' 'norwegian' 'portuguese' 'romanian' - 'russian' 'spanish' 'swedish' 'turkish') +VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', + 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', + 'russian', 'spanish', 'swedish', 'turkish', + 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', + 'ro', 'ru', 'es', 'sv', 'tr') @singledispatch @@ -108,12 +110,12 @@ def init_database(connection=None, dbname=None): def validate_if_exists_asset_language(tx_body): data = tx_body['asset'].get('data', {}) - if data and 'language' in data: + if data and ('language' in data): language = data.get('language') backend = bigchaindb.config['database']['backend'] - if backend == 'mongodb' and language not in VALID_LANGUAGES: + if (backend == 'mongodb') and (language not in VALID_LANGUAGES): error_str = ('MongoDB does not support text search for the ' 'language "{}". If you do not understand this error ' 'message then please rename key/field "language" to ' diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 8b4d31bf..f33c47b1 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -48,22 +48,14 @@ def test_post_create_transaction_endpoint(b, client): @pytest.mark.parametrize("language,expected_status_code", [ - ('danish', 202), - ('dutch', 202), - ('english', 202), - ('finnish', 202), - ('french', 202), - ('german', 202), - ('hungarian', 202), - ('italian', 202), - ('norwegian', 202), - ('portuguese', 202), - ('romanian', 202), - ('russian', 202), - ('spanish', 202), - ('swedish', 202), - ('turkish', 202), - ('any', 400), + ('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202), + ('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202), + ('norwegian', 202), ('portuguese', 202), ('romanian', 202), + ('russian', 202), ('spanish', 202), ('swedish', 202), ('turkish', 202), + ('da', 202), ('nl', 202), ('en', 202), ('fi', 202), ('fr', 202), + ('de', 202), ('hu', 202), ('it', 202), ('nb', 202), ('pt', 202), + ('ro', 202), ('ru', 202), ('es', 202), ('sv', 202), ('tr', 202), + ('any', 400) ]) @pytest.mark.language @pytest.mark.bdb From f3da30aea082347a8ae72d1b8ddd97e0def5d37d Mon Sep 17 00:00:00 2001 From: kansi Date: Wed, 1 Nov 2017 17:26:16 +0530 Subject: [PATCH 07/24] Validate nested keys for asset.data and metadata --- bigchaindb/common/utils.py | 28 +++++++++++++++++++--------- bigchaindb/models.py | 5 +++-- tests/web/test_transactions.py | 29 ++++++++++++++++++----------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index fd9386e1..35163f14 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -51,15 +51,25 @@ def deserialize(data): return rapidjson.loads(data) -def validate_asset_data_keys(tx_body): +def validate_txn_obj(obj_name, obj, key, validation_fun): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = tx_body['asset'].get('data', {}) - keys = data.keys() if data else [] - for key in keys: - if re.search(r'^[$]|\.', key): - error_str = ('Invalid key name "{}" in asset object. The ' - 'key name cannot contain characters ' - '"." and "$"').format(key) - raise ValidationError(error_str) from ValueError() + data = obj.get(key, {}) or {} + validate_all_keys(obj_name, data, validation_fun) + + +def validate_all_keys(obj_name, obj, validation_fun): + for key, value in obj.items(): + validation_fun(obj_name, key) + if type(value) is dict: + validate_all_keys(obj_name, value, validation_fun) + return + + +def validate_key(obj_name, key): + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 9704372d..1ecd964e 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -9,7 +9,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, DuplicateTransaction) from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import (gen_timestamp, serialize, - validate_asset_data_keys) + validate_txn_obj, validate_key) from bigchaindb.common.schema import validate_transaction_schema @@ -85,7 +85,8 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) - validate_asset_data_keys(tx_body) + validate_txn_obj('asset', tx_body['asset'], 'data', validate_key) + validate_txn_obj('metadata', tx_body, 'metadata', validate_key) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 4c2a6ed9..e5034697 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,22 +47,29 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub -@pytest.mark.parametrize("key,expected_status_code", [ - ('bad.key', 400), - ('$bad.key', 400), - ('$badkey', 400), - ('good_key', 202) +@pytest.mark.parametrize("field", ['asset', 'metadata']) +@pytest.mark.parametrize("value,err_key,expected_status_code", [ + ({'bad.key': 'v'}, 'bad.key', 400), + ({'$bad.key': 'v'}, '$bad.key', 400), + ({'$badkey': 'v'}, '$badkey', 400), + ({'bad\x00key': 'v'}, 'bad\x00key', 400), + ({'good_key': {'bad.key': 'v'}}, 'bad.key', 400), + ({'good_key': 'v'}, 'good_key', 202) ]) -@pytest.mark.assetkey @pytest.mark.bdb -def test_post_create_transaction_with_invalid_asset_key(b, client, key, expected_status_code): +def test_post_create_transaction_with_invalid_key(b, client, field, value, + err_key, expected_status_code): from bigchaindb.models import Transaction from bigchaindb.backend.mongodb.connection import MongoDBConnection user_priv, user_pub = crypto.generate_key_pair() if isinstance(b.connection, MongoDBConnection): - tx = Transaction.create([user_pub], [([user_pub], 1)], - asset={key: 'random_value'}) + if field == 'asset': + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset=value) + elif field == 'metadata': + tx = Transaction.create([user_pub], [([user_pub], 1)], + metadata=value) tx = tx.sign([user_priv]) res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) @@ -70,8 +77,8 @@ def test_post_create_transaction_with_invalid_asset_key(b, client, key, expected if res.status_code == 400: expected_error_message = ( 'Invalid transaction (ValidationError): Invalid key name "{}" ' - 'in asset object. The key name cannot contain characters ' - '"." and "$"').format(key) + 'in {} object. The key name cannot contain characters ' + '".", "$" or null characters').format(err_key, field) assert res.json['message'] == expected_error_message From 263e9a25f66cd883095d237bedd84c69337d614a Mon Sep 17 00:00:00 2001 From: kansi Date: Wed, 1 Nov 2017 21:23:06 +0530 Subject: [PATCH 08/24] Unique index for bigchain collection, fixed test case --- bigchaindb/backend/mongodb/schema.py | 5 +++-- tests/backend/mongodb/test_queries.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 01eac29c..572acff9 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -50,9 +50,10 @@ def drop_database(conn, dbname): def create_bigchain_secondary_index(conn, dbname): logger.info('Create `bigchain` secondary index.') - # secondary index on block id which is should be unique + # secondary index on block id which should be unique conn.conn[dbname]['bigchain'].create_index('id', - name='block_id') + name='block_id', + unique=True) # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index 0fd7229a..3ea7db28 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -299,12 +299,13 @@ def test_count_blocks(signed_create_tx): from bigchaindb.models import Block conn = connect() + assert query.count_blocks(conn) == 0 + # create and insert some blocks block = Block(transactions=[signed_create_tx]) conn.db.bigchain.insert_one(block.to_dict()) - conn.db.bigchain.insert_one(block.to_dict()) - assert query.count_blocks(conn) == 2 + assert query.count_blocks(conn) == 1 def test_count_backlog(signed_create_tx): From 8fdf8f6ca6cfdc3c20ac863c1b205dcfe365894e Mon Sep 17 00:00:00 2001 From: kansi Date: Thu, 2 Nov 2017 18:02:11 +0530 Subject: [PATCH 09/24] Added docstrings --- bigchaindb/common/utils.py | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index 35163f14..e472f380 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -52,6 +52,22 @@ def deserialize(data): def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validates value associated to `key` in `obj` by applying + `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': @@ -60,6 +76,20 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): def validate_all_keys(obj_name, obj, validation_fun): + """Validates all (nested) keys in `obj` by using `validation_fun` + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ for key, value in obj.items(): validation_fun(obj_name, key) if type(value) is dict: @@ -68,6 +98,19 @@ def validate_all_keys(obj_name, obj, validation_fun): def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: raise execption incase of regex match. + """ if re.search(r'^[$]|\.|\x00', key): error_str = ('Invalid key name "{}" in {} object. The ' 'key name cannot contain characters ' From 99aa38b21742ef412bf591f81e6f8c8763db68bb Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 11:38:38 +0530 Subject: [PATCH 10/24] Added "none" to language whitelist --- bigchaindb/backend/schema.py | 49 +++++++++++++----- bigchaindb/common/utils.py | 96 +++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 14 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 3b7413a2..07fe4210 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -17,13 +17,14 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect from bigchaindb.common.exceptions import ValidationError +from bigchaindb.common.utils import validate_all_value_for_key logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', - 'russian', 'spanish', 'swedish', 'turkish', + 'russian', 'spanish', 'swedish', 'turkish', 'none', 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', 'ro', 'ru', 'es', 'sv', 'tr') @@ -107,19 +108,41 @@ def init_database(connection=None, dbname=None): create_indexes(connection, dbname) -def validate_if_exists_asset_language(tx_body): - data = tx_body['asset'].get('data', {}) +def validate_if_exists_language(obj, key): + """Validate all nested "language" key in `obj`. - if data and ('language' in data): + Args: + obj (dict): dictonary whose "language" key is to be validated. - language = data.get('language') - backend = bigchaindb.config['database']['backend'] + Returns: + None: validation successfull - if (backend == 'mongodb') and (language not in VALID_LANGUAGES): - error_str = ('MongoDB does not support text search for the ' - 'language "{}". If you do not understand this error ' - 'message then please rename key/field "language" to ' - 'something else like "lang".').format(language) - raise ValidationError(error_str) from ValueError() + Raises: + ValidationError: raises execption incase language is not valid. + """ + backend = bigchaindb.config['database']['backend'] - return + if backend == 'mongodb': + data = obj.get(key, {}) or {} + validate_all_value_for_key(data, 'language', validate_language) + + +def validate_language(value): + """Check if `value` is a valid language + https://docs.mongodb.com/manual/reference/text-search-languages/ + + Args: + value (str): language to validated + + Returns: + None: validation successfull + + Raises: + ValidationError: raises execption incase language is not valid. + """ + if value not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(value) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index f6f671db..00621b2e 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -1,7 +1,10 @@ import time - +import re import rapidjson +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + def gen_timestamp(): """The Unix time, rounded to the nearest second. @@ -46,3 +49,94 @@ def deserialize(data): string. """ return rapidjson.loads(data) + + +def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validates value associated to `key` in `obj` by applying + `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = obj.get(key, {}) or {} + validate_all_keys(obj_name, data, validation_fun) + + +def validate_all_keys(obj_name, obj, validation_fun): + """Validates all (nested) keys in `obj` by using `validation_fun` + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for key, value in obj.items(): + validation_fun(obj_name, key) + if type(value) is dict: + validate_all_keys(obj_name, value, validation_fun) + return + + +def validate_all_value_for_key(obj, key, validation_fun): + """Validates value for all (nested) occurences of `key` in `obj` + using `validation_fun` + + Args: + obj (dict): dictonary object. + key (str): key whose value is to be validated. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for vkey, value in obj.items(): + if vkey == key: + validation_fun(value) + elif type(value) is dict: + validate_all_value_for_key(value, key, validation_fun) + return + + +def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: raise execption incase of regex match. + """ + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) from ValueError() From 3b33cdb111fb1f749b5e1e2971970fd89da5362d Mon Sep 17 00:00:00 2001 From: Trent McConaghy Date: Fri, 3 Nov 2017 13:48:08 +0100 Subject: [PATCH 11/24] Update bft.md --- docs/root/source/bft.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/root/source/bft.md b/docs/root/source/bft.md index 1f1a9bab..fce8ca3d 100644 --- a/docs/root/source/bft.md +++ b/docs/root/source/bft.md @@ -1,6 +1,8 @@ # BigchainDB and Byzantine Fault Tolerance -While BigchainDB is not currently [Byzantine fault tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance), we plan to offer it as an option. -We anticipate that turning it on will cause a severe dropoff in performance. See [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). +While BigchainDB is not currently [Byzantine fault tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance), we plan to offer it as an option. +Update Nov 2017: we're actively working on this, the next release or two will likely have support. More details to come in blog form and github issues + +Related issue: [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). We anticipate that turning on BFT will cause a dropoff in performance (for a gain in security). In the meantime, there are practical things that one can do to increase security (e.g. firewalls, key management, and access controls). From a29fd7e84f61f6df299faf4d955596edb090a8e7 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 18:37:19 +0530 Subject: [PATCH 12/24] Fix variable type check and docstrings --- bigchaindb/backend/schema.py | 8 ++++---- bigchaindb/common/utils.py | 11 +++-------- bigchaindb/models.py | 4 ++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 07fe4210..fcad33a6 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -108,14 +108,14 @@ def init_database(connection=None, dbname=None): create_indexes(connection, dbname) -def validate_if_exists_language(obj, key): +def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. Args: obj (dict): dictonary whose "language" key is to be validated. Returns: - None: validation successfull + None: validation successful Raises: ValidationError: raises execption incase language is not valid. @@ -135,7 +135,7 @@ def validate_language(value): value (str): language to validated Returns: - None: validation successfull + None: validation successful Raises: ValidationError: raises execption incase language is not valid. @@ -145,4 +145,4 @@ def validate_language(value): 'language "{}". If you do not understand this error ' 'message then please rename key/field "language" to ' 'something else like "lang".').format(value) - raise ValidationError(error_str) from ValueError() + raise ValidationError(error_str) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index 00621b2e..b21f41d6 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -92,9 +92,8 @@ def validate_all_keys(obj_name, obj, validation_fun): """ for key, value in obj.items(): validation_fun(obj_name, key) - if type(value) is dict: + if isinstance(value, dict): validate_all_keys(obj_name, value, validation_fun) - return def validate_all_value_for_key(obj, key, validation_fun): @@ -107,18 +106,14 @@ def validate_all_value_for_key(obj, key, validation_fun): validation_fun (function): function used to validate the value of `key`. - Returns: - None: indicates validation successfull - Raises: ValidationError: `validation_fun` will raise this error on failure """ for vkey, value in obj.items(): if vkey == key: validation_fun(value) - elif type(value) is dict: + elif isinstance(value, dict): validate_all_value_for_key(value, key, validation_fun) - return def validate_key(obj_name, key): @@ -139,4 +134,4 @@ def validate_key(obj_name, key): error_str = ('Invalid key name "{}" in {} object. The ' 'key name cannot contain characters ' '".", "$" or null characters').format(key, obj_name) - raise ValidationError(error_str) from ValueError() + raise ValidationError(error_str) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 152a8bcd..8e7a6bde 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -11,7 +11,7 @@ from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import (gen_timestamp, serialize, validate_txn_obj, validate_key) from bigchaindb.common.schema import validate_transaction_schema -from bigchaindb.backend.schema import validate_if_exists_language +from bigchaindb.backend.schema import validate_language_key class Transaction(Transaction): @@ -88,7 +88,7 @@ class Transaction(Transaction): validate_transaction_schema(tx_body) validate_txn_obj('asset', tx_body['asset'], 'data', validate_key) validate_txn_obj('metadata', tx_body, 'metadata', validate_key) - validate_if_exists_language(tx_body['asset'], 'data') + validate_language_key(tx_body['asset'], 'data') return super().from_dict(tx_body) @classmethod From e2c2c4b097e3e77fb35c8af44b1c5052229df0b7 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 19:15:32 +0530 Subject: [PATCH 13/24] Fix spell errors. --- bigchaindb/backend/schema.py | 12 ++++++------ bigchaindb/common/utils.py | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index fcad33a6..0e2815db 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -17,7 +17,7 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect from bigchaindb.common.exceptions import ValidationError -from bigchaindb.common.utils import validate_all_value_for_key +from bigchaindb.common.utils import validate_all_values_for_key logger = logging.getLogger(__name__) @@ -112,23 +112,23 @@ def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. Args: - obj (dict): dictonary whose "language" key is to be validated. + obj (dict): dictionary whose "language" key is to be validated. Returns: None: validation successful Raises: - ValidationError: raises execption incase language is not valid. + ValidationError: will raise exception in case language is not valid. """ backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': data = obj.get(key, {}) or {} - validate_all_value_for_key(data, 'language', validate_language) + validate_all_values_for_key(data, 'language', validate_language) def validate_language(value): - """Check if `value` is a valid language + """Check if `value` is a valid language. https://docs.mongodb.com/manual/reference/text-search-languages/ Args: @@ -138,7 +138,7 @@ def validate_language(value): None: validation successful Raises: - ValidationError: raises execption incase language is not valid. + ValidationError: will raise exception in case language is not valid. """ if value not in VALID_LANGUAGES: error_str = ('MongoDB does not support text search for the ' diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index b21f41d6..b1aa5c12 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -52,21 +52,20 @@ def deserialize(data): def validate_txn_obj(obj_name, obj, key, validation_fun): - """Validates value associated to `key` in `obj` by applying - `validation_fun`. + """Validate value of `key` in `obj` using `validation_fun`. Args: obj_name (str): name for `obj` being validated. - obj (dict): dictonary object. + obj (dict): dictionary object. key (str): key to be validated in `obj`. validation_fun (function): function used to validate the value of `key`. Returns: - None: indicates validation successfull + None: indicates validation successful Raises: - ValidationError: `validation_fun` will raise this error on failure + ValidationError: `validation_fun` will raise exception on failure """ backend = bigchaindb.config['database']['backend'] @@ -76,16 +75,16 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): def validate_all_keys(obj_name, obj, validation_fun): - """Validates all (nested) keys in `obj` by using `validation_fun` + """Validate all (nested) keys in `obj` by using `validation_fun`. Args: obj_name (str): name for `obj` being validated. - obj (dict): dictonary object. + obj (dict): dictionary object. validation_fun (function): function used to validate the value of `key`. Returns: - None: indicates validation successfull + None: indicates validation successful Raises: ValidationError: `validation_fun` will raise this error on failure @@ -96,12 +95,12 @@ def validate_all_keys(obj_name, obj, validation_fun): validate_all_keys(obj_name, value, validation_fun) -def validate_all_value_for_key(obj, key, validation_fun): - """Validates value for all (nested) occurences of `key` in `obj` - using `validation_fun` +def validate_all_values_for_key(obj, key, validation_fun): + """Validate value for all (nested) occurrence of `key` in `obj` + using `validation_fun`. Args: - obj (dict): dictonary object. + obj (dict): dictionary object. key (str): key whose value is to be validated. validation_fun (function): function used to validate the value of `key`. @@ -113,11 +112,12 @@ def validate_all_value_for_key(obj, key, validation_fun): if vkey == key: validation_fun(value) elif isinstance(value, dict): - validate_all_value_for_key(value, key, validation_fun) + validate_all_values_for_key(value, key, validation_fun) def validate_key(obj_name, key): - """Check if `key` contains ".", "$" or null characters + """Check if `key` contains ".", "$" or null characters. + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names Args: @@ -125,10 +125,10 @@ def validate_key(obj_name, key): key (str): key to validated Returns: - None: indicates validation successfull + None: validation successful Raises: - ValidationError: raise execption incase of regex match. + ValidationError: will raise exception in case of regex match. """ if re.search(r'^[$]|\.|\x00', key): error_str = ('Invalid key name "{}" in {} object. The ' From 7941922ac04ea7173f9fcdd2dcdf3e08458c6ab0 Mon Sep 17 00:00:00 2001 From: kansi Date: Mon, 6 Nov 2017 10:43:54 +0530 Subject: [PATCH 14/24] Added type validation for data --- bigchaindb/backend/schema.py | 5 +++-- bigchaindb/common/utils.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 0e2815db..4f2ebd6c 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -123,8 +123,9 @@ def validate_language_key(obj, key): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = obj.get(key, {}) or {} - validate_all_values_for_key(data, 'language', validate_language) + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_values_for_key(data, 'language', validate_language) def validate_language(value): diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index b1aa5c12..9ad448f5 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -70,8 +70,9 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = obj.get(key, {}) or {} - validate_all_keys(obj_name, data, validation_fun) + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_keys(obj_name, data, validation_fun) def validate_all_keys(obj_name, obj, validation_fun): From 3cbec5a864f004b5e35d2c6b45acb7bcecb059dd Mon Sep 17 00:00:00 2001 From: kansi Date: Mon, 6 Nov 2017 12:06:01 +0530 Subject: [PATCH 15/24] Updated metadata and asset model docs --- docs/server/source/data-models/asset-model.md | 8 ++++++++ docs/server/source/data-models/transaction-model.rst | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index eefa81bb..2d7ac1e8 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -18,3 +18,11 @@ In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pa "id": "38100137cea87fb9bd751e2372abb2c73e7d5bcf39d940a5516a324d9c7fb88d" } ``` + + +.. note:: + + When using MongoDB for storage certain restriction apply to all (including nested) keys of `"data"` JSON document i.e. valid keys should **not** begin with ``$`` character and cannot contain ``.`` or null character (Unicode code point 0000). Furthermore, the key `"language"` (at any level in the hierarchy) in `"data"` JSON document is a spcial key and used for specifying text search language. Its value must be one of the allowed values, see `Text search languages `_ . It must be noted that only the languages supported by MongoDB community edition are allowed. + + + diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 38e523bd..2b28b592 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -46,6 +46,10 @@ Here's some explanation of the contents: - **metadata**: User-provided transaction metadata. It can be any valid JSON document, or ``null``. + **NOTE** when using MongoDB for storage certain restriction apply to all + (including nested) keys of ``"metadata"`` JSON document i.e. valid keys + should **not** begin with ``$`` character and cannot contain ``.`` or + null character (Unicode code point 0000). **How the transaction ID is computed.** 1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, From 636c28d31177121d382859ad27b35935a9d92e16 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Nov 2017 14:22:17 +0100 Subject: [PATCH 16/24] Edited changes to the docs about the asset model --- docs/server/source/data-models/asset-model.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index 2d7ac1e8..054a4083 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -2,6 +2,8 @@ To avoid redundant data in transactions, the asset model is different for `CREATE` and `TRANSFER` transactions. +## In CREATE Transactions + In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair. The key must be `"data"` and the value can be any valid JSON document, or `null`. For example: ```json { @@ -12,17 +14,18 @@ In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair } ``` +When using MongoDB for storage, certain restriction apply to all (including nested) keys of the `"data"` JSON document: + +* Keys (i.e. key names, not values) must **not** begin with the `$` character. +* Keys must not contain `.` or the null character (Unicode code point 0000). +* The key `"language"` (at any level in the hierarchy) is a special key and used for specifying text search language. Its value must be one of the allowed values; see the valid [Text Search Languages](https://docs.mongodb.com/manual/reference/text-search-languages/) in the MongoDB Docs. In BigchainDB, only the languages supported by _MongoDB community edition_ are allowed. + + +## In TRANSFER Transactions + In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pair. They key must be `"id"` and the value must contain a transaction ID (i.e. a SHA3-256 hash: the ID of the `CREATE` transaction which created the asset, which also serves as the asset ID). For example: ```json { "id": "38100137cea87fb9bd751e2372abb2c73e7d5bcf39d940a5516a324d9c7fb88d" } ``` - - -.. note:: - - When using MongoDB for storage certain restriction apply to all (including nested) keys of `"data"` JSON document i.e. valid keys should **not** begin with ``$`` character and cannot contain ``.`` or null character (Unicode code point 0000). Furthermore, the key `"language"` (at any level in the hierarchy) in `"data"` JSON document is a spcial key and used for specifying text search language. Its value must be one of the allowed values, see `Text search languages `_ . It must be noted that only the languages supported by MongoDB community edition are allowed. - - - From cae883e9cbca276f491fbf0412b3997be3b74e5b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Nov 2017 14:30:01 +0100 Subject: [PATCH 17/24] Edited the changes to transaction model docs --- docs/server/source/data-models/transaction-model.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 2b28b592..b4f2a4f1 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -46,10 +46,10 @@ Here's some explanation of the contents: - **metadata**: User-provided transaction metadata. It can be any valid JSON document, or ``null``. - **NOTE** when using MongoDB for storage certain restriction apply to all - (including nested) keys of ``"metadata"`` JSON document i.e. valid keys - should **not** begin with ``$`` character and cannot contain ``.`` or - null character (Unicode code point 0000). + **NOTE:** When using MongoDB for storage, certain restriction apply + to all (including nested) keys of the ``"data"`` JSON document: + 1) keys (i.e. key names, not values) must **not** begin with the ``$`` character, and + 2) keys must not contain ``.`` or the null character (Unicode code point 0000). **How the transaction ID is computed.** 1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, From d55004601c6a1e6e8444b73f9ee1f1b3d70dd61e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Nov 2017 10:17:21 +0100 Subject: [PATCH 18/24] Fix command to run RethinkDB in bg w/ docker-compose --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 252fcda4..9fc9b9d5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -105,7 +105,7 @@ $ docker-compose build First, start `RethinkDB` in the background: ```text -$ docker-compose up -d rdb +$ docker-compose -f docker-compose.rdb.yml up -d rdb ``` then run the tests using: From 6cab2f26e92317f16b97174cb72c6b8e4c5838dd Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Nov 2017 10:20:41 +0100 Subject: [PATCH 19/24] Fix command to run tests w/ docker-compose & bdb-rdb --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 9fc9b9d5..e3c9133c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -111,7 +111,7 @@ $ docker-compose -f docker-compose.rdb.yml up -d rdb then run the tests using: ```text -$ docker-compose run --rm bdb-rdb py.test -v +$ docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb py.test -v ``` to rebuild all the images (usually you only need to rebuild the `bdb` and From 220465f7016bd2b71fb14f0e428a84ef418596c7 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 7 Nov 2017 14:55:05 +0530 Subject: [PATCH 20/24] Added "--no-init" flag for "bigchaindb start" command --- bigchaindb/commands/bigchaindb.py | 9 +++++---- .../source/appendices/azure-quickstart-template.md | 2 +- docs/server/source/dev-and-test/setup-bdb-host.md | 4 ++-- docs/server/source/quickstart.md | 2 +- docs/server/source/server-reference/bigchaindb-cli.md | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 9705065c..6320d279 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -196,7 +196,7 @@ def run_start(args): logger.info('RethinkDB started with PID %s' % proc.pid) try: - if args.initialize_database: + if not args.skip_initialize_database: logger.info('Initializing database') _run_init() except DatabaseAlreadyExists: @@ -302,10 +302,11 @@ def create_parser(): action='store_true', help='Run RethinkDB on start') - start_parser.add_argument('--init', - dest='initialize_database', + start_parser.add_argument('--no-init', + dest='skip_initialize_database', + default=False, action='store_true', - help='Force initialize database') + help='Skip database initialization') # parser for configuring the number of shards sharding_parser = subparsers.add_parser('set-shards', diff --git a/docs/server/source/appendices/azure-quickstart-template.md b/docs/server/source/appendices/azure-quickstart-template.md index 13cda281..59f52fd3 100644 --- a/docs/server/source/appendices/azure-quickstart-template.md +++ b/docs/server/source/appendices/azure-quickstart-template.md @@ -33,7 +33,7 @@ API Server bind? (default `localhost:9984`): 0.0.0.0:9984 Finally, run BigchainDB Server by doing: ```text -bigchaindb start --init +bigchaindb start ``` BigchainDB Server should now be running on the Azure virtual machine. diff --git a/docs/server/source/dev-and-test/setup-bdb-host.md b/docs/server/source/dev-and-test/setup-bdb-host.md index 5feb8c42..cdee3c0b 100644 --- a/docs/server/source/dev-and-test/setup-bdb-host.md +++ b/docs/server/source/dev-and-test/setup-bdb-host.md @@ -27,7 +27,7 @@ waiting for connections on port 27017 To run BigchainDB Server, do: ```text -$ bigchaindb start --init +$ bigchaindb start ``` You can [run all the unit tests](running-all-tests.html) to test your installation. @@ -55,7 +55,7 @@ You can verify that RethinkDB is running by opening the RethinkDB web interface To run BigchainDB Server, do: ```text -$ bigchaindb start --init +$ bigchaindb start ``` You can [run all the unit tests](running-all-tests.html) to test your installation. diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 2375fd5f..63ab8643 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -54,7 +54,7 @@ $ bigchaindb -y configure mongodb I. Run BigchainDB Server: ```text -$ bigchaindb start --init +$ bigchaindb start ``` J. Verify BigchainDB Server setup by visiting the BigchainDB Root URL in your browser: diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index fddfd3f5..790cb453 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -61,7 +61,7 @@ If you want to force-drop the database (i.e. skipping the yes/no prompt), then u ## bigchaindb start -Start BigchainDB assuming that the database has already been initialized using `bigchaindb init`. If that is not the case then passing the flag `--init` will initialize the database and start BigchainDB. +Start BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`. The database initialization step is optional and can be skipped by passing the `--no-init` flag i.e. `bigchaindb start --no-init`. You can also use the `--dev-start-rethinkdb` command line option to automatically start rethinkdb with bigchaindb if rethinkdb is not already running, e.g. `bigchaindb --dev-start-rethinkdb start`. Note that this will also shutdown rethinkdb when the bigchaindb process stops. The option `--dev-allow-temp-keypair` will generate a keypair on the fly if no keypair is found, this is useful when you want to run a temporary instance of BigchainDB in a Docker container, for example. From 8aba802425bd9383ac15af32e2278fd649362c05 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 7 Nov 2017 15:41:55 +0530 Subject: [PATCH 21/24] Fix tests --- tests/commands/conftest.py | 2 +- tests/commands/rethinkdb/test_commands.py | 2 +- tests/commands/test_commands.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index 1aef2d30..46f8a8f6 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -49,7 +49,7 @@ def run_start_args(request): config=param.get('config'), start_rethinkdb=param.get('start_rethinkdb', False), allow_temp_keypair=param.get('allow_temp_keypair', False), - initialize_database=param.get('initialize_database', True), + skip_initialize_database=param.get('skip_initialize_database', False), ) diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index 29a84972..c8990582 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -14,7 +14,7 @@ def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, from bigchaindb import config from bigchaindb.commands.bigchaindb import run_start args = Namespace(start_rethinkdb=True, allow_temp_keypair=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mock_start_rethinkdb.assert_called_with() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 15aa9302..423e4614 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -40,7 +40,7 @@ def test_bigchain_run_start(mock_run_configure, from bigchaindb import config from bigchaindb.commands.bigchaindb import run_start args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with(user_log_config=config['log']) @@ -290,7 +290,7 @@ def test_allow_temp_keypair_generates_one_on_the_fly( bigchaindb.config['keypair'] = {'private': None, 'public': None} args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with( @@ -317,7 +317,7 @@ def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, assert isinstance(original_private_key, str) args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with( From 1266c43ff87366ee7b3fb1f8a668feeae537a75e Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 7 Nov 2017 18:44:51 +0530 Subject: [PATCH 22/24] Fix docker files --- docker-compose.benchmark.yml | 2 +- docker-compose.rdb.yml | 2 +- docker-compose.yml | 2 +- tests/integration/test_integration.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.benchmark.yml b/docker-compose.benchmark.yml index 2a2aacc2..c7319040 100644 --- a/docker-compose.benchmark.yml +++ b/docker-compose.benchmark.yml @@ -25,7 +25,7 @@ services: BIGCHAINDB_GRAPHITE_HOST: graphite ports: - "9984" - command: bigchaindb start --init + command: bigchaindb start graphite: image: hopsoft/graphite-statsd diff --git a/docker-compose.rdb.yml b/docker-compose.rdb.yml index e02aa444..15f91675 100644 --- a/docker-compose.rdb.yml +++ b/docker-compose.rdb.yml @@ -45,4 +45,4 @@ services: BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 ports: - "9984" - command: bigchaindb start --init + command: bigchaindb start diff --git a/docker-compose.yml b/docker-compose.yml index 8f774106..cd6aa2aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,4 +30,4 @@ services: BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 ports: - "9984" - command: bigchaindb start --init + command: bigchaindb start diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 2bf0ebcd..21b783e6 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -12,9 +12,9 @@ def test_double_create(b, user_pk): metadata={'test': 'test'}).sign([b.me_private]) b.write_transaction(tx) - time.sleep(2) + time.sleep(5) b.write_transaction(tx) - time.sleep(2) + time.sleep(5) tx_returned = b.get_transaction(tx.id) # test that the tx can be queried From e725c23691a69d3e6ebdd35fa9f0c69320474509 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Nov 2017 14:24:33 +0100 Subject: [PATCH 23/24] Added note about 'make clean-pyc' --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index e3c9133c..146d3bc6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -115,7 +115,7 @@ $ docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb py.test -v ``` to rebuild all the images (usually you only need to rebuild the `bdb` and - `bdb-rdb` images). + `bdb-rdb` images). If that fails, then do `make clean-pyc` and try again. ## Automated Testing of All Pull Requests From 5ce2eb1f971086868f1cc4869db23affd1487ac8 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Thu, 9 Nov 2017 00:11:17 +0530 Subject: [PATCH 24/24] Bypass CI issues for rethinkdb (#1821) Add custom marker to skip some travis/rdb tests --- tests/conftest.py | 9 +++++++++ tests/integration/test_federation.py | 3 +++ tests/integration/test_integration.py | 1 + 3 files changed, 13 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index b930e4ec..4b5c7946 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,15 @@ USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie' USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' +def pytest_runtest_setup(item): + if isinstance(item, item.Function): + if item.get_marker('skip_travis_rdb'): + if (os.getenv('TRAVIS_CI') == 'true' and + os.getenv('BIGCHAINDB_DATABASE_BACKEND') == 'rethinkdb'): + pytest.skip( + 'Skip test during Travis CI build when using rethinkdb') + + def pytest_addoption(parser): from bigchaindb.backend.connection import BACKENDS diff --git a/tests/integration/test_federation.py b/tests/integration/test_federation.py index 598412ff..22e2e8da 100644 --- a/tests/integration/test_federation.py +++ b/tests/integration/test_federation.py @@ -97,6 +97,7 @@ def process_vote(steps, result=None): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_valid(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) @@ -115,6 +116,7 @@ def test_elect_valid(federation_3): @pytest.mark.bdb +@pytest.mark.skip_travis_rdb @pytest.mark.genesis def test_elect_invalid(federation_3): [bx, (s0, s1, s2)] = federation_3 @@ -135,6 +137,7 @@ def test_elect_invalid(federation_3): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_sybill(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 2bf0ebcd..50734fee 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -5,6 +5,7 @@ import pytest pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('processes')] +@pytest.mark.skip_travis_rdb def test_double_create(b, user_pk): from bigchaindb.models import Transaction from bigchaindb.backend.query import count_blocks