Problem: Public key format not standardized (#2508)

Solution: Add 'type' field to public key so that it can be decoded properly

bigchaindb/commands/bigchaindb.py

@@ -124,7 +124,8 @@ def run_upsert_validator_new(args, bigchain):
     """
 
     new_validator = {
-        'public_key': public_key_from_base64(args.public_key),
+        'public_key': {'value': public_key_from_base64(args.public_key),
+                       'type': 'ed25519-base16'},
         'power': args.power,
         'node_id': args.node_id
     }
@@ -207,7 +208,7 @@ def run_upsert_validator_show(args, bigchain):
 
     new_validator = election.asset['data']
 
-    public_key = public_key_to_base64(new_validator['public_key'])
+    public_key = public_key_to_base64(new_validator['public_key']['value'])
     power = new_validator['power']
     node_id = new_validator['node_id']
     status = election.get_status(bigchain)

bigchaindb/common/exceptions.py

@@ -112,3 +112,7 @@ class UnequalValidatorSet(ValidationError):
 
 class InvalidPowerChange(ValidationError):
     """Raised if proposed power change in validator set is >=1/3 total power"""
+
+
+class InvalidPublicKey(ValidationError):
+    """Raised if public key doesn't match the encoding type"""

bigchaindb/common/schema/transaction_validator_election_v2.0.yaml

@@ -23,7 +23,20 @@ properties:
           node_id:
             type: string
           public_key:
-            type: string
+            type: object
+            additionalProperties: false
+            required:
+            - value
+            - type
+            properties:
+              value:
+                type: string
+              type:
+                type: string
+                enum:
+                  - ed25519-base16
+                  - ed25519-base32
+                  - ed25519-base64
           power:
             "$ref": "#/definitions/positiveInteger"
         required:

bigchaindb/upsert_validator/validator_election.py

@@ -19,7 +19,10 @@ from bigchaindb.common.schema import (_validate_schema,
                                       TX_SCHEMA_COMMON,
                                       TX_SCHEMA_CREATE)
 from . import ValidatorElectionVote
-from .validator_utils import (new_validator_set, encode_validator)
+from .validator_utils import (new_validator_set,
+                              encode_validator,
+                              encode_pk_to_base16,
+                              validate_asset_public_key)
 
 
 class ValidatorElection(Transaction):
@@ -58,7 +61,7 @@ class ValidatorElection(Transaction):
         validators = {}
         for validator in bigchain.get_validators(height):
             # NOTE: we assume that Tendermint encodes public key in base64
-            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['data']))
+            public_key = public_key_from_ed25519_key(key_from_base64(validator['public_key']['value']))
             validators[public_key] = validator['voting_power']
 
         return validators
@@ -155,6 +158,7 @@ class ValidatorElection(Transaction):
         _validate_schema(TX_SCHEMA_COMMON, tx)
         _validate_schema(TX_SCHEMA_CREATE, tx)
         _validate_schema(TX_SCHEMA_VALIDATOR_ELECTION, tx)
+        validate_asset_public_key(tx['asset']['data']['public_key'])
 
     @classmethod
     def create(cls, tx_signers, recipients, metadata=None, asset=None):
@@ -236,7 +240,10 @@ class ValidatorElection(Transaction):
 
                 updated_validator_set = [v for v in updated_validator_set if v['voting_power'] > 0]
                 bigchain.store_validator_set(new_height+1, updated_validator_set, election.id)
-                return [encode_validator(election.asset['data'])]
+
+                validator16 = encode_pk_to_base16(election.asset['data'])
+                return [encode_validator(validator16)]
+
         return []
 
     def get_validator_update_by_election_id(self, election_id, bigchain):

bigchaindb/upsert_validator/validator_utils.py

@@ -1,12 +1,14 @@
 import codecs
+import base64
+import binascii
 
 from abci.types_pb2 import (Validator,
                             PubKey)
-from bigchaindb.tendermint_utils import public_key_to_base64
+from bigchaindb.common.exceptions import InvalidPublicKey
 
 
 def encode_validator(v):
-    ed25519_public_key = v['public_key']
+    ed25519_public_key = v['public_key']['value']
     # NOTE: tendermint expects public to be encoded in go-amino format
     pub_key = PubKey(type='ed25519',
                      data=bytes.fromhex(ed25519_public_key))
@@ -16,22 +18,60 @@ def encode_validator(v):
 
 
 def decode_validator(v):
-    return {'pub_key': {'type': v.pub_key.type,
+    return {'public_key': {'type': 'ed25519-base64',
-                        'data': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
+                           'value': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
             'voting_power': v.power}
 
 
 def new_validator_set(validators, updates):
     validators_dict = {}
     for v in validators:
-        validators_dict[v['pub_key']['data']] = v
+        validators_dict[v['public_key']['value']] = v
 
     updates_dict = {}
     for u in updates:
-        public_key64 = public_key_to_base64(u['public_key'])
+        decoder = get_public_key_decoder(u['public_key'])
-        updates_dict[public_key64] = {'pub_key': {'type': 'ed25519',
+        public_key64 = base64.b64encode(decoder(u['public_key']['value'])).decode('utf-8')
-                                                  'data': public_key64},
+        updates_dict[public_key64] = {'public_key': {'type': 'ed25519-base64',
+                                                     'value': public_key64},
                                       'voting_power': u['power']}
 
     new_validators_dict = {**validators_dict, **updates_dict}
     return list(new_validators_dict.values())
+
+
+def encode_pk_to_base16(validator):
+    pk = validator['public_key']
+    decoder = get_public_key_decoder(pk)
+    public_key16 = base64.b16encode(decoder(pk['value'])).decode('utf-8')
+
+    validator['public_key']['value'] = public_key16
+    return validator
+
+
+def validate_asset_public_key(pk):
+    pk_binary = pk['value'].encode('utf-8')
+    decoder = get_public_key_decoder(pk)
+    try:
+        pk_decoded = decoder(pk_binary)
+        if len(pk_decoded) != 32:
+            raise InvalidPublicKey('Public key should be of size 32 bytes')
+
+    except binascii.Error as e:
+        raise InvalidPublicKey('Invalid `type` specified for public key `value`')
+
+
+def get_public_key_decoder(pk):
+    encoding = pk['type']
+    decoder = base64.b64decode
+
+    if encoding == 'ed25519-base16':
+        decoder = base64.b16decode
+    elif encoding == 'ed25519-base32':
+        decoder = base64.b32decode
+    elif encoding == 'ed25519-base64':
+        decoder = base64.b64decode
+    else:
+        raise InvalidPublicKey('Invalid `type` specified for public key `value`')
+
+    return decoder

tests/commands/test_commands.py

@@ -353,7 +353,7 @@ def test_upsert_validator_new_with_tendermint(b, priv_validator_path, user_sk, v
     from bigchaindb.commands.bigchaindb import run_upsert_validator_new
 
     new_args = Namespace(action='new',
-                         public_key='8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie',
+                         public_key='HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52GnGuPY=',
                          power=1,
                          node_id='unique_node_id_for_test_upsert_validator_new_with_tendermint',
                          sk=priv_validator_path,
@@ -444,6 +444,7 @@ def test_upsert_validator_approve_with_tendermint(b, priv_validator_path, user_s
                          config={})
 
     election_id = run_upsert_validator_new(new_args, b)
+    assert election_id
 
     args = Namespace(action='approve',
                      election_id=election_id,
@@ -524,8 +525,8 @@ def mock_get_validators(height):
     keys = node_keys()
     pub_key = list(keys.keys())[0]
     return [
-        {'pub_key': {'data': pub_key,
+        {'public_key': {'value': pub_key,
-                     'type': 'tendermint/PubKeyEd25519'},
+                        'type': 'ed25519-base64'},
          'voting_power': 10}
     ]
 

tests/conftest.py

@@ -649,9 +649,8 @@ def validators(b, node_keys):
     (public_key, private_key) = list(node_keys.items())[0]
 
     validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2',
-                      'pub_key': {
+                      'public_key': {'value': public_key,
-                          'data': public_key,
+                                     'type': 'ed25519-base64'},
-                          'type': 'ed25519'},
                       'voting_power': 10}]
 
     validator_update = {'validators': validator_set,
@@ -687,6 +686,7 @@ def new_validator():
     power = 1
     node_id = 'fake_node_id'
 
-    return {'public_key': public_key,
+    return {'public_key': {'value': public_key,
+                           'type': 'ed25519-base16'},
             'power': power,
             'node_id': node_id}

tests/tendermint/test_core.py

@@ -415,12 +415,14 @@ def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
 
 
 def test_new_validator_set(b):
-    node1 = {'pub_key': {'type': 'ed25519',
+    node1 = {'public_key': {'type': 'ed25519-base64',
-                         'data': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
+                            'value': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
              'voting_power': 10}
-    node1_new_power = {'public_key': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+    node1_new_power = {'public_key': {'value': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+                                      'type': 'ed25519-base16'},
                        'power': 20}
-    node2 = {'public_key': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+    node2 = {'public_key': {'value': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+                            'type': 'ed25519-base16'},
              'power': 10}
 
     validators = [node1]
@@ -430,8 +432,8 @@ def test_new_validator_set(b):
 
     updated_validators = []
     for u in updates:
-        updated_validators.append({'pub_key': {'type': 'ed25519',
+        updated_validators.append({'public_key': {'type': 'ed25519-base64',
-                                               'data': public_key_to_base64(u['public_key'])},
+                                                  'value': public_key_to_base64(u['public_key']['value'])},
                                    'voting_power':  u['power']})
 
     assert updated_validator_set == updated_validators

tests/tendermint/test_integration.py

@@ -40,7 +40,7 @@ def test_app(b, init_chain_request):
 
     pk = codecs.encode(init_chain_request.validators[0].pub_key.data, 'base64').decode().strip('\n')
     [validator] = b.get_validators(height=1)
-    assert validator['pub_key']['data'] == pk
+    assert validator['public_key']['value'] == pk
     assert validator['voting_power'] == 10
 
     alice = generate_key_pair()

tests/upsert_validator/conftest.py

@@ -22,7 +22,7 @@ def mock_get_validators(network_validators):
         validators = []
         for public_key, power in network_validators.items():
             validators.append({
-                'pub_key': {'type': 'AC26791624DE60', 'data': public_key},
+                'public_key': {'type': 'ed25519-base64', 'value': public_key},
                 'voting_power': power
             })
         return validators

tests/upsert_validator/test_validator_election.py

@@ -24,6 +24,19 @@ def test_upsert_validator_valid_election(b_mock, new_validator, node_key):
     assert election.validate(b_mock)
 
 
+def test_upsert_validator_invalid_election_public_key(b_mock, new_validator, node_key):
+    from bigchaindb.common.exceptions import InvalidPublicKey
+
+    for iv in ['ed25519-base32', 'ed25519-base64']:
+        new_validator['public_key']['type'] = iv
+        voters = ValidatorElection.recipients(b_mock)
+
+        with pytest.raises(InvalidPublicKey):
+            ValidatorElection.generate([node_key.public_key],
+                                       voters,
+                                       new_validator, None).sign([node_key.private_key])
+
+
 def test_upsert_validator_invalid_power_election(b_mock, new_validator, node_key):
     voters = ValidatorElection.recipients(b_mock)
     new_validator['power'] = 30
@@ -147,7 +160,7 @@ def test_upsert_validator_show(caplog, ongoing_election, b):
     from bigchaindb.commands.bigchaindb import run_upsert_validator_show
 
     election_id = ongoing_election.id
-    public_key = public_key_to_base64(ongoing_election.asset['data']['public_key'])
+    public_key = public_key_to_base64(ongoing_election.asset['data']['public_key']['value'])
     power = ongoing_election.asset['data']['power']
     node_id = ongoing_election.asset['data']['node_id']
     status = ValidatorElection.ONGOING

tests/upsert_validator/test_validator_election_vote.py

@@ -228,8 +228,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
 
     (node_pub, _) = list(node_keys.items())[0]
 
-    validators = [{'pub_key': {'type': 'ed25519',
+    validators = [{'public_key': {'type': 'ed25519-base64', 'value': node_pub},
-                               'data': node_pub},
                    'voting_power': 10}]
 
     latest_block = b.get_latest_block()
@@ -239,7 +238,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
     public_key64 = public_key_to_base64(public_key)
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
 
@@ -268,7 +267,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
     new_validator_set = b.get_validators()
     validator_pub_keys = []
     for v in new_validator_set:
-        validator_pub_keys.append(v['pub_key']['data'])
+        validator_pub_keys.append(v['public_key']['value'])
 
     assert (public_key64 in validator_pub_keys)
 
@@ -281,7 +280,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
     public_key64 = public_key_to_base64(public_key)
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
     voters = ValidatorElection.recipients(b)
@@ -316,7 +315,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
 
     # remove validator
     power = 0
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
     voters = ValidatorElection.recipients(b)
@@ -339,7 +338,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
 
     # assert that the public key is not a part of the current validator set
     for v in b.get_validators(10):
-        assert not v['pub_key']['data'] == public_key64
+        assert not v['public_key']['value'] == public_key64
 
 
 # ============================================================================
@@ -365,7 +364,7 @@ def gen_vote(election, i, ed25519_node_keys):
 def reset_validator_set(b, node_keys, height):
     validators = []
     for (node_pub, _) in node_keys.items():
-        validators.append({'pub_key': {'type': 'ed25519',
+        validators.append({'public_key': {'type': 'ed25519-base64',
-                                       'data': node_pub},
+                                          'value': node_pub},
                            'voting_power': 10})
     b.store_validator_set(height, validators, 'election_id')