125 lines
4.5 KiB
Python
125 lines
4.5 KiB
Python
# Copyright © 2020 Interplanetary Database Association e.V.,
|
|
# BigchainDB and IPDB software contributors.
|
|
# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
|
|
# Code is Apache-2.0 and docs are CC-BY-4.0
|
|
|
|
import multiprocessing as mp
|
|
from collections import defaultdict
|
|
|
|
from bigchaindb import App, BigchainDB
|
|
from bigchaindb.tendermint_utils import decode_transaction
|
|
from abci import CodeTypeOk
|
|
|
|
|
|
class ParallelValidationApp(App):
|
|
def __init__(self, bigchaindb=None, events_queue=None, abci=None):
|
|
super().__init__(bigchaindb, events_queue, abci=abci)
|
|
self.parallel_validator = ParallelValidator()
|
|
self.parallel_validator.start()
|
|
|
|
def check_tx(self, raw_transaction):
|
|
return self.abci.ResponseCheckTx(code=CodeTypeOk)
|
|
|
|
def deliver_tx(self, raw_transaction):
|
|
self.parallel_validator.validate(raw_transaction)
|
|
return self.abci.ResponseDeliverTx(code=CodeTypeOk)
|
|
|
|
def end_block(self, request_end_block):
|
|
result = self.parallel_validator.result(timeout=30)
|
|
for transaction in result:
|
|
if transaction:
|
|
self.block_txn_ids.append(transaction.id)
|
|
self.block_transactions.append(transaction)
|
|
|
|
return super().end_block(request_end_block)
|
|
|
|
|
|
RESET = 'reset'
|
|
EXIT = 'exit'
|
|
|
|
|
|
class ParallelValidator:
|
|
def __init__(self, number_of_workers=mp.cpu_count()):
|
|
self.number_of_workers = number_of_workers
|
|
self.transaction_index = 0
|
|
self.routing_queues = [mp.Queue() for _ in range(self.number_of_workers)]
|
|
self.workers = []
|
|
self.results_queue = mp.Queue()
|
|
|
|
def start(self):
|
|
for routing_queue in self.routing_queues:
|
|
worker = ValidationWorker(routing_queue, self.results_queue)
|
|
process = mp.Process(target=worker.run)
|
|
process.start()
|
|
self.workers.append(process)
|
|
|
|
def stop(self):
|
|
for routing_queue in self.routing_queues:
|
|
routing_queue.put(EXIT)
|
|
|
|
def validate(self, raw_transaction):
|
|
dict_transaction = decode_transaction(raw_transaction)
|
|
index = int(dict_transaction['id'], 16) % self.number_of_workers
|
|
self.routing_queues[index].put((self.transaction_index, dict_transaction))
|
|
self.transaction_index += 1
|
|
|
|
def result(self, timeout=None):
|
|
result_buffer = [None] * self.transaction_index
|
|
for _ in range(self.transaction_index):
|
|
index, transaction = self.results_queue.get(timeout=timeout)
|
|
result_buffer[index] = transaction
|
|
self.transaction_index = 0
|
|
for routing_queue in self.routing_queues:
|
|
routing_queue.put(RESET)
|
|
return result_buffer
|
|
|
|
|
|
class ValidationWorker:
|
|
"""Run validation logic in a loop. This Worker is suitable for a Process
|
|
life: no thrills, just a queue to get some values, and a queue to return results.
|
|
|
|
Note that a worker is expected to validate multiple transactions in
|
|
multiple rounds, and it needs to keep in memory all transactions already
|
|
validated, until a new round starts. To trigger a new round of validation,
|
|
a ValidationWorker expects a `RESET` message. To exit the infinite loop the
|
|
worker is in, it expects an `EXIT` message.
|
|
"""
|
|
|
|
def __init__(self, in_queue, results_queue):
|
|
self.in_queue = in_queue
|
|
self.results_queue = results_queue
|
|
self.bigchaindb = BigchainDB()
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
# We need a place to store already validated transactions,
|
|
# in case of dependant transactions in the same block.
|
|
# `validated_transactions` maps an `asset_id` with the list
|
|
# of all other transactions sharing the same asset.
|
|
self.validated_transactions = defaultdict(list)
|
|
|
|
def validate(self, dict_transaction):
|
|
try:
|
|
asset_id = dict_transaction['asset']['id']
|
|
except KeyError:
|
|
asset_id = dict_transaction['id']
|
|
|
|
transaction = self.bigchaindb.is_valid_transaction(
|
|
dict_transaction,
|
|
self.validated_transactions[asset_id])
|
|
|
|
if transaction:
|
|
self.validated_transactions[asset_id].append(transaction)
|
|
return transaction
|
|
|
|
def run(self):
|
|
while True:
|
|
message = self.in_queue.get()
|
|
if message == RESET:
|
|
self.reset()
|
|
elif message == EXIT:
|
|
return
|
|
else:
|
|
index, transaction = message
|
|
self.results_queue.put((index, self.validate(transaction)))
|