From c37c169d6af78c1cfd67130ba48d41a9a70b9a77 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 15 Sep 2020 01:08:47 -0700 Subject: [PATCH] add decentralized DDO --- .travis.yml | 4 +- package-lock.json | 5 + package.json | 1 + src/metadatastore/OnChainMetaData.ts | 211 +++++++++++++++++++++++ src/models/Config.ts | 11 ++ src/ocean/Assets.ts | 10 +- src/ocean/Ocean.ts | 12 +- src/utils/ConfigHelper.ts | 28 ++- test/integration/ComputeFlow.test.ts | 6 +- test/integration/Marketplaceflow.test.ts | 6 +- 10 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 src/metadatastore/OnChainMetaData.ts diff --git a/.travis.yml b/.travis.yml index 1db2198d..652fc236 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,10 @@ before_script: # Barge setup - git clone https://github.com/oceanprotocol/barge - cd barge - - git checkout v3 + - git checkout feature/ocean-contracts - export PROVIDER_VERSION=latest + - export ADDRESS_FILE="${HOME}/.ocean/ocean-contracts/artifacts/address.json" + - export AQUARIUS_URI="http://172.15.0.5:5000" - bash -x start_ocean.sh --no-dashboard 2>&1 > start_ocean.log & - cd .. - sleep 300 diff --git a/package-lock.json b/package-lock.json index 86c19473..dff86387 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6603,6 +6603,11 @@ "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", "dev": true }, + "lzma": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/lzma/-/lzma-2.3.2.tgz", + "integrity": "sha1-N4OySFi5wOdHoN88vx+1/KqSxEE=" + }, "macos-release": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.1.tgz", diff --git a/package.json b/package.json index 97f512f4..d08f6452 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@oceanprotocol/contracts": "^0.4.1", "decimal.js": "^10.2.0", "fs": "0.0.1-security", + "lzma": "^2.3.2", "node-fetch": "^2.6.1", "save-file": "^2.3.1", "uuid": "^8.3.0", diff --git a/src/metadatastore/OnChainMetaData.ts b/src/metadatastore/OnChainMetaData.ts new file mode 100644 index 00000000..bc9d8ec3 --- /dev/null +++ b/src/metadatastore/OnChainMetaData.ts @@ -0,0 +1,211 @@ +import { DDO } from '../ddo/DDO' +import { TransactionReceipt } from 'web3-core' +import { Contract } from 'web3-eth-contract' +import { AbiItem } from 'web3-utils/types' +import Web3 from 'web3' +import defaultDDOContractABI from '@oceanprotocol/contracts/artifacts/DDO.json' +import { didZeroX } from '../utils' +import { LZMA } from 'lzma' + +/** + * Provides an interface with Metadata Store. + * Metadata Store provides an off-chain database store for metadata about data assets. + */ +export class OnChainMetadataStore { + public DDOContractAddress: string + public DDOContractABI: AbiItem | AbiItem[] + public web3: Web3 + public DDOContract: Contract = null + /** + * Instantiate OnChainMetadata Store for on-chain interaction. + */ + constructor( + web3: Web3, + DDOContractAddress: string = null, + DDOContractABI: AbiItem | AbiItem[] = null + ) { + this.web3 = web3 + this.DDOContractAddress = DDOContractAddress + this.DDOContractABI = DDOContractABI || (defaultDDOContractABI.abi as AbiItem[]) + if (web3) + this.DDOContract = new this.web3.eth.Contract( + this.DDOContractABI, + this.DDOContractAddress + ) + } + + /** + * Publish a new DDO + * @param {String} did + * @param {DDO} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async publish( + did: string, + ddo: DDO, + consumerAccount: string + ): Promise { + let flags = 0 + const data = DDO.serialize(ddo) + const lzma = new LZMA() + // see https://github.com/LZMA-JS/LZMA-JS/issues/44 + lzma.disableEndMark = true + let compressed = lzma.compress(data, 9) + compressed = this.getHex(compressed) + flags = flags | 1 + return this.publishRaw(didZeroX(did), flags, compressed, consumerAccount) + } + + /** + * Update DDO + * @param {String} did + * @param {DDO} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async update( + did: string, + ddo: DDO, + consumerAccount: string + ): Promise { + let flags = 0 + const data = DDO.serialize(ddo) + const lzma = new LZMA() + // see https://github.com/LZMA-JS/LZMA-JS/issues/44 + lzma.disableEndMark = true + let compressed = lzma.compress(data, 9) + compressed = this.getHex(compressed) + flags = flags | 1 + return this.updateRaw(didZeroX(did), flags, compressed, consumerAccount) + } + + /** + * Raw publish ddo + * @param {String} did + * @param {Any} flags + * @param {Any} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async publishRaw( + did: string, + flags: any, + data: any, + consumerAccount: string + ): Promise { + if (!this.DDOContract) { + console.error('Missing DDOContract') + return null + } + try { + // const data = this.web3.utils.bytesToHex(ddo) + const estGas = await this.DDOContract.methods + .create(did, flags, data) + .estimateGas(function (err, estGas) { + if (err) console.log('OnChainMetadataStore: ' + err) + return estGas + }) + const trxReceipt = await this.DDOContract.methods + .create(did, flags, data) + .send({ from: consumerAccount, gas: estGas + 1 }) + return trxReceipt + } catch (e) { + console.error(e) + return null + } + } + + /** + * Raw update of a ddo + * @param {String} did + * @param {Any} flags + * @param {Any} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async updateRaw( + did: string, + flags: any, + ddo: any, + consumerAccount: string + ): Promise { + if (!this.DDOContract) { + console.error('Missing DDOContract') + return null + } + try { + const data = this.web3.utils.bytesToHex(ddo) + const estGas = await this.DDOContract.methods + .update(did, flags, data) + .estimateGas(function (err, estGas) { + if (err) console.log('OnChainMetadataStore: ' + err) + return estGas + }) + const trxReceipt = await this.DDOContract.methods + .update(did, flags, data) + .send({ from: consumerAccount, gas: estGas + 1 }) + return trxReceipt + } catch (e) { + console.error(e) + return null + } + } + + /** + * Transfer Ownership of a DDO + * @param {String} did + * @param {String} newOwner + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async transferOwnership( + did: string, + newOwner: string, + consumerAccount: string + ): Promise { + if (!this.DDOContract) return null + try { + const trxReceipt = await this.DDOContract.methods + .transferOwnership(didZeroX(did), newOwner) + .send({ + from: consumerAccount + }) + return trxReceipt + } catch (e) { + console.error(e) + return null + } + } + + public getHex(message: any) { + const hexChar = [ + '0', + '1', + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + 'A', + 'B', + 'C', + 'D', + 'E', + 'F' + ] + let hex = '' + try { + for (let i = 0; i < message.length; i++) { + hex += '' + hexChar[(message[i] >> 4) & 0x0f] + hexChar[message[i] & 0x0f] + } + } catch (e) { + console.error(e) + } + const hexMessage = '0x' + hex + return hexMessage + } +} diff --git a/src/models/Config.ts b/src/models/Config.ts index 04a66e45..c09140fc 100644 --- a/src/models/Config.ts +++ b/src/models/Config.ts @@ -85,6 +85,17 @@ export class Config { * @type {any} */ public fixedRateExchangeAddressABI?: AbiItem | AbiItem[] + /** + * DDOContractAddress + * @type {string} + */ + public DDOContractAddress?: string + + /** + * DDOContractABI + * @type {any} + */ + public DDOContractABI?: AbiItem | AbiItem[] /** * Log level. * @type {boolean | LogLevel} diff --git a/src/ocean/Assets.ts b/src/ocean/Assets.ts index 8181c7b0..41fbb789 100644 --- a/src/ocean/Assets.ts +++ b/src/ocean/Assets.ts @@ -161,10 +161,16 @@ export class Assets extends Instantiable { observer.next(CreateProgressStep.ProofGenerated) this.logger.log('Storing DDO') observer.next(CreateProgressStep.StoringDdo) - const storedDdo = await this.ocean.metadatastore.storeDDO(ddo) + // const storedDdo = await this.ocean.metadatastore.storeDDO(ddo) + const storeTx = await this.ocean.OnChainMetadataStore.publish( + ddo.id, + ddo, + publisher.getId() + ) this.logger.log('DDO stored') observer.next(CreateProgressStep.DdoStored) - return storedDdo + if (storeTx) return ddo + else return null }) } diff --git a/src/ocean/Ocean.ts b/src/ocean/Ocean.ts index c6e6fd7f..b92febd9 100644 --- a/src/ocean/Ocean.ts +++ b/src/ocean/Ocean.ts @@ -3,6 +3,7 @@ import { Assets } from './Assets' import { Versions } from './Versions' import { OceanUtils } from './utils/Utils' import { MetadataStore } from '../metadatastore/MetadataStore' +import { OnChainMetadataStore } from '../metadatastore/OnChainMetaData' import { Provider } from '../provider/Provider' import { DataTokens } from '../datatokens/Datatokens' import { Network } from '../datatokens/Network' @@ -64,6 +65,11 @@ export class Ocean extends Instantiable { instanceConfig.config.fixedRateExchangeAddressABI, instanceConfig.config.oceanTokenAddress ) + instance.OnChainMetadataStore = new OnChainMetadataStore( + instanceConfig.config.web3Provider, + instanceConfig.config.DDOContractAddress, + instanceConfig.config.DDOContractABI + ) instance.versions = await Versions.getInstance(instanceConfig) instance.network = new Network() return instance @@ -92,7 +98,11 @@ export class Ocean extends Instantiable { * @type {MetadataStore} */ public metadatastore: MetadataStore - + /** + * OnChainMetadataStore instance. + * @type {OnChainMetadataStore} + */ + public OnChainMetadataStore: OnChainMetadataStore /** * Ocean account submodule * @type {Accounts} diff --git a/src/utils/ConfigHelper.ts b/src/utils/ConfigHelper.ts index 73cb6322..d1df332c 100644 --- a/src/utils/ConfigHelper.ts +++ b/src/utils/ConfigHelper.ts @@ -1,5 +1,6 @@ import Config from '../models/Config' import { Logger } from '../lib' +import fs from 'fs' export declare type ConfigHelperNetworkName = | 'mainnet' @@ -23,7 +24,8 @@ const configs: ConfigHelperConfig[] = [ metadataStoreUri: 'http://127.0.0.1:5000', providerUri: 'http://127.0.0.1:8030', poolFactoryAddress: null, - fixedRateExchangeAddress: null + fixedRateExchangeAddress: null, + DDOContractAddress: null }, { chainId: 4, @@ -34,7 +36,8 @@ const configs: ConfigHelperConfig[] = [ metadataStoreUri: 'https://aquarius.rinkeby.v3.dev-ocean.com', providerUri: 'https://provider.rinkeby.v3.dev-ocean.com', poolFactoryAddress: '0x9B90A1358fbeEC1C4bB1DA7D4E85C708f87556Ec', - fixedRateExchangeAddress: '0x991c08bD00761A299d3126a81a985329096896D4' + fixedRateExchangeAddress: '0x991c08bD00761A299d3126a81a985329096896D4', + DDOContractAddress: '0xEfA25E39192b3175d451D79C1c0a41Fa3C32c87d' }, { chainId: 1, @@ -45,15 +48,34 @@ const configs: ConfigHelperConfig[] = [ metadataStoreUri: null, providerUri: null, poolFactoryAddress: null, - fixedRateExchangeAddress: null + fixedRateExchangeAddress: null, + DDOContractAddress: null } ] export class ConfigHelper { + /* Load config from env ADDRESS_FILE (generated by ocean-contracts) */ + public loadAddressesFromEnv() { + try { + const data = JSON.parse(fs.readFileSync(process.env.ADDRESS_FILE, 'utf8')) + if (data) { + if (data.ganache) { + if (data.ganache.DTFactory) configs[0].factoryAddress = data.ganache.DTFactory + if (data.ganache.BFactory) configs[0].poolFactoryAddress = data.ganache.BFactory + if (data.ganache.FixedRateExchange) + configs[0].fixedRateExchangeAddress = data.ganache.FixedRateExchange + if (data.ganache.DDO) configs[0].DDOContractAddress = data.ganache.DDO + } + } + if (process.env.AQUARIUS_URI) configs[0].metadataStoreUri = process.env.AQUARIUS_URI + } catch (e) {} + } + public getConfig( network: ConfigHelperNetworkName | ConfigHelperNetworkId, infuraProjectId?: string ): Config { + if (network === 'development') this.loadAddressesFromEnv() const filterBy = typeof network === 'string' ? 'network' : 'chainId' const config = configs.find((c) => c[filterBy] === network) diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 1e390f10..c87c0735 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -2,7 +2,8 @@ import { AbiItem } from 'web3-utils/types' import { TestContractHandler } from '../TestContractHandler' import { DataTokens } from '../../src/datatokens/Datatokens' import { Ocean } from '../../src/ocean/Ocean' -import config from './config' +import { ConfigHelper } from '../../src/utils/ConfigHelper' + import { assert } from 'console' import { ServiceComputePrivacy } from '../../src/ddo/interfaces/Service' import Web3 from 'web3' @@ -61,7 +62,8 @@ describe('Compute flow', () => { factory.bytecode, web3 ) - + const config = new ConfigHelper().getConfig('development') + config.web3Provider = web3 ocean = await Ocean.getInstance(config) owner = (await ocean.accounts.list())[0] alice = (await ocean.accounts.list())[1] diff --git a/test/integration/Marketplaceflow.test.ts b/test/integration/Marketplaceflow.test.ts index 0c50351f..91c683dc 100644 --- a/test/integration/Marketplaceflow.test.ts +++ b/test/integration/Marketplaceflow.test.ts @@ -2,7 +2,8 @@ import { AbiItem } from 'web3-utils/types' import { TestContractHandler } from '../TestContractHandler' import { DataTokens } from '../../src/datatokens/Datatokens' import { Ocean } from '../../src/ocean/Ocean' -import config from './config' +import { ConfigHelper } from '../../src/utils/ConfigHelper' +// import config from './config' import { assert } from 'console' import Web3 from 'web3' @@ -38,7 +39,8 @@ describe('Marketplace flow', () => { factory.bytecode, web3 ) - + const config = new ConfigHelper().getConfig('development') + config.web3Provider = web3 ocean = await Ocean.getInstance(config) owner = (await ocean.accounts.list())[0] alice = (await ocean.accounts.list())[1]