diff --git a/.travis.yml b/.travis.yml index 201312d5..aaf87cac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,14 @@ before_script: # Barge setup - git clone https://github.com/oceanprotocol/barge - cd barge - - git checkout v3 + - git checkout feature/ocean-contracts - export PROVIDER_VERSION=v0.3.0 + - export ADDRESS_FILE="${HOME}/.ocean/ocean-contracts/artifacts/address.json" + - export AQUARIUS_URI="http://172.15.0.5:5000" + - export DEPLOY_CONTRACTS=true - bash -x start_ocean.sh --no-dashboard 2>&1 > start_ocean.log & - cd .. - - sleep 300 + - ./scripts/waitforcontracts.sh script: - npm test diff --git a/package-lock.json b/package-lock.json index ebcb1d3a..dfae268c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6534,6 +6534,11 @@ "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", "dev": true }, + "lzma": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/lzma/-/lzma-2.3.2.tgz", + "integrity": "sha1-N4OySFi5wOdHoN88vx+1/KqSxEE=" + }, "macos-release": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.1.tgz", diff --git a/package.json b/package.json index 31161754..612b0732 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@oceanprotocol/contracts": "^0.4.2", "decimal.js": "^10.2.0", "fs": "0.0.1-security", + "lzma": "^2.3.2", "node-fetch": "^2.6.1", "save-file": "^2.3.1", "uuid": "^8.3.0", diff --git a/scripts/waitforcontracts.sh b/scripts/waitforcontracts.sh new file mode 100755 index 00000000..eef49198 --- /dev/null +++ b/scripts/waitforcontracts.sh @@ -0,0 +1,5 @@ +if [ "${DEPLOY_CONTRACTS}" = "true" ]; then + while [ ! -f "${HOME}/.ocean/ocean-contracts/artifacts/ready" ]; do + sleep 2 + done +fi diff --git a/src/ddo/interfaces/ServicePrices.ts b/src/ddo/interfaces/ServicePrices.ts index 1e632198..0c85ccaa 100644 --- a/src/ddo/interfaces/ServicePrices.ts +++ b/src/ddo/interfaces/ServicePrices.ts @@ -1,4 +1,4 @@ export interface ServicePrices { serviceIndex: number - price: string + cost: string } diff --git a/src/metadatastore/OnChainMetaData.ts b/src/metadatastore/OnChainMetaData.ts new file mode 100644 index 00000000..85ca016f --- /dev/null +++ b/src/metadatastore/OnChainMetaData.ts @@ -0,0 +1,206 @@ +import { DDO } from '../ddo/DDO' +import { TransactionReceipt } from 'web3-core' +import { Contract } from 'web3-eth-contract' +import { AbiItem } from 'web3-utils/types' +import Web3 from 'web3' +import defaultDDOContractABI from '@oceanprotocol/contracts/artifacts/DDO.json' +import { didZeroX } from '../utils' +import { LZMA } from 'lzma' + +/** + * Provides an interface with Metadata Store. + * Metadata Store provides an off-chain database store for metadata about data assets. + */ +export class OnChainMetadataStore { + public DDOContractAddress: string + public DDOContractABI: AbiItem | AbiItem[] + public web3: Web3 + public DDOContract: Contract = null + /** + * Instantiate OnChainMetadata Store for on-chain interaction. + */ + constructor( + web3: Web3, + DDOContractAddress: string = null, + DDOContractABI: AbiItem | AbiItem[] = null + ) { + this.web3 = web3 + this.DDOContractAddress = DDOContractAddress + this.DDOContractABI = DDOContractABI || (defaultDDOContractABI.abi as AbiItem[]) + if (web3) + this.DDOContract = new this.web3.eth.Contract( + this.DDOContractABI, + this.DDOContractAddress + ) + } + + /** Compress DDO using LZMA + * + */ + public async LZMACompressDDO(ddo: DDO): Promise { + const data = DDO.serialize(ddo) + const lzma = new LZMA() + // see https://github.com/LZMA-JS/LZMA-JS/issues/44 + lzma.disableEndMark = true + let compressed = lzma.compress(data, 9) + compressed = this.getHex(compressed) + return compressed + } + + /** + * Publish a new DDO + * @param {String} did + * @param {DDO} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async publish( + did: string, + ddo: DDO, + consumerAccount: string + ): Promise { + let flags = 0 + const compressed = await this.LZMACompressDDO(ddo) + flags = flags | 1 + return this.publishRaw(didZeroX(did), flags, compressed, consumerAccount) + } + + /** + * Update DDO + * @param {String} did + * @param {DDO} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async update( + did: string, + ddo: DDO, + consumerAccount: string + ): Promise { + let flags = 0 + const compressed = await this.LZMACompressDDO(ddo) + flags = flags | 1 + return this.updateRaw(didZeroX(did), flags, compressed, consumerAccount) + } + + /** + * Raw publish ddo + * @param {String} did + * @param {Any} flags + * @param {Any} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async publishRaw( + did: string, + flags: any, + data: any, + consumerAccount: string + ): Promise { + if (!this.DDOContract) { + console.error('Missing DDOContract') + return null + } + try { + const estGas = await this.DDOContract.methods + .create(did, flags, data) + .estimateGas(function (err, estGas) { + if (err) console.log('OnChainMetadataStore: ' + err) + return estGas + }) + const trxReceipt = await this.DDOContract.methods + .create(did, flags, data) + .send({ from: consumerAccount, gas: estGas + 1 }) + return trxReceipt + } catch (e) { + console.error(e) + return null + } + } + + /** + * Raw update of a ddo + * @param {String} did + * @param {Any} flags + * @param {Any} ddo + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async updateRaw( + did: string, + flags: any, + data: any, + consumerAccount: string + ): Promise { + if (!this.DDOContract) { + console.error('Missing DDOContract') + return null + } + try { + const trxReceipt = await this.DDOContract.methods + .update(did, flags, data) + .send({ from: consumerAccount }) + return trxReceipt + } catch (e) { + console.error(e) + return null + } + } + + /** + * Transfer Ownership of a DDO + * @param {String} did + * @param {String} newOwner + * @param {String} consumerAccount + * @return {Promise} exchangeId + */ + public async transferOwnership( + did: string, + newOwner: string, + consumerAccount: string + ): Promise { + if (!this.DDOContract) return null + try { + const trxReceipt = await this.DDOContract.methods + .transferOwnership(didZeroX(did), newOwner) + .send({ + from: consumerAccount + }) + return trxReceipt + } catch (e) { + console.error(e) + return null + } + } + + public getHex(message: any) { + const hexChar = [ + '0', + '1', + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + 'A', + 'B', + 'C', + 'D', + 'E', + 'F' + ] + let hex = '' + try { + for (let i = 0; i < message.length; i++) { + hex += '' + hexChar[(message[i] >> 4) & 0x0f] + hexChar[message[i] & 0x0f] + } + } catch (e) { + console.error(e) + } + const hexMessage = '0x' + hex + return hexMessage + } +} diff --git a/src/models/Config.ts b/src/models/Config.ts index 04a66e45..c09140fc 100644 --- a/src/models/Config.ts +++ b/src/models/Config.ts @@ -85,6 +85,17 @@ export class Config { * @type {any} */ public fixedRateExchangeAddressABI?: AbiItem | AbiItem[] + /** + * DDOContractAddress + * @type {string} + */ + public DDOContractAddress?: string + + /** + * DDOContractABI + * @type {any} + */ + public DDOContractABI?: AbiItem | AbiItem[] /** * Log level. * @type {boolean | LogLevel} diff --git a/src/ocean/Assets.ts b/src/ocean/Assets.ts index 2b023041..27332382 100644 --- a/src/ocean/Assets.ts +++ b/src/ocean/Assets.ts @@ -163,10 +163,16 @@ export class Assets extends Instantiable { observer.next(CreateProgressStep.ProofGenerated) this.logger.log('Storing DDO') observer.next(CreateProgressStep.StoringDdo) - const storedDdo = await this.ocean.metadatastore.storeDDO(ddo) - this.logger.log('DDO stored') + // const storedDdo = await this.ocean.metadatastore.storeDDO(ddo) + const storeTx = await this.ocean.OnChainMetadataStore.publish( + ddo.id, + ddo, + publisher.getId() + ) + this.logger.log('DDO stored ' + ddo.id) observer.next(CreateProgressStep.DdoStored) - return storedDdo + if (storeTx) return ddo + else return null }) } @@ -219,23 +225,39 @@ export class Assets extends Instantiable { did: string, newMetadata: EditableMetadata, account: Account - ): Promise { + ): Promise { const oldDdo = await this.ocean.metadatastore.retrieveDDO(did) - // get a signature - const signature = await this.ocean.utils.signature.signForAquarius( - oldDdo.updated, - account + let i + for (i = 0; i < oldDdo.service.length; i++) { + if (oldDdo.service[i].type === 'metadata') { + if (newMetadata.title) oldDdo.service[i].attributes.main.name = newMetadata.title + if (!oldDdo.service[i].attributes.additionalInformation) + oldDdo.service[i].attributes.additionalInformation = Object() + if (newMetadata.description) + oldDdo.service[i].attributes.additionalInformation.description = + newMetadata.description + if (newMetadata.links) + oldDdo.service[i].attributes.additionalInformation.links = newMetadata.links + } + } + if (newMetadata.servicePrices) { + for (i = 0; i < newMetadata.servicePrices.length; i++) { + if ( + newMetadata.servicePrices[i].cost && + newMetadata.servicePrices[i].serviceIndex + ) { + oldDdo.service[newMetadata.servicePrices[i].serviceIndex].attributes.main.cost = + newMetadata.servicePrices[i].cost + } + } + } + const storeTx = await this.ocean.OnChainMetadataStore.update( + oldDdo.id, + oldDdo, + account.getId() ) - let result = null - if (signature != null) - result = await this.ocean.metadatastore.editMetadata( - did, - newMetadata, - oldDdo.updated, - signature - ) - - return result + if (storeTx) return oldDdo + else return null } /** @@ -251,45 +273,22 @@ export class Assets extends Instantiable { serviceIndex: number, computePrivacy: ServiceComputePrivacy, account: Account - ): Promise { + ): Promise { const oldDdo = await this.ocean.metadatastore.retrieveDDO(did) - // get a signature - const signature = await this.ocean.utils.signature.signForAquarius( - oldDdo.updated, - account + if (oldDdo.service[serviceIndex].type !== 'compute') return null + oldDdo.service[serviceIndex].attributes.main.privacy.allowRawAlgorithm = + computePrivacy.allowRawAlgorithm + oldDdo.service[serviceIndex].attributes.main.privacy.allowNetworkAccess = + computePrivacy.allowNetworkAccess + oldDdo.service[serviceIndex].attributes.main.privacy.trustedAlgorithms = + computePrivacy.trustedAlgorithms + const storeTx = await this.ocean.OnChainMetadataStore.update( + oldDdo.id, + oldDdo, + account.getId() ) - let result = null - if (signature != null) - result = await this.ocean.metadatastore.updateComputePrivacy( - did, - serviceIndex, - computePrivacy.allowRawAlgorithm, - computePrivacy.allowNetworkAccess, - computePrivacy.trustedAlgorithms, - oldDdo.updated, - signature - ) - - return result - } - - /** - * Retire a DDO (Delete) - * @param {did} string DID. - * @param {Account} account Ethereum account of owner to sign and prove the ownership. - * @return {Promise} - */ - public async retire(did: string, account: Account): Promise { - const oldDdo = await this.ocean.metadatastore.retrieveDDO(did) - // get a signature - const signature = await this.ocean.utils.signature.signForAquarius( - oldDdo.updated, - account - ) - let result = null - if (signature != null) - result = await this.ocean.metadatastore.retire(did, oldDdo.updated, signature) - return result + if (storeTx) return oldDdo + else return null } /** diff --git a/src/ocean/Ocean.ts b/src/ocean/Ocean.ts index c6e6fd7f..b92febd9 100644 --- a/src/ocean/Ocean.ts +++ b/src/ocean/Ocean.ts @@ -3,6 +3,7 @@ import { Assets } from './Assets' import { Versions } from './Versions' import { OceanUtils } from './utils/Utils' import { MetadataStore } from '../metadatastore/MetadataStore' +import { OnChainMetadataStore } from '../metadatastore/OnChainMetaData' import { Provider } from '../provider/Provider' import { DataTokens } from '../datatokens/Datatokens' import { Network } from '../datatokens/Network' @@ -64,6 +65,11 @@ export class Ocean extends Instantiable { instanceConfig.config.fixedRateExchangeAddressABI, instanceConfig.config.oceanTokenAddress ) + instance.OnChainMetadataStore = new OnChainMetadataStore( + instanceConfig.config.web3Provider, + instanceConfig.config.DDOContractAddress, + instanceConfig.config.DDOContractABI + ) instance.versions = await Versions.getInstance(instanceConfig) instance.network = new Network() return instance @@ -92,7 +98,11 @@ export class Ocean extends Instantiable { * @type {MetadataStore} */ public metadatastore: MetadataStore - + /** + * OnChainMetadataStore instance. + * @type {OnChainMetadataStore} + */ + public OnChainMetadataStore: OnChainMetadataStore /** * Ocean account submodule * @type {Accounts} diff --git a/src/utils/ConfigHelper.ts b/src/utils/ConfigHelper.ts index 73cb6322..d1df332c 100644 --- a/src/utils/ConfigHelper.ts +++ b/src/utils/ConfigHelper.ts @@ -1,5 +1,6 @@ import Config from '../models/Config' import { Logger } from '../lib' +import fs from 'fs' export declare type ConfigHelperNetworkName = | 'mainnet' @@ -23,7 +24,8 @@ const configs: ConfigHelperConfig[] = [ metadataStoreUri: 'http://127.0.0.1:5000', providerUri: 'http://127.0.0.1:8030', poolFactoryAddress: null, - fixedRateExchangeAddress: null + fixedRateExchangeAddress: null, + DDOContractAddress: null }, { chainId: 4, @@ -34,7 +36,8 @@ const configs: ConfigHelperConfig[] = [ metadataStoreUri: 'https://aquarius.rinkeby.v3.dev-ocean.com', providerUri: 'https://provider.rinkeby.v3.dev-ocean.com', poolFactoryAddress: '0x9B90A1358fbeEC1C4bB1DA7D4E85C708f87556Ec', - fixedRateExchangeAddress: '0x991c08bD00761A299d3126a81a985329096896D4' + fixedRateExchangeAddress: '0x991c08bD00761A299d3126a81a985329096896D4', + DDOContractAddress: '0xEfA25E39192b3175d451D79C1c0a41Fa3C32c87d' }, { chainId: 1, @@ -45,15 +48,34 @@ const configs: ConfigHelperConfig[] = [ metadataStoreUri: null, providerUri: null, poolFactoryAddress: null, - fixedRateExchangeAddress: null + fixedRateExchangeAddress: null, + DDOContractAddress: null } ] export class ConfigHelper { + /* Load config from env ADDRESS_FILE (generated by ocean-contracts) */ + public loadAddressesFromEnv() { + try { + const data = JSON.parse(fs.readFileSync(process.env.ADDRESS_FILE, 'utf8')) + if (data) { + if (data.ganache) { + if (data.ganache.DTFactory) configs[0].factoryAddress = data.ganache.DTFactory + if (data.ganache.BFactory) configs[0].poolFactoryAddress = data.ganache.BFactory + if (data.ganache.FixedRateExchange) + configs[0].fixedRateExchangeAddress = data.ganache.FixedRateExchange + if (data.ganache.DDO) configs[0].DDOContractAddress = data.ganache.DDO + } + } + if (process.env.AQUARIUS_URI) configs[0].metadataStoreUri = process.env.AQUARIUS_URI + } catch (e) {} + } + public getConfig( network: ConfigHelperNetworkName | ConfigHelperNetworkId, infuraProjectId?: string ): Config { + if (network === 'development') this.loadAddressesFromEnv() const filterBy = typeof network === 'string' ? 'network' : 'chainId' const config = configs.find((c) => c[filterBy] === network) diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 533a500c..eeb6632f 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -2,7 +2,8 @@ import { AbiItem } from 'web3-utils/types' import { TestContractHandler } from '../TestContractHandler' import { DataTokens } from '../../src/datatokens/Datatokens' import { Ocean } from '../../src/ocean/Ocean' -import config from './config' +import { ConfigHelper } from '../../src/utils/ConfigHelper' + import { assert } from 'console' import { ServiceComputePrivacy } from '../../src/ddo/interfaces/Service' import Web3 from 'web3' @@ -10,6 +11,12 @@ import factory from '@oceanprotocol/contracts/artifacts/DTFactory.json' import datatokensTemplate from '@oceanprotocol/contracts/artifacts/DataTokenTemplate.json' const web3 = new Web3('http://127.0.0.1:8545') +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms) + }) +} + describe('Compute flow', () => { let owner let bob @@ -61,7 +68,8 @@ describe('Compute flow', () => { factory.bytecode, web3 ) - + const config = new ConfigHelper().getConfig('development') + config.web3Provider = web3 ocean = await Ocean.getInstance(config) owner = (await ocean.accounts.list())[0] alice = (await ocean.accounts.list())[1] @@ -154,6 +162,7 @@ describe('Compute flow', () => { ) ddo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) assert(ddo.dataToken === tokenAddress) + await sleep(6000) }) // alex @@ -178,6 +187,7 @@ describe('Compute flow', () => { tokenAddress ) assert(datasetNoRawAlgo.dataToken === tokenAddress) + await sleep(6000) }) it('should publish a dataset with a compute service object that allows only algo with did:op:1234', async () => { @@ -201,6 +211,7 @@ describe('Compute flow', () => { tokenAddress ) assert(datasetWithTrustedAlgo.dataToken === tokenAddress) + await sleep(6000) }) it('should publish an algorithm', async () => { @@ -239,6 +250,7 @@ describe('Compute flow', () => { ) algorithmAsset = await ocean.assets.create(algoAsset, alice, [service1], tokenAddress) assert(algorithmAsset.dataToken === tokenAddress) + await sleep(6000) }) it('Alice mints 100 DTs and tranfers them to the compute marketplace', async () => { @@ -358,6 +370,42 @@ describe('Compute flow', () => { jobId = response.jobId assert(response.status >= 10) }) + it('Alice updates Compute Privacy', async () => { + const newComputePrivacy = { + allowRawAlgorithm: false, + allowNetworkAccess: true, + trustedAlgorithms: ['did:op:1234', 'did:op:1235'] + } + let computeIndex = 0 + for (let i = 0; i < ddo.service.length; i++) { + if (ddo.service[i].type === 'compute') { + computeIndex = i + break + } + } + assert(computeIndex > 0) + const newDdo = await ocean.assets.updateComputePrivacy( + ddo.id, + computeIndex, + newComputePrivacy, + alice + ) + assert(newDdo !== null) + await sleep(6000) + const metaData = await ocean.assets.getServiceByType(ddo.id, 'compute') + assert( + metaData.attributes.main.privacy.allowRawAlgorithm === + newComputePrivacy.allowRawAlgorithm + ) + assert( + metaData.attributes.main.privacy.allowNetworkAccess === + newComputePrivacy.allowNetworkAccess + ) + assert( + metaData.attributes.main.privacy.trustedAlgorithms === + newComputePrivacy.trustedAlgorithms + ) + }) // it('Bob restarts compute job', async () => {}) // it('Bob gets outputs', async () => {}) diff --git a/test/integration/Marketplaceflow.test.ts b/test/integration/Marketplaceflow.test.ts index a998b743..7942772e 100644 --- a/test/integration/Marketplaceflow.test.ts +++ b/test/integration/Marketplaceflow.test.ts @@ -2,14 +2,23 @@ import { AbiItem } from 'web3-utils/types' import { TestContractHandler } from '../TestContractHandler' import { DataTokens } from '../../src/datatokens/Datatokens' import { Ocean } from '../../src/ocean/Ocean' -import config from './config' +import { ConfigHelper } from '../../src/utils/ConfigHelper' + +// import config from './config' import { assert } from 'console' import Web3 from 'web3' import factory from '@oceanprotocol/contracts/artifacts/DTFactory.json' import datatokensTemplate from '@oceanprotocol/contracts/artifacts/DataTokenTemplate.json' +import { EditableMetadata } from '../../src/lib' const web3 = new Web3('http://127.0.0.1:8545') +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms) + }) +} + describe('Marketplace flow', () => { let owner let bob @@ -38,7 +47,8 @@ describe('Marketplace flow', () => { factory.bytecode, web3 ) - + const config = new ConfigHelper().getConfig('development') + config.web3Provider = web3 ocean = await Ocean.getInstance(config) owner = (await ocean.accounts.list())[0] alice = (await ocean.accounts.list())[1] @@ -101,6 +111,7 @@ describe('Marketplace flow', () => { ) ddo = await ocean.assets.create(asset, alice, [service1], tokenAddress) assert(ddo.dataToken === tokenAddress) + await sleep(6000) }) it('Alice mints 100 tokens', async () => { @@ -190,4 +201,20 @@ describe('Marketplace flow', () => { const assets = await ocean.assets.ownerAssets(alice.getId()) assert(assets.length > 0) }) + it('Alice updates metadata', async () => { + const newMetaData: EditableMetadata = { + description: 'new description', + title: 'new title', + links: [{ name: 'link1', type: 'sample', url: 'http://example.net' }] + } + const newDdo = await ocean.assets.editMetadata(ddo.id, newMetaData, alice) + assert(newDdo !== null) + await sleep(6000) + const metaData = await ocean.assets.getServiceByType(ddo.id, 'metadata') + assert(metaData.attributes.main.name === newMetaData.title) + assert( + metaData.attributes.additionalInformation.description === newMetaData.description + ) + assert(metaData.attributes.additionalInformation.links === newMetaData.links) + }) })