Merge pull request #287 from oceanprotocol/feature/decentralized_DDO

Feature/decentralized ddo  (based on feature/StartOrder)

.travis.yml

@@ -19,11 +19,14 @@ before_script:
   # Barge setup
   - git clone https://github.com/oceanprotocol/barge
   - cd barge
-  - git checkout v3
+  - git checkout feature/ocean-contracts
   - export PROVIDER_VERSION=v0.3.0
+  - export ADDRESS_FILE="${HOME}/.ocean/ocean-contracts/artifacts/address.json"
+  - export AQUARIUS_URI="http://172.15.0.5:5000"
+  - export DEPLOY_CONTRACTS=true
   - bash -x start_ocean.sh --no-dashboard 2>&1 > start_ocean.log &
   - cd ..
-  - sleep 300
+  - ./scripts/waitforcontracts.sh
 
 script:
   - npm test

package-lock.json

@@ -6534,6 +6534,11 @@
       "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==",
       "dev": true
     },
+    "lzma": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/lzma/-/lzma-2.3.2.tgz",
+      "integrity": "sha1-N4OySFi5wOdHoN88vx+1/KqSxEE="
+    },
     "macos-release": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.1.tgz",

package.json

@@ -45,6 +45,7 @@
     "@oceanprotocol/contracts": "^0.4.2",
     "decimal.js": "^10.2.0",
     "fs": "0.0.1-security",
+    "lzma": "^2.3.2",
     "node-fetch": "^2.6.1",
     "save-file": "^2.3.1",
     "uuid": "^8.3.0",

scripts/waitforcontracts.sh

@@ -0,0 +1,5 @@
+if [ "${DEPLOY_CONTRACTS}" = "true" ]; then
+  while [ ! -f "${HOME}/.ocean/ocean-contracts/artifacts/ready" ]; do
+    sleep 2
+  done
+fi

src/ddo/interfaces/ServicePrices.ts

@@ -1,4 +1,4 @@
 export interface ServicePrices {
   serviceIndex: number
-  price: string
+  cost: string
 }

src/metadatastore/OnChainMetaData.ts

@@ -0,0 +1,206 @@
+import { DDO } from '../ddo/DDO'
+import { TransactionReceipt } from 'web3-core'
+import { Contract } from 'web3-eth-contract'
+import { AbiItem } from 'web3-utils/types'
+import Web3 from 'web3'
+import defaultDDOContractABI from '@oceanprotocol/contracts/artifacts/DDO.json'
+import { didZeroX } from '../utils'
+import { LZMA } from 'lzma'
+
+/**
+ * Provides an interface with Metadata Store.
+ * Metadata Store provides an off-chain database store for metadata about data assets.
+ */
+export class OnChainMetadataStore {
+  public DDOContractAddress: string
+  public DDOContractABI: AbiItem | AbiItem[]
+  public web3: Web3
+  public DDOContract: Contract = null
+  /**
+   * Instantiate OnChainMetadata Store for on-chain interaction.
+   */
+  constructor(
+    web3: Web3,
+    DDOContractAddress: string = null,
+    DDOContractABI: AbiItem | AbiItem[] = null
+  ) {
+    this.web3 = web3
+    this.DDOContractAddress = DDOContractAddress
+    this.DDOContractABI = DDOContractABI || (defaultDDOContractABI.abi as AbiItem[])
+    if (web3)
+      this.DDOContract = new this.web3.eth.Contract(
+        this.DDOContractABI,
+        this.DDOContractAddress
+      )
+  }
+
+  /** Compress DDO using LZMA
+   *
+   */
+  public async LZMACompressDDO(ddo: DDO): Promise<any> {
+    const data = DDO.serialize(ddo)
+    const lzma = new LZMA()
+    // see https://github.com/LZMA-JS/LZMA-JS/issues/44
+    lzma.disableEndMark = true
+    let compressed = lzma.compress(data, 9)
+    compressed = this.getHex(compressed)
+    return compressed
+  }
+
+  /**
+   * Publish a new DDO
+   * @param {String} did
+   * @param {DDO} ddo
+   * @param {String} consumerAccount
+   * @return {Promise<TransactionReceipt>} exchangeId
+   */
+  public async publish(
+    did: string,
+    ddo: DDO,
+    consumerAccount: string
+  ): Promise<TransactionReceipt> {
+    let flags = 0
+    const compressed = await this.LZMACompressDDO(ddo)
+    flags = flags | 1
+    return this.publishRaw(didZeroX(did), flags, compressed, consumerAccount)
+  }
+
+  /**
+   * Update DDO
+   * @param {String} did
+   * @param {DDO} ddo
+   * @param {String} consumerAccount
+   * @return {Promise<TransactionReceipt>} exchangeId
+   */
+  public async update(
+    did: string,
+    ddo: DDO,
+    consumerAccount: string
+  ): Promise<TransactionReceipt> {
+    let flags = 0
+    const compressed = await this.LZMACompressDDO(ddo)
+    flags = flags | 1
+    return this.updateRaw(didZeroX(did), flags, compressed, consumerAccount)
+  }
+
+  /**
+   * Raw publish ddo
+   * @param {String} did
+   * @param {Any} flags
+   * @param {Any} ddo
+   * @param {String} consumerAccount
+   * @return {Promise<TransactionReceipt>} exchangeId
+   */
+  public async publishRaw(
+    did: string,
+    flags: any,
+    data: any,
+    consumerAccount: string
+  ): Promise<TransactionReceipt> {
+    if (!this.DDOContract) {
+      console.error('Missing DDOContract')
+      return null
+    }
+    try {
+      const estGas = await this.DDOContract.methods
+        .create(did, flags, data)
+        .estimateGas(function (err, estGas) {
+          if (err) console.log('OnChainMetadataStore: ' + err)
+          return estGas
+        })
+      const trxReceipt = await this.DDOContract.methods
+        .create(did, flags, data)
+        .send({ from: consumerAccount, gas: estGas + 1 })
+      return trxReceipt
+    } catch (e) {
+      console.error(e)
+      return null
+    }
+  }
+
+  /**
+   * Raw update of a ddo
+   * @param {String} did
+   * @param {Any} flags
+   * @param {Any} ddo
+   * @param {String} consumerAccount
+   * @return {Promise<TransactionReceipt>} exchangeId
+   */
+  public async updateRaw(
+    did: string,
+    flags: any,
+    data: any,
+    consumerAccount: string
+  ): Promise<TransactionReceipt> {
+    if (!this.DDOContract) {
+      console.error('Missing DDOContract')
+      return null
+    }
+    try {
+      const trxReceipt = await this.DDOContract.methods
+        .update(did, flags, data)
+        .send({ from: consumerAccount })
+      return trxReceipt
+    } catch (e) {
+      console.error(e)
+      return null
+    }
+  }
+
+  /**
+   * Transfer Ownership of a DDO
+   * @param {String} did
+   * @param {String} newOwner
+   * @param {String} consumerAccount
+   * @return {Promise<TransactionReceipt>} exchangeId
+   */
+  public async transferOwnership(
+    did: string,
+    newOwner: string,
+    consumerAccount: string
+  ): Promise<TransactionReceipt> {
+    if (!this.DDOContract) return null
+    try {
+      const trxReceipt = await this.DDOContract.methods
+        .transferOwnership(didZeroX(did), newOwner)
+        .send({
+          from: consumerAccount
+        })
+      return trxReceipt
+    } catch (e) {
+      console.error(e)
+      return null
+    }
+  }
+
+  public getHex(message: any) {
+    const hexChar = [
+      '0',
+      '1',
+      '2',
+      '3',
+      '4',
+      '5',
+      '6',
+      '7',
+      '8',
+      '9',
+      'A',
+      'B',
+      'C',
+      'D',
+      'E',
+      'F'
+    ]
+    let hex = ''
+    try {
+      for (let i = 0; i < message.length; i++) {
+        hex += '' + hexChar[(message[i] >> 4) & 0x0f] + hexChar[message[i] & 0x0f]
+      }
+    } catch (e) {
+      console.error(e)
+    }
+    const hexMessage = '0x' + hex
+    return hexMessage
+  }
+}

src/models/Config.ts

@@ -85,6 +85,17 @@ export class Config {
    * @type {any}
    */
   public fixedRateExchangeAddressABI?: AbiItem | AbiItem[]
+  /**
+   * DDOContractAddress
+   * @type {string}
+   */
+  public DDOContractAddress?: string
+
+  /**
+   * DDOContractABI
+   * @type {any}
+   */
+  public DDOContractABI?: AbiItem | AbiItem[]
   /**
    * Log level.
    * @type {boolean | LogLevel}

src/ocean/Assets.ts

@@ -163,10 +163,16 @@ export class Assets extends Instantiable {
       observer.next(CreateProgressStep.ProofGenerated)
       this.logger.log('Storing DDO')
       observer.next(CreateProgressStep.StoringDdo)
-      const storedDdo = await this.ocean.metadatastore.storeDDO(ddo)
-      this.logger.log('DDO stored')
+      // const storedDdo = await this.ocean.metadatastore.storeDDO(ddo)
+      const storeTx = await this.ocean.OnChainMetadataStore.publish(
+        ddo.id,
+        ddo,
+        publisher.getId()
+      )
+      this.logger.log('DDO stored ' + ddo.id)
       observer.next(CreateProgressStep.DdoStored)
-      return storedDdo
+      if (storeTx) return ddo
+      else return null
     })
   }
 
@@ -219,23 +225,39 @@ export class Assets extends Instantiable {
     did: string,
     newMetadata: EditableMetadata,
     account: Account
-  ): Promise<string> {
+  ): Promise<DDO> {
     const oldDdo = await this.ocean.metadatastore.retrieveDDO(did)
-    // get a signature
-    const signature = await this.ocean.utils.signature.signForAquarius(
-      oldDdo.updated,
-      account
+    let i
+    for (i = 0; i < oldDdo.service.length; i++) {
+      if (oldDdo.service[i].type === 'metadata') {
+        if (newMetadata.title) oldDdo.service[i].attributes.main.name = newMetadata.title
+        if (!oldDdo.service[i].attributes.additionalInformation)
+          oldDdo.service[i].attributes.additionalInformation = Object()
+        if (newMetadata.description)
+          oldDdo.service[i].attributes.additionalInformation.description =
+            newMetadata.description
+        if (newMetadata.links)
+          oldDdo.service[i].attributes.additionalInformation.links = newMetadata.links
+      }
+    }
+    if (newMetadata.servicePrices) {
+      for (i = 0; i < newMetadata.servicePrices.length; i++) {
+        if (
+          newMetadata.servicePrices[i].cost &&
+          newMetadata.servicePrices[i].serviceIndex
+        ) {
+          oldDdo.service[newMetadata.servicePrices[i].serviceIndex].attributes.main.cost =
+            newMetadata.servicePrices[i].cost
+        }
+      }
+    }
+    const storeTx = await this.ocean.OnChainMetadataStore.update(
+      oldDdo.id,
+      oldDdo,
+      account.getId()
     )
-    let result = null
-    if (signature != null)
-      result = await this.ocean.metadatastore.editMetadata(
-        did,
-        newMetadata,
-        oldDdo.updated,
-        signature
-      )
-
-    return result
+    if (storeTx) return oldDdo
+    else return null
   }
 
   /**
@@ -251,45 +273,22 @@ export class Assets extends Instantiable {
     serviceIndex: number,
     computePrivacy: ServiceComputePrivacy,
     account: Account
-  ): Promise<string> {
+  ): Promise<DDO> {
     const oldDdo = await this.ocean.metadatastore.retrieveDDO(did)
-    // get a signature
-    const signature = await this.ocean.utils.signature.signForAquarius(
-      oldDdo.updated,
-      account
+    if (oldDdo.service[serviceIndex].type !== 'compute') return null
+    oldDdo.service[serviceIndex].attributes.main.privacy.allowRawAlgorithm =
+      computePrivacy.allowRawAlgorithm
+    oldDdo.service[serviceIndex].attributes.main.privacy.allowNetworkAccess =
+      computePrivacy.allowNetworkAccess
+    oldDdo.service[serviceIndex].attributes.main.privacy.trustedAlgorithms =
+      computePrivacy.trustedAlgorithms
+    const storeTx = await this.ocean.OnChainMetadataStore.update(
+      oldDdo.id,
+      oldDdo,
+      account.getId()
     )
-    let result = null
-    if (signature != null)
-      result = await this.ocean.metadatastore.updateComputePrivacy(
-        did,
-        serviceIndex,
-        computePrivacy.allowRawAlgorithm,
-        computePrivacy.allowNetworkAccess,
-        computePrivacy.trustedAlgorithms,
-        oldDdo.updated,
-        signature
-      )
-
-    return result
-  }
-
-  /**
-   * Retire a DDO (Delete)
-   * @param  {did} string DID.
-   * @param  {Account} account Ethereum account of owner to sign and prove the ownership.
-   * @return {Promise<string>}
-   */
-  public async retire(did: string, account: Account): Promise<string> {
-    const oldDdo = await this.ocean.metadatastore.retrieveDDO(did)
-    // get a signature
-    const signature = await this.ocean.utils.signature.signForAquarius(
-      oldDdo.updated,
-      account
-    )
-    let result = null
-    if (signature != null)
-      result = await this.ocean.metadatastore.retire(did, oldDdo.updated, signature)
-    return result
+    if (storeTx) return oldDdo
+    else return null
   }
 
   /**

src/ocean/Ocean.ts

@@ -3,6 +3,7 @@ import { Assets } from './Assets'
 import { Versions } from './Versions'
 import { OceanUtils } from './utils/Utils'
 import { MetadataStore } from '../metadatastore/MetadataStore'
+import { OnChainMetadataStore } from '../metadatastore/OnChainMetaData'
 import { Provider } from '../provider/Provider'
 import { DataTokens } from '../datatokens/Datatokens'
 import { Network } from '../datatokens/Network'
@@ -64,6 +65,11 @@ export class Ocean extends Instantiable {
       instanceConfig.config.fixedRateExchangeAddressABI,
       instanceConfig.config.oceanTokenAddress
     )
+    instance.OnChainMetadataStore = new OnChainMetadataStore(
+      instanceConfig.config.web3Provider,
+      instanceConfig.config.DDOContractAddress,
+      instanceConfig.config.DDOContractABI
+    )
     instance.versions = await Versions.getInstance(instanceConfig)
     instance.network = new Network()
     return instance
@@ -92,7 +98,11 @@ export class Ocean extends Instantiable {
    * @type {MetadataStore}
    */
   public metadatastore: MetadataStore
-
+  /**
+   * OnChainMetadataStore instance.
+   * @type {OnChainMetadataStore}
+   */
+  public OnChainMetadataStore: OnChainMetadataStore
   /**
    * Ocean account submodule
    * @type {Accounts}

src/utils/ConfigHelper.ts

@@ -1,5 +1,6 @@
 import Config from '../models/Config'
 import { Logger } from '../lib'
+import fs from 'fs'
 
 export declare type ConfigHelperNetworkName =
   | 'mainnet'
@@ -23,7 +24,8 @@ const configs: ConfigHelperConfig[] = [
     metadataStoreUri: 'http://127.0.0.1:5000',
     providerUri: 'http://127.0.0.1:8030',
     poolFactoryAddress: null,
-    fixedRateExchangeAddress: null
+    fixedRateExchangeAddress: null,
+    DDOContractAddress: null
   },
   {
     chainId: 4,
@@ -34,7 +36,8 @@ const configs: ConfigHelperConfig[] = [
     metadataStoreUri: 'https://aquarius.rinkeby.v3.dev-ocean.com',
     providerUri: 'https://provider.rinkeby.v3.dev-ocean.com',
     poolFactoryAddress: '0x9B90A1358fbeEC1C4bB1DA7D4E85C708f87556Ec',
-    fixedRateExchangeAddress: '0x991c08bD00761A299d3126a81a985329096896D4'
+    fixedRateExchangeAddress: '0x991c08bD00761A299d3126a81a985329096896D4',
+    DDOContractAddress: '0xEfA25E39192b3175d451D79C1c0a41Fa3C32c87d'
   },
   {
     chainId: 1,
@@ -45,15 +48,34 @@ const configs: ConfigHelperConfig[] = [
     metadataStoreUri: null,
     providerUri: null,
     poolFactoryAddress: null,
-    fixedRateExchangeAddress: null
+    fixedRateExchangeAddress: null,
+    DDOContractAddress: null
   }
 ]
 
 export class ConfigHelper {
+  /* Load config from env ADDRESS_FILE (generated by ocean-contracts) */
+  public loadAddressesFromEnv() {
+    try {
+      const data = JSON.parse(fs.readFileSync(process.env.ADDRESS_FILE, 'utf8'))
+      if (data) {
+        if (data.ganache) {
+          if (data.ganache.DTFactory) configs[0].factoryAddress = data.ganache.DTFactory
+          if (data.ganache.BFactory) configs[0].poolFactoryAddress = data.ganache.BFactory
+          if (data.ganache.FixedRateExchange)
+            configs[0].fixedRateExchangeAddress = data.ganache.FixedRateExchange
+          if (data.ganache.DDO) configs[0].DDOContractAddress = data.ganache.DDO
+        }
+      }
+      if (process.env.AQUARIUS_URI) configs[0].metadataStoreUri = process.env.AQUARIUS_URI
+    } catch (e) {}
+  }
+
   public getConfig(
     network: ConfigHelperNetworkName | ConfigHelperNetworkId,
     infuraProjectId?: string
   ): Config {
+    if (network === 'development') this.loadAddressesFromEnv()
     const filterBy = typeof network === 'string' ? 'network' : 'chainId'
     const config = configs.find((c) => c[filterBy] === network)
 

test/integration/ComputeFlow.test.ts

@@ -2,7 +2,8 @@ import { AbiItem } from 'web3-utils/types'
 import { TestContractHandler } from '../TestContractHandler'
 import { DataTokens } from '../../src/datatokens/Datatokens'
 import { Ocean } from '../../src/ocean/Ocean'
-import config from './config'
+import { ConfigHelper } from '../../src/utils/ConfigHelper'
+
 import { assert } from 'console'
 import { ServiceComputePrivacy } from '../../src/ddo/interfaces/Service'
 import Web3 from 'web3'
@@ -10,6 +11,12 @@ import factory from '@oceanprotocol/contracts/artifacts/DTFactory.json'
 import datatokensTemplate from '@oceanprotocol/contracts/artifacts/DataTokenTemplate.json'
 const web3 = new Web3('http://127.0.0.1:8545')
 
+function sleep(ms) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms)
+  })
+}
+
 describe('Compute flow', () => {
   let owner
   let bob
@@ -61,7 +68,8 @@ describe('Compute flow', () => {
       factory.bytecode,
       web3
     )
-
+    const config = new ConfigHelper().getConfig('development')
+    config.web3Provider = web3
     ocean = await Ocean.getInstance(config)
     owner = (await ocean.accounts.list())[0]
     alice = (await ocean.accounts.list())[1]
@@ -154,6 +162,7 @@ describe('Compute flow', () => {
     )
     ddo = await ocean.assets.create(asset, alice, [computeService], tokenAddress)
     assert(ddo.dataToken === tokenAddress)
+    await sleep(6000)
   })
 
   // alex
@@ -178,6 +187,7 @@ describe('Compute flow', () => {
       tokenAddress
     )
     assert(datasetNoRawAlgo.dataToken === tokenAddress)
+    await sleep(6000)
   })
 
   it('should publish a dataset with a compute service object that allows only algo with did:op:1234', async () => {
@@ -201,6 +211,7 @@ describe('Compute flow', () => {
       tokenAddress
     )
     assert(datasetWithTrustedAlgo.dataToken === tokenAddress)
+    await sleep(6000)
   })
 
   it('should publish an algorithm', async () => {
@@ -239,6 +250,7 @@ describe('Compute flow', () => {
     )
     algorithmAsset = await ocean.assets.create(algoAsset, alice, [service1], tokenAddress)
     assert(algorithmAsset.dataToken === tokenAddress)
+    await sleep(6000)
   })
 
   it('Alice mints 100 DTs and tranfers them to the compute marketplace', async () => {
@@ -358,6 +370,42 @@ describe('Compute flow', () => {
     jobId = response.jobId
     assert(response.status >= 10)
   })
+  it('Alice updates Compute Privacy', async () => {
+    const newComputePrivacy = {
+      allowRawAlgorithm: false,
+      allowNetworkAccess: true,
+      trustedAlgorithms: ['did:op:1234', 'did:op:1235']
+    }
+    let computeIndex = 0
+    for (let i = 0; i < ddo.service.length; i++) {
+      if (ddo.service[i].type === 'compute') {
+        computeIndex = i
+        break
+      }
+    }
+    assert(computeIndex > 0)
+    const newDdo = await ocean.assets.updateComputePrivacy(
+      ddo.id,
+      computeIndex,
+      newComputePrivacy,
+      alice
+    )
+    assert(newDdo !== null)
+    await sleep(6000)
+    const metaData = await ocean.assets.getServiceByType(ddo.id, 'compute')
+    assert(
+      metaData.attributes.main.privacy.allowRawAlgorithm ===
+        newComputePrivacy.allowRawAlgorithm
+    )
+    assert(
+      metaData.attributes.main.privacy.allowNetworkAccess ===
+        newComputePrivacy.allowNetworkAccess
+    )
+    assert(
+      metaData.attributes.main.privacy.trustedAlgorithms ===
+        newComputePrivacy.trustedAlgorithms
+    )
+  })
 
   // it('Bob restarts compute job', async () => {})
   // it('Bob gets outputs', async () => {})

test/integration/Marketplaceflow.test.ts

@@ -2,14 +2,23 @@ import { AbiItem } from 'web3-utils/types'
 import { TestContractHandler } from '../TestContractHandler'
 import { DataTokens } from '../../src/datatokens/Datatokens'
 import { Ocean } from '../../src/ocean/Ocean'
-import config from './config'
+import { ConfigHelper } from '../../src/utils/ConfigHelper'
+
+// import config from './config'
 import { assert } from 'console'
 
 import Web3 from 'web3'
 import factory from '@oceanprotocol/contracts/artifacts/DTFactory.json'
 import datatokensTemplate from '@oceanprotocol/contracts/artifacts/DataTokenTemplate.json'
+import { EditableMetadata } from '../../src/lib'
 const web3 = new Web3('http://127.0.0.1:8545')
 
+function sleep(ms) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms)
+  })
+}
+
 describe('Marketplace flow', () => {
   let owner
   let bob
@@ -38,7 +47,8 @@ describe('Marketplace flow', () => {
       factory.bytecode,
       web3
     )
-
+    const config = new ConfigHelper().getConfig('development')
+    config.web3Provider = web3
     ocean = await Ocean.getInstance(config)
     owner = (await ocean.accounts.list())[0]
     alice = (await ocean.accounts.list())[1]
@@ -101,6 +111,7 @@ describe('Marketplace flow', () => {
     )
     ddo = await ocean.assets.create(asset, alice, [service1], tokenAddress)
     assert(ddo.dataToken === tokenAddress)
+    await sleep(6000)
   })
 
   it('Alice mints 100 tokens', async () => {
@@ -190,4 +201,20 @@ describe('Marketplace flow', () => {
     const assets = await ocean.assets.ownerAssets(alice.getId())
     assert(assets.length > 0)
   })
+  it('Alice updates metadata', async () => {
+    const newMetaData: EditableMetadata = {
+      description: 'new description',
+      title: 'new title',
+      links: [{ name: 'link1', type: 'sample', url: 'http://example.net' }]
+    }
+    const newDdo = await ocean.assets.editMetadata(ddo.id, newMetaData, alice)
+    assert(newDdo !== null)
+    await sleep(6000)
+    const metaData = await ocean.assets.getServiceByType(ddo.id, 'metadata')
+    assert(metaData.attributes.main.name === newMetaData.title)
+    assert(
+      metaData.attributes.additionalInformation.description === newMetaData.description
+    )
+    assert(metaData.attributes.additionalInformation.links === newMetaData.links)
+  })
 })