validate ddo before publish on chain (#873)

* validate ddo before publish on chain

src/ddo/interfaces/Metadata.ts

@@ -10,3 +10,8 @@ export interface Metadata {
   curation?: Curation
   status?: Status
 }
+
+export interface ValidateMetadata {
+  valid: Boolean
+  errors?: Object
+}

src/metadatacache/MetadataCache.ts

@@ -1,9 +1,10 @@
 import { DDO } from '../ddo/DDO'
 import DID from '../ocean/DID'
 import { EditableMetadata } from '../ddo/interfaces/EditableMetadata'
-import { Logger } from '../utils'
+import { Logger, isDdo } from '../utils'
 import { WebServiceConnector } from '../ocean/utils/WebServiceConnector'
 import { Response } from 'node-fetch'
+import { Metadata, ValidateMetadata } from '../ddo/interfaces'
 
 const apiPath = '/api/v1/aquarius/assets/ddo'
 
@@ -164,6 +165,39 @@ export class MetadataCache {
     return result
   }
 
+  /**
+   * Validate Metadata
+   * @param  {Metadata} metadata  metadata to be validated. If it's a Metadata, it will be validated agains the local schema. Else, it's validated agains the remote schema
+   * @return {Promise<Boolean|Object>}  Result.
+   */
+
+  public async validateMetadata(metadata: Metadata | DDO): Promise<ValidateMetadata> {
+    const status: ValidateMetadata = {
+      valid: false
+    }
+    const path = isDdo(metadata) ? '/validate-remote' : '/validate'
+    try {
+      const response = await this.fetch.post(
+        `${this.url}${apiPath}${path}`,
+        JSON.stringify(metadata)
+      )
+      if (response.ok) {
+        const errors = await response.json()
+        if (errors === true) status.valid = true
+        else status.errors = errors
+      } else {
+        this.logger.error(
+          'validate Metadata failed:',
+          response.status,
+          response.statusText
+        )
+      }
+    } catch (error) {
+      this.logger.error('Error validating metadata: ', error)
+    }
+    return status
+  }
+
   /**
    * Retrieves a DDO by DID.
    * @param  {DID | string} did DID of the asset.

src/metadatacache/OnChainMetaData.ts

@@ -66,14 +66,23 @@ export class OnChainMetadata {
    * @param {String} did
    * @param {DDO} ddo
    * @param {String} consumerAccount
+   * @param {Boolean} encrypt If the DDO should be encrypted
+   * @param {Boolean} validate If the DDO should be validated against Aqua prior to publish
    * @return {Promise<TransactionReceipt>} exchangeId
    */
   public async publish(
     did: string,
     ddo: DDO,
     consumerAccount: string,
-    encrypt: boolean = false
+    encrypt: boolean = false,
+    validate: boolean = true
   ): Promise<TransactionReceipt> {
+    if (validate) {
+      const valid = await this.metadataCache.validateMetadata(ddo)
+      if (!valid.valid) {
+        throw new Error(`DDO has failed validation`)
+      }
+    }
     const rawData = await this.prepareRawData(ddo, encrypt)
     if (!rawData) {
       throw new Error(`Could not prepare raw data for publish`)
@@ -86,14 +95,23 @@ export class OnChainMetadata {
    * @param {String} did
    * @param {DDO} ddo
    * @param {String} consumerAccount
+   * @param {Boolean} encrypt If the DDO should be encrypted
+   * @param {Boolean} validate If the DDO should be validated against Aqua prior to publish
    * @return {Promise<TransactionReceipt>} exchangeId
    */
   public async update(
     did: string,
     ddo: DDO,
     consumerAccount: string,
-    encrypt: boolean = false
+    encrypt: boolean = false,
+    validate: boolean = true
   ): Promise<TransactionReceipt> {
+    if (validate) {
+      const valid = await this.metadataCache.validateMetadata(ddo)
+      if (!valid.valid) {
+        throw new Error(`DDO has failed validation`)
+      }
+    }
     const rawData = await this.prepareRawData(ddo, encrypt)
     if (!rawData) {
       throw new Error(`Could not prepare raw data for udate`)

src/utils/AssetResolverHelper.ts

@@ -6,7 +6,7 @@ export interface AssetResolved {
   ddo: DDO
 }
 
-function isDdo(arg: any): arg is DDO {
+export function isDdo(arg: any): arg is DDO {
   return arg.id !== undefined
 }
 

test/integration/ComputeFlow.test.ts

@@ -241,6 +241,7 @@ describe('Compute flow', () => {
         type: 'dataset',
         name: 'UK Weather information 2011',
         dateCreated: dateCreated,
+        datePublished: dateCreated,
         author: 'Met Office',
         license: 'CC-BY',
         files: [
@@ -477,6 +478,7 @@ describe('Compute flow', () => {
         type: 'algorithm',
         name: 'Test Algo',
         dateCreated: dateCreated,
+        datePublished: dateCreated,
         author: 'DevOps',
         license: 'CC-BY',
         files: [
@@ -529,6 +531,7 @@ describe('Compute flow', () => {
         type: 'algorithm',
         name: 'Test Algo with Compute',
         dateCreated: dateCreated,
+        datePublished: dateCreated,
         author: 'DevOps',
         license: 'CC-BY',
         files: [
@@ -590,6 +593,7 @@ describe('Compute flow', () => {
         type: 'algorithm',
         name: 'Remote Algorithm',
         dateCreated: dateCreated,
+        datePublished: dateCreated,
         author: 'DevOps',
         license: 'CC-BY',
         files: [
@@ -654,6 +658,7 @@ describe('Compute flow', () => {
         type: 'algorithm',
         name: 'Remote Algorithm',
         dateCreated: dateCreated,
+        datePublished: dateCreated,
         author: 'DevOps',
         license: 'CC-BY',
         files: [

test/integration/Marketplaceflow.test.ts

@@ -60,6 +60,7 @@ describe('Marketplace flow', () => {
   let assetWithPool
   let assetWithBadUrl
   let assetWithEncrypt
+  let assetInvalidNoName
   let marketplace: Account
   let contracts: TestContractHandler
   let datatoken: DataTokens
@@ -67,6 +68,7 @@ describe('Marketplace flow', () => {
   let tokenAddressWithPool: string
   let tokenAddressForBadUrlAsset: string
   let tokenAddressEncrypted: string
+  let tokenAddressInvalidNoName: string
   let service1: ServiceAccess
   let price: string
   let ocean: Ocean
@@ -143,6 +145,15 @@ describe('Marketplace flow', () => {
       'AliceDT',
       'DTA'
     )
+    assert(tokenAddressEncrypted != null)
+    tokenAddressInvalidNoName = await datatoken.create(
+      blob,
+      alice.getId(),
+      '10000000000',
+      'AliceDT',
+      'DTA'
+    )
+    assert(tokenAddressInvalidNoName != null)
   })
 
   it('Generates metadata', async () => {
@@ -155,6 +166,7 @@ describe('Marketplace flow', () => {
         license: 'MIT',
         files: [
           {
+            index: 0,
             url: 'https://s3.amazonaws.com/testfiles.oceanprotocol.com/info.0.json',
             checksum: 'efb2c764274b745f5fc37f97c6b0e761',
             contentLength: '4535431',
@@ -170,6 +182,7 @@ describe('Marketplace flow', () => {
         type: 'dataset',
         name: 'test-dataset-with-pools',
         dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
+        datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
         author: 'oceanprotocol-team',
         license: 'MIT',
         files: [
@@ -189,6 +202,7 @@ describe('Marketplace flow', () => {
         type: 'dataset encrypted',
         name: 'test-dataset-encrypted',
         dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
+        datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
         author: 'oceanprotocol-team',
         license: 'MIT',
         files: [
@@ -208,6 +222,26 @@ describe('Marketplace flow', () => {
         type: 'datasetWithBadUrl',
         name: 'test-dataset-withBadUrl',
         dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
+        datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
+        author: 'oceanprotocol-team',
+        license: 'MIT',
+        files: [
+          {
+            url: 'https://s3.amazonaws.com/testfiles.oceanprotocol.com/nosuchfile',
+            checksum: 'efb2c764274b745f5fc37f97c6b0e761',
+            contentLength: '4535431',
+            contentType: 'text/csv',
+            encoding: 'UTF-8',
+            compression: 'zip'
+          }
+        ]
+      }
+    }
+    assetInvalidNoName = {
+      main: {
+        type: 'datasetInvalid',
+        dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
+        datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds
         author: 'oceanprotocol-team',
         license: 'MIT',
         files: [
@@ -223,7 +257,14 @@ describe('Marketplace flow', () => {
       }
     }
   })
-
+  it('Should validate local metadata', async () => {
+    const valid = await ocean.metadataCache.validateMetadata(asset)
+    assert(valid.valid, 'This metadata should be valid')
+  })
+  it('Should invalidate invalid local metadata', async () => {
+    const valid = await ocean.metadataCache.validateMetadata(assetInvalidNoName)
+    assert(!valid.valid, 'This metadata should be invalid')
+  })
   it('Alice publishes all datasets', async () => {
     price = '10' // in datatoken
     const publishedDate = new Date(Date.now()).toISOString().split('.')[0] + 'Z'
@@ -234,6 +275,7 @@ describe('Marketplace flow', () => {
       publishedDate,
       timeout
     )
+    asset.main.datePublished = asset.main.dateCreated
     ddo = await ocean.assets.create(asset, alice, [service1], tokenAddress)
     assert(ddo.dataToken === tokenAddress)
     const storeTx = await ocean.onChainMetadata.publish(ddo.id, ddo, alice.getId())
@@ -331,6 +373,38 @@ describe('Marketplace flow', () => {
     await waitForAqua(ocean, ddoWithCredentials.id)
   })
 
+  it('Alice should fail to publish invalid dataset', async () => {
+    price = '10' // in datatoken
+    const publishedDate = new Date(Date.now()).toISOString().split('.')[0] + 'Z'
+    const timeout = 0
+    service1 = await ocean.assets.createAccessServiceAttributes(
+      alice,
+      price,
+      publishedDate,
+      timeout
+    )
+    const invalidDdo = await ocean.assets.create(
+      assetInvalidNoName,
+      alice,
+      [service1],
+      tokenAddressInvalidNoName
+    )
+    assert(invalidDdo.dataToken === tokenAddressInvalidNoName)
+    let storeTx
+    try {
+      storeTx = await ocean.onChainMetadata.publish(
+        invalidDdo.id,
+        invalidDdo,
+        alice.getId()
+      )
+    } catch (e) {
+      console.error(e)
+      storeTx = null
+    }
+    console.error(storeTx)
+    assert(!storeTx, 'Alice should not be able to publish invalid datasets')
+  })
+
   it('Marketplace should resolve asset using DID', async () => {
     await ocean.assets.resolve(ddo.id).then((newDDO) => {
       assert(newDDO.id === ddo.id)