diff --git a/src/ddo/interfaces/Metadata.ts b/src/ddo/interfaces/Metadata.ts index 17333def..6eb48cc9 100644 --- a/src/ddo/interfaces/Metadata.ts +++ b/src/ddo/interfaces/Metadata.ts @@ -10,3 +10,8 @@ export interface Metadata { curation?: Curation status?: Status } + +export interface ValidateMetadata { + valid: Boolean + errors?: Object +} diff --git a/src/metadatacache/MetadataCache.ts b/src/metadatacache/MetadataCache.ts index 20e2c654..432fd9a4 100644 --- a/src/metadatacache/MetadataCache.ts +++ b/src/metadatacache/MetadataCache.ts @@ -1,9 +1,10 @@ import { DDO } from '../ddo/DDO' import DID from '../ocean/DID' import { EditableMetadata } from '../ddo/interfaces/EditableMetadata' -import { Logger } from '../utils' +import { Logger, isDdo } from '../utils' import { WebServiceConnector } from '../ocean/utils/WebServiceConnector' import { Response } from 'node-fetch' +import { Metadata, ValidateMetadata } from '../ddo/interfaces' const apiPath = '/api/v1/aquarius/assets/ddo' @@ -164,6 +165,39 @@ export class MetadataCache { return result } + /** + * Validate Metadata + * @param {Metadata} metadata metadata to be validated. If it's a Metadata, it will be validated agains the local schema. Else, it's validated agains the remote schema + * @return {Promise} Result. + */ + + public async validateMetadata(metadata: Metadata | DDO): Promise { + const status: ValidateMetadata = { + valid: false + } + const path = isDdo(metadata) ? '/validate-remote' : '/validate' + try { + const response = await this.fetch.post( + `${this.url}${apiPath}${path}`, + JSON.stringify(metadata) + ) + if (response.ok) { + const errors = await response.json() + if (errors === true) status.valid = true + else status.errors = errors + } else { + this.logger.error( + 'validate Metadata failed:', + response.status, + response.statusText + ) + } + } catch (error) { + this.logger.error('Error validating metadata: ', error) + } + return status + } + /** * Retrieves a DDO by DID. * @param {DID | string} did DID of the asset. diff --git a/src/metadatacache/OnChainMetaData.ts b/src/metadatacache/OnChainMetaData.ts index 4bfaba36..ce5c9144 100644 --- a/src/metadatacache/OnChainMetaData.ts +++ b/src/metadatacache/OnChainMetaData.ts @@ -66,14 +66,23 @@ export class OnChainMetadata { * @param {String} did * @param {DDO} ddo * @param {String} consumerAccount + * @param {Boolean} encrypt If the DDO should be encrypted + * @param {Boolean} validate If the DDO should be validated against Aqua prior to publish * @return {Promise} exchangeId */ public async publish( did: string, ddo: DDO, consumerAccount: string, - encrypt: boolean = false + encrypt: boolean = false, + validate: boolean = true ): Promise { + if (validate) { + const valid = await this.metadataCache.validateMetadata(ddo) + if (!valid.valid) { + throw new Error(`DDO has failed validation`) + } + } const rawData = await this.prepareRawData(ddo, encrypt) if (!rawData) { throw new Error(`Could not prepare raw data for publish`) @@ -86,14 +95,23 @@ export class OnChainMetadata { * @param {String} did * @param {DDO} ddo * @param {String} consumerAccount + * @param {Boolean} encrypt If the DDO should be encrypted + * @param {Boolean} validate If the DDO should be validated against Aqua prior to publish * @return {Promise} exchangeId */ public async update( did: string, ddo: DDO, consumerAccount: string, - encrypt: boolean = false + encrypt: boolean = false, + validate: boolean = true ): Promise { + if (validate) { + const valid = await this.metadataCache.validateMetadata(ddo) + if (!valid.valid) { + throw new Error(`DDO has failed validation`) + } + } const rawData = await this.prepareRawData(ddo, encrypt) if (!rawData) { throw new Error(`Could not prepare raw data for udate`) diff --git a/src/utils/AssetResolverHelper.ts b/src/utils/AssetResolverHelper.ts index 1969c7f5..cd665b89 100644 --- a/src/utils/AssetResolverHelper.ts +++ b/src/utils/AssetResolverHelper.ts @@ -6,7 +6,7 @@ export interface AssetResolved { ddo: DDO } -function isDdo(arg: any): arg is DDO { +export function isDdo(arg: any): arg is DDO { return arg.id !== undefined } diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index d3fec4bd..e6f0bdfe 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -241,6 +241,7 @@ describe('Compute flow', () => { type: 'dataset', name: 'UK Weather information 2011', dateCreated: dateCreated, + datePublished: dateCreated, author: 'Met Office', license: 'CC-BY', files: [ @@ -477,6 +478,7 @@ describe('Compute flow', () => { type: 'algorithm', name: 'Test Algo', dateCreated: dateCreated, + datePublished: dateCreated, author: 'DevOps', license: 'CC-BY', files: [ @@ -529,6 +531,7 @@ describe('Compute flow', () => { type: 'algorithm', name: 'Test Algo with Compute', dateCreated: dateCreated, + datePublished: dateCreated, author: 'DevOps', license: 'CC-BY', files: [ @@ -590,6 +593,7 @@ describe('Compute flow', () => { type: 'algorithm', name: 'Remote Algorithm', dateCreated: dateCreated, + datePublished: dateCreated, author: 'DevOps', license: 'CC-BY', files: [ @@ -654,6 +658,7 @@ describe('Compute flow', () => { type: 'algorithm', name: 'Remote Algorithm', dateCreated: dateCreated, + datePublished: dateCreated, author: 'DevOps', license: 'CC-BY', files: [ diff --git a/test/integration/Marketplaceflow.test.ts b/test/integration/Marketplaceflow.test.ts index f1365059..8e563026 100644 --- a/test/integration/Marketplaceflow.test.ts +++ b/test/integration/Marketplaceflow.test.ts @@ -60,6 +60,7 @@ describe('Marketplace flow', () => { let assetWithPool let assetWithBadUrl let assetWithEncrypt + let assetInvalidNoName let marketplace: Account let contracts: TestContractHandler let datatoken: DataTokens @@ -67,6 +68,7 @@ describe('Marketplace flow', () => { let tokenAddressWithPool: string let tokenAddressForBadUrlAsset: string let tokenAddressEncrypted: string + let tokenAddressInvalidNoName: string let service1: ServiceAccess let price: string let ocean: Ocean @@ -143,6 +145,15 @@ describe('Marketplace flow', () => { 'AliceDT', 'DTA' ) + assert(tokenAddressEncrypted != null) + tokenAddressInvalidNoName = await datatoken.create( + blob, + alice.getId(), + '10000000000', + 'AliceDT', + 'DTA' + ) + assert(tokenAddressInvalidNoName != null) }) it('Generates metadata', async () => { @@ -155,6 +166,7 @@ describe('Marketplace flow', () => { license: 'MIT', files: [ { + index: 0, url: 'https://s3.amazonaws.com/testfiles.oceanprotocol.com/info.0.json', checksum: 'efb2c764274b745f5fc37f97c6b0e761', contentLength: '4535431', @@ -170,6 +182,7 @@ describe('Marketplace flow', () => { type: 'dataset', name: 'test-dataset-with-pools', dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds + datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds author: 'oceanprotocol-team', license: 'MIT', files: [ @@ -189,6 +202,7 @@ describe('Marketplace flow', () => { type: 'dataset encrypted', name: 'test-dataset-encrypted', dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds + datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds author: 'oceanprotocol-team', license: 'MIT', files: [ @@ -208,6 +222,26 @@ describe('Marketplace flow', () => { type: 'datasetWithBadUrl', name: 'test-dataset-withBadUrl', dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds + datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds + author: 'oceanprotocol-team', + license: 'MIT', + files: [ + { + url: 'https://s3.amazonaws.com/testfiles.oceanprotocol.com/nosuchfile', + checksum: 'efb2c764274b745f5fc37f97c6b0e761', + contentLength: '4535431', + contentType: 'text/csv', + encoding: 'UTF-8', + compression: 'zip' + } + ] + } + } + assetInvalidNoName = { + main: { + type: 'datasetInvalid', + dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds + datePublished: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds author: 'oceanprotocol-team', license: 'MIT', files: [ @@ -223,7 +257,14 @@ describe('Marketplace flow', () => { } } }) - + it('Should validate local metadata', async () => { + const valid = await ocean.metadataCache.validateMetadata(asset) + assert(valid.valid, 'This metadata should be valid') + }) + it('Should invalidate invalid local metadata', async () => { + const valid = await ocean.metadataCache.validateMetadata(assetInvalidNoName) + assert(!valid.valid, 'This metadata should be invalid') + }) it('Alice publishes all datasets', async () => { price = '10' // in datatoken const publishedDate = new Date(Date.now()).toISOString().split('.')[0] + 'Z' @@ -234,6 +275,7 @@ describe('Marketplace flow', () => { publishedDate, timeout ) + asset.main.datePublished = asset.main.dateCreated ddo = await ocean.assets.create(asset, alice, [service1], tokenAddress) assert(ddo.dataToken === tokenAddress) const storeTx = await ocean.onChainMetadata.publish(ddo.id, ddo, alice.getId()) @@ -331,6 +373,38 @@ describe('Marketplace flow', () => { await waitForAqua(ocean, ddoWithCredentials.id) }) + it('Alice should fail to publish invalid dataset', async () => { + price = '10' // in datatoken + const publishedDate = new Date(Date.now()).toISOString().split('.')[0] + 'Z' + const timeout = 0 + service1 = await ocean.assets.createAccessServiceAttributes( + alice, + price, + publishedDate, + timeout + ) + const invalidDdo = await ocean.assets.create( + assetInvalidNoName, + alice, + [service1], + tokenAddressInvalidNoName + ) + assert(invalidDdo.dataToken === tokenAddressInvalidNoName) + let storeTx + try { + storeTx = await ocean.onChainMetadata.publish( + invalidDdo.id, + invalidDdo, + alice.getId() + ) + } catch (e) { + console.error(e) + storeTx = null + } + console.error(storeTx) + assert(!storeTx, 'Alice should not be able to publish invalid datasets') + }) + it('Marketplace should resolve asset using DID', async () => { await ocean.assets.resolve(ddo.id).then((newDDO) => { assert(newDDO.id === ddo.id)