From 59e2accf68f0b3f76a051b3d3d1f0057c98cdfc2 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sat, 9 May 2020 22:14:57 +0300 Subject: [PATCH 1/7] add trusted algo --- .travis.yml | 2 +- src/aquarius/Aquarius.ts | 54 ++++++++++++++++++++++ src/ddo/Service.ts | 7 +++ src/ocean/OceanAssets.ts | 37 ++++++++++++++- src/ocean/OceanCompute.ts | 10 ++-- test/integration/ocean/AssetOwners.test.ts | 53 +++++++++++++++++++++ 6 files changed, 158 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8faa400..99a4e31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_script: - ganache-cli --port 18545 > ganache-cli.log & - git clone https://github.com/oceanprotocol/barge - cd barge - - export AQUARIUS_VERSION=unstable + - export AQUARIUS_VERSION=ctd_trusted_algo - export BRIZO_VERSION=v0.9.5 - export KEEPER_VERSION=v0.13.2 - export EVENTS_HANDLER_VERSION=v0.4.7 diff --git a/src/aquarius/Aquarius.ts b/src/aquarius/Aquarius.ts index d69b046..3611a53 100644 --- a/src/aquarius/Aquarius.ts +++ b/src/aquarius/Aquarius.ts @@ -261,6 +261,60 @@ export class Aquarius { return result } +/** + * Update Compute Privacy + * @param {DID | string} did DID of the asset to update. + * @param {number } serviceIndex Service index + * @param {boolean} allowRawAlgorithm Allow Raw Algorithms + * @param {boolean} allowNetworkAccess Allow Raw Algorithms + * @param {String[]} trustedAlgorithms Allow Raw Algorithms + * @param {String} updated Updated field of the DDO + * @param {String} signature Signature using updated field to verify that the consumer has rights + * @return {Promise} Result. + */ + public async updateComputePrivacy( + did: DID | string, + serviceIndex: number, + allowRawAlgorithm: boolean, + allowNetworkAccess: boolean, + trustedAlgorithms: string[], + updated: string, + signature: string + ): Promise { + did = did && DID.parse(did) + const fullUrl = `${this.url}${apiPath}/computePrivacy/update/${did.getDid()}` + const result = await this.fetch + .put( + fullUrl, + JSON.stringify({ + signature: signature, + updated: updated, + serviceIndex: serviceIndex, + allowRawAlgorithm: allowRawAlgorithm, + allowNetworkAccess: allowNetworkAccess, + trustedAlgorithms: trustedAlgorithms + }) + ) + .then((response: any) => { + if (response.ok) { + return response.text + } + this.logger.log( + 'update compute privacy failed:', + response.status, + response.statusText + ) + return null + }) + + .catch(error => { + this.logger.error('Error updating compute privacy: ', error) + return null + }) + + return result + } + /** * Edit Metadata for a DDO. * @param {did} string DID. diff --git a/src/ddo/Service.ts b/src/ddo/Service.ts index 42a7497..64d2fc7 100644 --- a/src/ddo/Service.ts +++ b/src/ddo/Service.ts @@ -25,6 +25,12 @@ export interface ServiceAccessAttributes extends ServiceCommonAttributes { timeout: number } } +export interface ServiceComputePrivacy{ + allowRawAlgorithm: boolean, + allowNetworkAccess: boolean, + trustedAlgorithms: string[] +} + export interface ServiceComputeAttributes extends ServiceCommonAttributes { main: { @@ -34,6 +40,7 @@ export interface ServiceComputeAttributes extends ServiceCommonAttributes { timeout: number provider?: ServiceComputeProvider name: string + privacy?: ServiceComputePrivacy } } diff --git a/src/ocean/OceanAssets.ts b/src/ocean/OceanAssets.ts index 9e0b9d5..5a07039 100644 --- a/src/ocean/OceanAssets.ts +++ b/src/ocean/OceanAssets.ts @@ -2,7 +2,7 @@ import { TransactionReceipt } from 'web3-core' import { SearchQuery } from '../aquarius/Aquarius' import { DDO } from '../ddo/DDO' import { MetaData, EditableMetaData } from '../ddo/MetaData' -import { Service, ServiceAccess } from '../ddo/Service' +import { Service, ServiceAccess, ServiceComputePrivacy } from '../ddo/Service' import Account from './Account' import DID from './DID' import { fillConditionsWithDDO, SubscribablePromise, didZeroX } from '../utils' @@ -414,6 +414,41 @@ export class OceanAssets extends Instantiable { return result } + /** + * Update Compute Privacy + * @param {did} string DID. + * @param {number} serviceIndex Index of the compute service in the DDO + * @param {ServiceComputePrivacy} computePrivacy ComputePrivacy fields & new values. + * @param {Account} account Ethereum account of owner to sign and prove the ownership. + * @return {Promise} + */ + public async updateComputePrivacy( + did: string, + serviceIndex: number, + computePrivacy: ServiceComputePrivacy, + account: Account + ): Promise { + const oldDdo = await this.ocean.aquarius.retrieveDDO(did) + // get a signature + const signature = await this.ocean.utils.signature.signForAquarius( + oldDdo.updated, + account + ) + let result = null + if (signature != null) + result = await this.ocean.aquarius.updateComputePrivacy( + did, + serviceIndex, + computePrivacy.allowRawAlgorithm, + computePrivacy.allowNetworkAccess, + computePrivacy.trustedAlgorithms, + oldDdo.updated, + signature + ) + + return result + } + /** * Retire a DDO (Delete) * @param {did} string DID. diff --git a/src/ocean/OceanCompute.ts b/src/ocean/OceanCompute.ts index 331f16d..48a488e 100644 --- a/src/ocean/OceanCompute.ts +++ b/src/ocean/OceanCompute.ts @@ -5,7 +5,7 @@ import { DDO } from '../ddo/DDO' import { SubscribablePromise } from '../utils' import { OrderProgressStep } from './utils/ServiceUtils' import { DID } from '../squid' -import { ServiceCompute } from '../ddo/Service' +import { ServiceCompute, ServiceComputePrivacy } from '../ddo/Service' export const ComputeJobStatus = Object.freeze({ Started: 10, @@ -266,12 +266,13 @@ export class OceanCompute extends Instantiable { public async createComputeServiceAttributes( consumerAccount: Account, price: string, - datePublished: string + datePublished: string, + computePrivacy?: ServiceComputePrivacy ): Promise { const { templates } = this.ocean.keeper const serviceAgreementTemplate = await templates.escrowComputeExecutionTemplate.getServiceAgreementTemplate() const name = 'dataAssetComputingServiceAgreement' - return { + let service={ type: 'compute', index: 3, serviceEndpoint: this.ocean.brizo.getComputeEndpoint(), @@ -287,5 +288,8 @@ export class OceanCompute extends Instantiable { serviceAgreementTemplate } } + if (computePrivacy) + service['attributes']['main']['privacy'] = computePrivacy + return service as ServiceCompute } } diff --git a/test/integration/ocean/AssetOwners.test.ts b/test/integration/ocean/AssetOwners.test.ts index 3f1af67..a85c000 100644 --- a/test/integration/ocean/AssetOwners.test.ts +++ b/test/integration/ocean/AssetOwners.test.ts @@ -2,6 +2,7 @@ import { assert } from 'chai' import { config } from '../config' import { getMetadata } from '../utils' import { Ocean, Account, EditableMetaData } from '../../../src' // @oceanprotocol/squid +import { ServiceComputePrivacy } from '../../../src/ddo/Service' describe('Asset Owners', () => { let ocean: Ocean @@ -202,4 +203,56 @@ describe('Asset Owners', () => { assert.equal(-1, remList.indexOf(consumer1.getId())) assert.notEqual(-1, remList.indexOf(consumer2.getId())) }) + + it('should be able to update computePrivacy', async () => { + const origComputePrivacy = { + allowRawAlgorithm: true, + allowNetworkAccess: true, + trustedAlgorithms: [] + } + + const newComputePrivacy = { + allowRawAlgorithm: false, + allowNetworkAccess: false, + trustedAlgorithms: ['did:op:123', 'did:op:1234'] + } + const computeService = await ocean.compute.createComputeServiceAttributes( + account1, + '0', + '2020-03-10T10:00:00Z', + origComputePrivacy as ServiceComputePrivacy + ) + const { id } = await ocean.assets.create(metadata as any, account1, [ + computeService + ]) + + const oldDDO = await ocean.assets.resolve(id) + let serviceIndex = null + + for (let index = 0; index < oldDDO.service.length; index++) { + if (oldDDO.service[index].type == 'compute') serviceIndex = index + } + + await ocean.assets.updateComputePrivacy( + id, + serviceIndex, + newComputePrivacy as ServiceComputePrivacy, + account1 + ) + + const newDDO = await ocean.assets.resolve(id) + + assert.equal( + newDDO.service[serviceIndex].attributes.main.privacy.allowRawAlgorithm, + newComputePrivacy.allowRawAlgorithm + ) + assert.equal( + newDDO.service[serviceIndex].attributes.main.privacy.allowNetworkAccess, + newComputePrivacy.allowNetworkAccess + ) + assert.equal( + newDDO.service[serviceIndex].attributes.main.privacy.trustedAlgorithms, + newComputePrivacy.trustedAlgorithms + ) + }) }) From c4921d129bc5edf87151e6ad72108e62495a30eb Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sat, 9 May 2020 22:20:40 +0300 Subject: [PATCH 2/7] fix lint --- src/aquarius/Aquarius.ts | 4 ++-- src/ddo/Service.ts | 7 +++---- src/ocean/OceanCompute.ts | 5 ++--- test/integration/ocean/AssetOwners.test.ts | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/aquarius/Aquarius.ts b/src/aquarius/Aquarius.ts index 3611a53..14d4fa0 100644 --- a/src/aquarius/Aquarius.ts +++ b/src/aquarius/Aquarius.ts @@ -261,8 +261,8 @@ export class Aquarius { return result } -/** - * Update Compute Privacy + /** + * Update Compute Privacy * @param {DID | string} did DID of the asset to update. * @param {number } serviceIndex Service index * @param {boolean} allowRawAlgorithm Allow Raw Algorithms diff --git a/src/ddo/Service.ts b/src/ddo/Service.ts index 64d2fc7..c2a7d78 100644 --- a/src/ddo/Service.ts +++ b/src/ddo/Service.ts @@ -25,13 +25,12 @@ export interface ServiceAccessAttributes extends ServiceCommonAttributes { timeout: number } } -export interface ServiceComputePrivacy{ - allowRawAlgorithm: boolean, - allowNetworkAccess: boolean, +export interface ServiceComputePrivacy { + allowRawAlgorithm: boolean + allowNetworkAccess: boolean trustedAlgorithms: string[] } - export interface ServiceComputeAttributes extends ServiceCommonAttributes { main: { creator: string diff --git a/src/ocean/OceanCompute.ts b/src/ocean/OceanCompute.ts index 48a488e..4299a85 100644 --- a/src/ocean/OceanCompute.ts +++ b/src/ocean/OceanCompute.ts @@ -272,7 +272,7 @@ export class OceanCompute extends Instantiable { const { templates } = this.ocean.keeper const serviceAgreementTemplate = await templates.escrowComputeExecutionTemplate.getServiceAgreementTemplate() const name = 'dataAssetComputingServiceAgreement' - let service={ + const service = { type: 'compute', index: 3, serviceEndpoint: this.ocean.brizo.getComputeEndpoint(), @@ -288,8 +288,7 @@ export class OceanCompute extends Instantiable { serviceAgreementTemplate } } - if (computePrivacy) - service['attributes']['main']['privacy'] = computePrivacy + if (computePrivacy) service.attributes.main.privacy = computePrivacy return service as ServiceCompute } } diff --git a/test/integration/ocean/AssetOwners.test.ts b/test/integration/ocean/AssetOwners.test.ts index a85c000..4074097 100644 --- a/test/integration/ocean/AssetOwners.test.ts +++ b/test/integration/ocean/AssetOwners.test.ts @@ -230,7 +230,7 @@ describe('Asset Owners', () => { let serviceIndex = null for (let index = 0; index < oldDDO.service.length; index++) { - if (oldDDO.service[index].type == 'compute') serviceIndex = index + if (oldDDO.service[index].type === 'compute') serviceIndex = index } await ocean.assets.updateComputePrivacy( From 25413b05c76b2e7475a753cb272249c9112e3af1 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sat, 9 May 2020 22:34:44 +0300 Subject: [PATCH 3/7] fix compute privacy attr --- src/ocean/OceanCompute.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ocean/OceanCompute.ts b/src/ocean/OceanCompute.ts index 4299a85..6e75735 100644 --- a/src/ocean/OceanCompute.ts +++ b/src/ocean/OceanCompute.ts @@ -283,6 +283,7 @@ export class OceanCompute extends Instantiable { datePublished, price, timeout: 3600, + privacy: {}, name }, serviceAgreementTemplate From 9a722d8686796ec993782d21173b406971876b5b Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sun, 10 May 2020 00:49:13 -0700 Subject: [PATCH 4/7] add checks before order & tests --- src/ocean/OceanCompute.ts | 52 +++++++++++---- test/integration/ocean/Compute.test.ts | 87 +++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 13 deletions(-) diff --git a/src/ocean/OceanCompute.ts b/src/ocean/OceanCompute.ts index 6e75735..fc4140a 100644 --- a/src/ocean/OceanCompute.ts +++ b/src/ocean/OceanCompute.ts @@ -5,7 +5,7 @@ import { DDO } from '../ddo/DDO' import { SubscribablePromise } from '../utils' import { OrderProgressStep } from './utils/ServiceUtils' import { DID } from '../squid' -import { ServiceCompute, ServiceComputePrivacy } from '../ddo/Service' +import { Service, ServiceCompute, ServiceComputePrivacy } from '../ddo/Service' export const ComputeJobStatus = Object.freeze({ Started: 10, @@ -66,16 +66,43 @@ export class OceanCompute extends Instantiable { * Starts an order of a compute service that is defined in an asset's services. * @param {Account} consumerAccount The account of the consumer ordering the service. * @param {string} datasetDid The DID of the dataset asset (of type `dataset`) to run the algorithm on. + * @param {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset. + * @param {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified. * @return {Promise} Returns the Service Agreement ID, representation of `bytes32` ID. + * + * Note: algorithmDid and algorithmMeta are optional, but if they are not passed, + * you can end up in the situation that you are ordering and paying for your agreement, + * but brizo will not allow the compute, due to privacy settings of the ddo */ public order( consumerAccount: Account, datasetDid: string, + algorithmDid?: string, + algorithmMeta?: MetaDataAlgorithm, provider?: string ): SubscribablePromise { return new SubscribablePromise(async observer => { const { assets, keeper, utils } = this.ocean const ddo: DDO = await assets.resolve(datasetDid) + const service: Service = ddo.findServiceByType('compute') + if (!service) return null + if (algorithmMeta) { + // check if raw algo is allowed + if (service.attributes.main.privacy) + if (!service.attributes.main.privacy.allowRawAlgorithm) return null + } + if (algorithmDid) { + // check if did is in trusted list + if (service.attributes.main.privacy) + if (service.attributes.main.privacy.trustedAlgorithms) + if (service.attributes.main.privacy.trustedAlgorithms.length > 0) + if ( + !service.attributes.main.privacy.trustedAlgorithms.includes( + algorithmDid + ) + ) + return null + } const condition = keeper.conditions.computeExecutionCondition const agreementId = await utils.services.order( @@ -137,17 +164,18 @@ export class OceanCompute extends Instantiable { output?: Output ): Promise { output = this.checkOutput(consumerAccount, output) - const computeJobsList = await this.ocean.brizo.compute( - 'post', - agreementId, - consumerAccount, - algorithmDid, - algorithmMeta, - undefined, - output - ) - - return computeJobsList[0] as ComputeJob + if (agreementId) { + const computeJobsList = await this.ocean.brizo.compute( + 'post', + agreementId, + consumerAccount, + algorithmDid, + algorithmMeta, + undefined, + output + ) + return computeJobsList[0] as ComputeJob + } else return null } /** diff --git a/test/integration/ocean/Compute.test.ts b/test/integration/ocean/Compute.test.ts index bf1e60d..9723748 100644 --- a/test/integration/ocean/Compute.test.ts +++ b/test/integration/ocean/Compute.test.ts @@ -11,7 +11,7 @@ import { MetaDataAlgorithm } from '../../../src' import { getMetadata } from '../utils' -import { ServiceCompute } from '../../../src/ddo/Service' +import { ServiceCompute, ServiceComputePrivacy } from '../../../src/ddo/Service' const metadataAsset = getMetadata() const metadataAlgorithm = getMetadata(0, 'algorithm') @@ -26,11 +26,24 @@ const customConfig: Config = { verbose: true } +export const rawAlgoMeta = { + rawcode: `console.log('Hello world'!)`, + format: 'docker-image', + version: '0.1', + container: { + entrypoint: 'node $ALGO', + image: 'node', + tag: '10' + } +} + describe('Compute', () => { let ocean: Ocean let account: Account let agreementId: string let dataset: DDO + let datasetNoRawAlgo: DDO + let datasetWithTrustedAlgo: DDO let algorithm: DDO let computeService: ServiceCompute let jobId: string @@ -63,6 +76,58 @@ describe('Compute', () => { assert.deepEqual(stepsAsset, [0, 1, 2, 3, 4, 5, 6, 7]) }) + it('should publish a dataset with a compute service object that does not allow rawAlgo', async () => { + const stepsAsset = [] + const origComputePrivacy = { + allowRawAlgorithm: false, + allowNetworkAccess: false, + trustedAlgorithms: [] + } + + computeService = await ocean.compute.createComputeServiceAttributes( + account, + '1000', + metadataAsset.main.datePublished, + origComputePrivacy as ServiceComputePrivacy + ) + datasetNoRawAlgo = await ocean.assets + .create(metadataAsset as MetaData, account, [computeService]) + .next(step => stepsAsset.push(step)) + + assert.instanceOf(datasetNoRawAlgo, DDO) + assert.isDefined( + dataset.findServiceByType('compute'), + `DDO compute service doesn't exist` + ) + assert.deepEqual(stepsAsset, [0, 1, 2, 3, 4, 5, 6, 7]) + }) + + it('should publish a dataset with a compute service object that allows only algo with did:op:1234', async () => { + const stepsAsset = [] + const origComputePrivacy = { + allowRawAlgorithm: false, + allowNetworkAccess: false, + trustedAlgorithms: ['did:op:1234'] + } + + computeService = await ocean.compute.createComputeServiceAttributes( + account, + '1000', + metadataAsset.main.datePublished, + origComputePrivacy as ServiceComputePrivacy + ) + datasetWithTrustedAlgo = await ocean.assets + .create(metadataAsset as MetaData, account, [computeService]) + .next(step => stepsAsset.push(step)) + + assert.instanceOf(datasetWithTrustedAlgo, DDO) + assert.isDefined( + dataset.findServiceByType('compute'), + `DDO compute service doesn't exist` + ) + assert.deepEqual(stepsAsset, [0, 1, 2, 3, 4, 5, 6, 7]) + }) + it('should publish an algorithm', async () => { const stepsAlgorithm = [] algorithm = await ocean.assets @@ -73,6 +138,26 @@ describe('Compute', () => { assert.deepEqual(stepsAlgorithm, [0, 1, 2, 3, 4, 5, 6, 7]) }) + it('should not allow order the compute service with raw algo for dataset that does not allow raw algo', async () => { + const steps = [] + + agreementId = await ocean.compute + .order(account, datasetNoRawAlgo.id, null, rawAlgoMeta) + .next(step => steps.push(step)) + + assert.equal(agreementId, null) + }) + + it('should not allow order the compute service with did != did:op:1234 for dataset that allows only did:op:1234 as algo', async () => { + const steps = [] + + agreementId = await ocean.compute + .order(account, datasetWithTrustedAlgo.id, 'did:op:233454', null) + .next(step => steps.push(step)) + + assert.equal(agreementId, null) + }) + it('should order the compute service of the dataset', async () => { const steps = [] try { From 3f1d595331719d26b4a11e2fe3e5d7e48ca4e752 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sun, 10 May 2020 01:08:09 -0700 Subject: [PATCH 5/7] fix test to use deepEqual --- test/integration/ocean/AssetOwners.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/ocean/AssetOwners.test.ts b/test/integration/ocean/AssetOwners.test.ts index 4074097..22f2471 100644 --- a/test/integration/ocean/AssetOwners.test.ts +++ b/test/integration/ocean/AssetOwners.test.ts @@ -250,7 +250,7 @@ describe('Asset Owners', () => { newDDO.service[serviceIndex].attributes.main.privacy.allowNetworkAccess, newComputePrivacy.allowNetworkAccess ) - assert.equal( + assert.deepEqual( newDDO.service[serviceIndex].attributes.main.privacy.trustedAlgorithms, newComputePrivacy.trustedAlgorithms ) From 4e5eb1bf1c348c3459b3645954a233547ead1a91 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sat, 16 May 2020 01:26:21 -0700 Subject: [PATCH 6/7] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 99a4e31..bf6bcc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ before_script: - git clone https://github.com/oceanprotocol/barge - cd barge - export AQUARIUS_VERSION=ctd_trusted_algo - - export BRIZO_VERSION=v0.9.5 + - export BRIZO_VERSION=v0.9.7 - export KEEPER_VERSION=v0.13.2 - export EVENTS_HANDLER_VERSION=v0.4.7 - export KEEPER_OWNER_ROLE_ADDRESS="0xe2DD09d719Da89e5a3D0F2549c7E24566e947260" From df95801ca8e0d142629a38d6f566b7e2fbc6c7b0 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 19 May 2020 05:16:00 -0700 Subject: [PATCH 7/7] fix tests broken due to master merge --- src/ocean/OceanCompute.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ocean/OceanCompute.ts b/src/ocean/OceanCompute.ts index 2028766..694b805 100644 --- a/src/ocean/OceanCompute.ts +++ b/src/ocean/OceanCompute.ts @@ -295,12 +295,13 @@ export class OceanCompute extends Instantiable { consumerAccount: Account, price: string, datePublished: string, - timeout: number = 3600, - computePrivacy?: ServiceComputePrivacy + computePrivacy?: ServiceComputePrivacy, + timeout?: number ): Promise { const { templates } = this.ocean.keeper const serviceAgreementTemplate = await templates.escrowComputeExecutionTemplate.getServiceAgreementTemplate() const name = 'dataAssetComputingServiceAgreement' + if (!timeout) timeout = 3600 const service = { type: 'compute', index: 3,