Merge pull request #410 from oceanprotocol/feature/ctd_trusted_algo

add trusted algo

.travis.yml

@@ -23,7 +23,7 @@ before_script:
   - ganache-cli --port 18545 > ganache-cli.log &
   - git clone https://github.com/oceanprotocol/barge
   - cd barge
-  - export AQUARIUS_VERSION=unstable
+  - export AQUARIUS_VERSION=v1.1.0
   - export BRIZO_VERSION=v0.9.7
   - export KEEPER_VERSION=v0.13.2
   - export EVENTS_HANDLER_VERSION=v0.4.7

src/aquarius/Aquarius.ts

@@ -261,6 +261,60 @@ export class Aquarius {
         return result
     }
 
+    /**
+     * Update Compute Privacy
+     * @param  {DID | string} did DID of the asset to update.
+     * @param  {number } serviceIndex Service index
+     * @param  {boolean} allowRawAlgorithm Allow Raw Algorithms
+     * @param  {boolean} allowNetworkAccess Allow Raw Algorithms
+     * @param  {String[]} trustedAlgorithms Allow Raw Algorithms
+     * @param  {String} updated Updated field of the DDO
+     * @param  {String} signature Signature using updated field to verify that the consumer has rights
+     * @return {Promise<String>} Result.
+     */
+    public async updateComputePrivacy(
+        did: DID | string,
+        serviceIndex: number,
+        allowRawAlgorithm: boolean,
+        allowNetworkAccess: boolean,
+        trustedAlgorithms: string[],
+        updated: string,
+        signature: string
+    ): Promise<string> {
+        did = did && DID.parse(did)
+        const fullUrl = `${this.url}${apiPath}/computePrivacy/update/${did.getDid()}`
+        const result = await this.fetch
+            .put(
+                fullUrl,
+                JSON.stringify({
+                    signature: signature,
+                    updated: updated,
+                    serviceIndex: serviceIndex,
+                    allowRawAlgorithm: allowRawAlgorithm,
+                    allowNetworkAccess: allowNetworkAccess,
+                    trustedAlgorithms: trustedAlgorithms
+                })
+            )
+            .then((response: any) => {
+                if (response.ok) {
+                    return response.text
+                }
+                this.logger.log(
+                    'update compute privacy failed:',
+                    response.status,
+                    response.statusText
+                )
+                return null
+            })
+
+            .catch(error => {
+                this.logger.error('Error updating compute privacy: ', error)
+                return null
+            })
+
+        return result
+    }
+
     /**
      * Edit Metadata for a DDO.
      * @param  {did} string DID.

src/ddo/Service.ts

@@ -25,6 +25,11 @@ export interface ServiceAccessAttributes extends ServiceCommonAttributes {
         timeout: number
     }
 }
+export interface ServiceComputePrivacy {
+    allowRawAlgorithm: boolean
+    allowNetworkAccess: boolean
+    trustedAlgorithms: string[]
+}
 
 export interface ServiceComputeAttributes extends ServiceCommonAttributes {
     main: {
@@ -34,6 +39,7 @@ export interface ServiceComputeAttributes extends ServiceCommonAttributes {
         timeout: number
         provider?: ServiceComputeProvider
         name: string
+        privacy?: ServiceComputePrivacy
     }
 }
 

src/ocean/OceanAssets.ts

@@ -2,7 +2,7 @@ import { TransactionReceipt } from 'web3-core'
 import { SearchQuery } from '../aquarius/Aquarius'
 import { DDO } from '../ddo/DDO'
 import { MetaData, EditableMetaData } from '../ddo/MetaData'
-import { Service, ServiceAccess } from '../ddo/Service'
+import { Service, ServiceAccess, ServiceComputePrivacy } from '../ddo/Service'
 import Account from './Account'
 import DID from './DID'
 import { fillConditionsWithDDO, SubscribablePromise, didZeroX } from '../utils'
@@ -414,6 +414,41 @@ export class OceanAssets extends Instantiable {
         return result
     }
 
+    /**
+     * Update Compute Privacy
+     * @param  {did} string DID.
+     * @param  {number} serviceIndex Index of the compute service in the DDO
+     * @param  {ServiceComputePrivacy} computePrivacy ComputePrivacy fields & new values.
+     * @param  {Account} account Ethereum account of owner to sign and prove the ownership.
+     * @return {Promise<string>}
+     */
+    public async updateComputePrivacy(
+        did: string,
+        serviceIndex: number,
+        computePrivacy: ServiceComputePrivacy,
+        account: Account
+    ): Promise<string> {
+        const oldDdo = await this.ocean.aquarius.retrieveDDO(did)
+        // get a signature
+        const signature = await this.ocean.utils.signature.signForAquarius(
+            oldDdo.updated,
+            account
+        )
+        let result = null
+        if (signature != null)
+            result = await this.ocean.aquarius.updateComputePrivacy(
+                did,
+                serviceIndex,
+                computePrivacy.allowRawAlgorithm,
+                computePrivacy.allowNetworkAccess,
+                computePrivacy.trustedAlgorithms,
+                oldDdo.updated,
+                signature
+            )
+
+        return result
+    }
+
     /**
      * Retire a DDO (Delete)
      * @param  {did} string DID.

src/ocean/OceanCompute.ts

@@ -5,7 +5,7 @@ import { DDO } from '../ddo/DDO'
 import { SubscribablePromise } from '../utils'
 import { OrderProgressStep } from './utils/ServiceUtils'
 import { DID } from '../squid'
-import { ServiceCompute } from '../ddo/Service'
+import { Service, ServiceCompute, ServiceComputePrivacy } from '../ddo/Service'
 
 export const ComputeJobStatus = Object.freeze({
     Started: 10,
@@ -66,16 +66,43 @@ export class OceanCompute extends Instantiable {
      * Starts an order of a compute service that is defined in an asset's services.
      * @param  {Account} consumerAccount The account of the consumer ordering the service.
      * @param  {string} datasetDid The DID of the dataset asset (of type `dataset`) to run the algorithm on.
+     * @param  {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset.
+     * @param  {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified.
      * @return {Promise<string>} Returns the Service Agreement ID, representation of `bytes32` ID.
+     *
+     * Note:  algorithmDid and algorithmMeta are optional, but if they are not passed,
+     * you can end up in the situation that you are ordering and paying for your agreement,
+     * but brizo will not allow the compute, due to privacy settings of the ddo
      */
     public order(
         consumerAccount: Account,
         datasetDid: string,
+        algorithmDid?: string,
+        algorithmMeta?: MetaDataAlgorithm,
         provider?: string
     ): SubscribablePromise<OrderProgressStep, string> {
         return new SubscribablePromise(async observer => {
             const { assets, keeper, utils } = this.ocean
             const ddo: DDO = await assets.resolve(datasetDid)
+            const service: Service = ddo.findServiceByType('compute')
+            if (!service) return null
+            if (algorithmMeta) {
+                // check if raw algo is allowed
+                if (service.attributes.main.privacy)
+                    if (!service.attributes.main.privacy.allowRawAlgorithm) return null
+            }
+            if (algorithmDid) {
+                // check if did is in trusted list
+                if (service.attributes.main.privacy)
+                    if (service.attributes.main.privacy.trustedAlgorithms)
+                        if (service.attributes.main.privacy.trustedAlgorithms.length > 0)
+                            if (
+                                !service.attributes.main.privacy.trustedAlgorithms.includes(
+                                    algorithmDid
+                                )
+                            )
+                                return null
+            }
             const condition = keeper.conditions.computeExecutionCondition
 
             const agreementId = await utils.services.order(
@@ -137,6 +164,7 @@ export class OceanCompute extends Instantiable {
         output?: Output
     ): Promise<ComputeJob> {
         output = this.checkOutput(consumerAccount, output)
+        if (agreementId) {
             const computeJobsList = await this.ocean.brizo.compute(
                 'post',
                 agreementId,
@@ -146,8 +174,8 @@ export class OceanCompute extends Instantiable {
                 undefined,
                 output
             )
-
             return computeJobsList[0] as ComputeJob
+        } else return null
     }
 
     /**
@@ -267,12 +295,14 @@ export class OceanCompute extends Instantiable {
         consumerAccount: Account,
         price: string,
         datePublished: string,
-        timeout: number = 3600
+        computePrivacy?: ServiceComputePrivacy,
+        timeout?: number
     ): Promise<ServiceCompute> {
         const { templates } = this.ocean.keeper
         const serviceAgreementTemplate = await templates.escrowComputeExecutionTemplate.getServiceAgreementTemplate()
         const name = 'dataAssetComputingServiceAgreement'
-        return {
+        if (!timeout) timeout = 3600
+        const service = {
             type: 'compute',
             index: 3,
             serviceEndpoint: this.ocean.brizo.getComputeEndpoint(),
@@ -282,11 +312,14 @@ export class OceanCompute extends Instantiable {
                     creator: consumerAccount.getId(),
                     datePublished,
                     price,
+                    privacy: {},
                     timeout: timeout,
                     name
                 },
                 serviceAgreementTemplate
             }
         }
+        if (computePrivacy) service.attributes.main.privacy = computePrivacy
+        return service as ServiceCompute
     }
 }

test/integration/ocean/AssetOwners.test.ts

@@ -2,6 +2,7 @@ import { assert } from 'chai'
 import { config } from '../config'
 import { getMetadata } from '../utils'
 import { Ocean, Account, EditableMetaData } from '../../../src' // @oceanprotocol/squid
+import { ServiceComputePrivacy } from '../../../src/ddo/Service'
 
 describe('Asset Owners', () => {
     let ocean: Ocean
@@ -202,4 +203,56 @@ describe('Asset Owners', () => {
         assert.equal(-1, remList.indexOf(consumer1.getId()))
         assert.notEqual(-1, remList.indexOf(consumer2.getId()))
     })
+
+    it('should be able to update computePrivacy', async () => {
+        const origComputePrivacy = {
+            allowRawAlgorithm: true,
+            allowNetworkAccess: true,
+            trustedAlgorithms: []
+        }
+
+        const newComputePrivacy = {
+            allowRawAlgorithm: false,
+            allowNetworkAccess: false,
+            trustedAlgorithms: ['did:op:123', 'did:op:1234']
+        }
+        const computeService = await ocean.compute.createComputeServiceAttributes(
+            account1,
+            '0',
+            '2020-03-10T10:00:00Z',
+            origComputePrivacy as ServiceComputePrivacy
+        )
+        const { id } = await ocean.assets.create(metadata as any, account1, [
+            computeService
+        ])
+
+        const oldDDO = await ocean.assets.resolve(id)
+        let serviceIndex = null
+
+        for (let index = 0; index < oldDDO.service.length; index++) {
+            if (oldDDO.service[index].type === 'compute') serviceIndex = index
+        }
+
+        await ocean.assets.updateComputePrivacy(
+            id,
+            serviceIndex,
+            newComputePrivacy as ServiceComputePrivacy,
+            account1
+        )
+
+        const newDDO = await ocean.assets.resolve(id)
+
+        assert.equal(
+            newDDO.service[serviceIndex].attributes.main.privacy.allowRawAlgorithm,
+            newComputePrivacy.allowRawAlgorithm
+        )
+        assert.equal(
+            newDDO.service[serviceIndex].attributes.main.privacy.allowNetworkAccess,
+            newComputePrivacy.allowNetworkAccess
+        )
+        assert.deepEqual(
+            newDDO.service[serviceIndex].attributes.main.privacy.trustedAlgorithms,
+            newComputePrivacy.trustedAlgorithms
+        )
+    })
 })

test/integration/ocean/Compute.test.ts

@@ -11,7 +11,7 @@ import {
     MetaDataAlgorithm
 } from '../../../src'
 import { getMetadata } from '../utils'
-import { ServiceCompute } from '../../../src/ddo/Service'
+import { ServiceCompute, ServiceComputePrivacy } from '../../../src/ddo/Service'
 
 const metadataAsset = getMetadata()
 const metadataAlgorithm = getMetadata(0, 'algorithm')
@@ -26,11 +26,24 @@ const customConfig: Config = {
     verbose: true
 }
 
+export const rawAlgoMeta = {
+    rawcode: `console.log('Hello world'!)`,
+    format: 'docker-image',
+    version: '0.1',
+    container: {
+        entrypoint: 'node $ALGO',
+        image: 'node',
+        tag: '10'
+    }
+}
+
 describe('Compute', () => {
     let ocean: Ocean
     let account: Account
     let agreementId: string
     let dataset: DDO
+    let datasetNoRawAlgo: DDO
+    let datasetWithTrustedAlgo: DDO
     let algorithm: DDO
     let computeService: ServiceCompute
     let jobId: string
@@ -63,6 +76,58 @@ describe('Compute', () => {
         assert.deepEqual(stepsAsset, [0, 1, 2, 3, 4, 5, 6, 7])
     })
 
+    it('should publish a dataset with a compute service object that does not allow rawAlgo', async () => {
+        const stepsAsset = []
+        const origComputePrivacy = {
+            allowRawAlgorithm: false,
+            allowNetworkAccess: false,
+            trustedAlgorithms: []
+        }
+
+        computeService = await ocean.compute.createComputeServiceAttributes(
+            account,
+            '1000',
+            metadataAsset.main.datePublished,
+            origComputePrivacy as ServiceComputePrivacy
+        )
+        datasetNoRawAlgo = await ocean.assets
+            .create(metadataAsset as MetaData, account, [computeService])
+            .next(step => stepsAsset.push(step))
+
+        assert.instanceOf(datasetNoRawAlgo, DDO)
+        assert.isDefined(
+            dataset.findServiceByType('compute'),
+            `DDO compute service doesn't exist`
+        )
+        assert.deepEqual(stepsAsset, [0, 1, 2, 3, 4, 5, 6, 7])
+    })
+
+    it('should publish a dataset with a compute service object that allows only algo with did:op:1234', async () => {
+        const stepsAsset = []
+        const origComputePrivacy = {
+            allowRawAlgorithm: false,
+            allowNetworkAccess: false,
+            trustedAlgorithms: ['did:op:1234']
+        }
+
+        computeService = await ocean.compute.createComputeServiceAttributes(
+            account,
+            '1000',
+            metadataAsset.main.datePublished,
+            origComputePrivacy as ServiceComputePrivacy
+        )
+        datasetWithTrustedAlgo = await ocean.assets
+            .create(metadataAsset as MetaData, account, [computeService])
+            .next(step => stepsAsset.push(step))
+
+        assert.instanceOf(datasetWithTrustedAlgo, DDO)
+        assert.isDefined(
+            dataset.findServiceByType('compute'),
+            `DDO compute service doesn't exist`
+        )
+        assert.deepEqual(stepsAsset, [0, 1, 2, 3, 4, 5, 6, 7])
+    })
+
     it('should publish an algorithm', async () => {
         const stepsAlgorithm = []
         algorithm = await ocean.assets
@@ -73,6 +138,26 @@ describe('Compute', () => {
         assert.deepEqual(stepsAlgorithm, [0, 1, 2, 3, 4, 5, 6, 7])
     })
 
+    it('should not allow order the compute service with raw algo for dataset that does not allow raw algo', async () => {
+        const steps = []
+
+        agreementId = await ocean.compute
+            .order(account, datasetNoRawAlgo.id, null, rawAlgoMeta)
+            .next(step => steps.push(step))
+
+        assert.equal(agreementId, null)
+    })
+
+    it('should not allow order the compute service with did != did:op:1234 for dataset that allows only did:op:1234 as algo', async () => {
+        const steps = []
+
+        agreementId = await ocean.compute
+            .order(account, datasetWithTrustedAlgo.id, 'did:op:233454', null)
+            .next(step => steps.push(step))
+
+        assert.equal(agreementId, null)
+    })
+
     it('should order the compute service of the dataset', async () => {
         const steps = []
         try {