From dad7a56c1e102c8666e7d44fcc2ed7e50ffc462f Mon Sep 17 00:00:00 2001 From: ssallam Date: Fri, 10 Jul 2020 16:17:57 +0200 Subject: [PATCH 01/14] wip: compute module, compute tests, create compute service. --- src/datatokens/Datatokens.ts | 6 +- src/ddo/interfaces/Service.ts | 6 +- src/ocean/Assets.ts | 215 ------------------- src/ocean/Compute.ts | 309 +++++++++++++++++++++++++++ src/ocean/Ocean.ts | 6 +- src/provider/Provider.ts | 11 + test/integration/ComputeFlow.test.ts | 73 ++++++- test/integration/Computeflow.test.ts | 188 ++++++++++++++++ 8 files changed, 584 insertions(+), 230 deletions(-) create mode 100644 src/ocean/Compute.ts create mode 100644 test/integration/Computeflow.test.ts diff --git a/src/datatokens/Datatokens.ts b/src/datatokens/Datatokens.ts index d8d1201d..b8f90f99 100644 --- a/src/datatokens/Datatokens.ts +++ b/src/datatokens/Datatokens.ts @@ -40,9 +40,9 @@ export class DataTokens { */ public async create(metaDataStoreURI: string, account: Account): Promise { // Create factory contract object - const factory = new this.web3.eth.Contract(this.factoryABI, this.factoryAddress, { - from: account - }) + const factory = new this.web3.eth.Contract( + this.factoryABI, this.factoryAddress, {from: account} + ) const estGas = await factory.methods .createToken(metaDataStoreURI) .estimateGas(function (err, estGas) { diff --git a/src/ddo/interfaces/Service.ts b/src/ddo/interfaces/Service.ts index c60d7c05..6e69c86b 100644 --- a/src/ddo/interfaces/Service.ts +++ b/src/ddo/interfaces/Service.ts @@ -33,7 +33,7 @@ export interface ServiceComputeAttributes extends ServiceCommonAttributes { main: { creator: string datePublished: string - price: string + cost: string timeout: number provider?: ServiceComputeProvider name: string @@ -57,7 +57,7 @@ export interface ServiceComputeProvider { supportedServers: { serverId: string serverType: string - price: string + cost: string cpu: string gpu: string memory: string @@ -74,13 +74,11 @@ export interface ServiceMetadata extends ServiceCommon { export interface ServiceAccess extends ServiceCommon { type: 'access' - templateId?: string attributes: ServiceAccessAttributes } export interface ServiceCompute extends ServiceCommon { type: 'compute' - templateId?: string attributes: ServiceComputeAttributes } diff --git a/src/ocean/Assets.ts b/src/ocean/Assets.ts index 1a8bf501..1143d54c 100644 --- a/src/ocean/Assets.ts +++ b/src/ocean/Assets.ts @@ -33,20 +33,6 @@ export enum OrderProgressStep { TransferDataToken } -export const ComputeJobStatus = Object.freeze({ - Started: 10, - ConfiguringVolumes: 20, - ProvisioningSuccess: 30, - DataProvisioningFailed: 31, - AlgorithmProvisioningFailed: 32, - RunningAlgorithm: 40, - FilteringResults: 50, - PublishingResult: 60, - Completed: 70, - Stopped: 80, - Deleted: 90 -}) - /** * Assets submodule of Ocean Protocol. */ @@ -458,205 +444,4 @@ export class Assets extends Instantiable { return serviceEndpoint } - - /** - * Start the execution of a compute job. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifer for the asset - * @param {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset. - * @param {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified. - * @param {Output} output Define algorithm output publishing. Publishing the result of a compute job is turned off by default. - * @return {Promise} Returns compute job ID under status.jobId - */ - public async start( - consumerAccount: Account, - did: string, - algorithmDid?: string, - algorithmMeta?: MetadataAlgorithm, - output?: Output - ): Promise { - output = this.checkOutput(consumerAccount, output) - if (did) { - const computeJobsList = await this.ocean.provider.compute( - 'post', - did, - consumerAccount, - algorithmDid, - algorithmMeta, - undefined, - output - ) - return computeJobsList[0] as ComputeJob - } else return null - } - - /** - * Ends a running compute job. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifier. - * @param {string} jobId The ID of the compute job to be stopped - * @return {Promise} Returns the new status of a job - */ - public async stop( - consumerAccount: Account, - did: string, - jobId: string - ): Promise { - const computeJobsList = await this.ocean.provider.compute( - 'put', - did, - consumerAccount, - undefined, - undefined, - jobId - ) - - return computeJobsList[0] as ComputeJob - } - - /** - * Deletes a compute job and all resources associated with the job. If job is running it will be stopped first. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifier. - * @param {string} jobId The ID of the compute job to be stopped - * @return {Promise} Returns the new status of a job - */ - public async delete( - consumerAccount: Account, - did: string, - jobId: string - ): Promise { - const computeJobsList = await this.ocean.provider.compute( - 'delete', - did, - consumerAccount, - undefined, - undefined, - jobId - ) - - return computeJobsList[0] as ComputeJob - } - - /** - * Ends a running compute job and starts it again. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifier. - * @param {string} jobId The ID of the compute job to be stopped - * @return {Promise} Returns the new status of a job - */ - public async restart( - consumerAccount: Account, - did: string, - jobId: string - ): Promise { - await this.stop(consumerAccount, did, jobId) - const result = await this.start(consumerAccount, did, jobId) - return result - } - - /** - * Returns information about the status of all compute jobs, or a single compute job. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifier. - * @param {string} jobId The ID of the compute job to be stopped - * @return {Promise} Returns the status - */ - public async status( - consumerAccount: Account, - did?: string, - jobId?: string - ): Promise { - const computeJobsList = await this.ocean.provider.compute( - 'get', - did, - consumerAccount, - undefined, - undefined, - jobId - ) - - return computeJobsList as ComputeJob[] - } - - /** - * Returns the final result of a specific compute job published as an asset. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifier. - * @param {string} jobId The ID of the compute job to be stopped. - * @return {Promise} Returns the DDO of the result asset. - */ - public async result( - consumerAccount: Account, - did: string, - jobId: string - ): Promise { - const computeJobsList = await this.ocean.provider.compute( - 'get', - did, - consumerAccount, - undefined, - undefined, - jobId - ) - - return computeJobsList[0] as ComputeJob - } - - public async createComputeServiceAttributes( - consumerAccount: Account, - price: string, - datePublished: string, - computePrivacy?: ServiceComputePrivacy, - timeout?: number - ): Promise { - const name = 'dataAssetComputingService' - if (!timeout) timeout = 3600 - // TODO - const service = { - type: 'compute', - index: 3, - serviceEndpoint: this.ocean.provider.getComputeEndpoint(), - attributes: { - main: { - creator: consumerAccount.getId(), - datePublished, - price, - privacy: {}, - timeout: timeout, - name - } - } - } - if (computePrivacy) service.attributes.main.privacy = computePrivacy - return service as ServiceCompute - } - - /** - * Check the output object and add default properties if needed - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {Output} output Output section used for publishing the result. - * @return {Promise} Returns output object - */ - private checkOutput(consumerAccount: Account, output?: Output): Output { - const isDefault = - !output || (!output.publishAlgorithmLog && !output.publishOutput) - - if (isDefault) { - return { - publishAlgorithmLog: false, - publishOutput: false - } - } - - return { - publishAlgorithmLog: output.publishAlgorithmLog, - publishOutput: output.publishOutput, - providerAddress: output.providerAddress || this.config.providerAddress, - providerUri: output.providerUri || this.config.providerUri, - metadataUri: output.metadataUri || this.config.metadataStoreUri, - nodeUri: output.nodeUri || this.config.nodeUri, - owner: output.owner || consumerAccount.getId() - } - } } diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts new file mode 100644 index 00000000..2b62c5e4 --- /dev/null +++ b/src/ocean/Compute.ts @@ -0,0 +1,309 @@ +import { SearchQuery } from '../metadatastore/MetadataStore' +import { DDO } from '../ddo/DDO' +import { Metadata } from '../ddo/interfaces/Metadata' +import { MetadataAlgorithm } from '../ddo/interfaces/MetadataAlgorithm' +import { + Service, + ServiceComputePrivacy, + ServiceCompute +} from '../ddo/interfaces/Service' +import { EditableMetadata } from '../ddo/interfaces/EditableMetadata' +import Account from './Account' +import DID from './DID' +import { SubscribablePromise } from '../utils' +import { Instantiable, InstantiableConfig } from '../Instantiable.abstract' +import {Output} from "./interfaces/ComputeOutput"; +import {ComputeJob} from "./interfaces/ComputeJob"; +// import { WebServiceConnector } from './utils/WebServiceConnector' +// import { Output } from './interfaces/ComputeOutput' +// import { ComputeJob } from './interfaces/ComputeJob' + +export enum OrderProgressStep { + TransferDataToken +} + +export const ComputeJobStatus = Object.freeze({ + Started: 10, + ConfiguringVolumes: 20, + ProvisioningSuccess: 30, + DataProvisioningFailed: 31, + AlgorithmProvisioningFailed: 32, + RunningAlgorithm: 40, + FilteringResults: 50, + PublishingResult: 60, + Completed: 70, + Stopped: 80, + Deleted: 90 +}) + +/** + * Compute submodule of Ocean Protocol. + */ +export class Compute extends Instantiable { + /** + * Returns the instance of Compute. + * @return {Promise} + */ + public static async getInstance(config: InstantiableConfig): Promise { + const instance = new Compute() + instance.setInstanceConfig(config) + + return instance + } + + /** + * Start the execution of a compute job. + * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {string} did Decentralized identifer for the asset + * @param {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset. + * @param {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified. + * @param {Output} output Define algorithm output publishing. Publishing the result of a compute job is turned off by default. + * @return {Promise} Returns compute job ID under status.jobId + */ + public async start( + consumerAccount: Account, + did: string, + algorithmDid?: string, + algorithmMeta?: MetadataAlgorithm, + output?: Output + ): Promise { + output = this.checkOutput(consumerAccount, output) + if (did) { + const computeJobsList = await this.ocean.provider.compute( + 'post', + did, + consumerAccount, + algorithmDid, + algorithmMeta, + undefined, + output + ) + return computeJobsList[0] as ComputeJob + } else return null + } + + /** + * Ends a running compute job. + * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {string} did Decentralized identifier. + * @param {string} jobId The ID of the compute job to be stopped + * @return {Promise} Returns the new status of a job + */ + public async stop( + consumerAccount: Account, + did: string, + jobId: string + ): Promise { + const computeJobsList = await this.ocean.provider.compute( + 'put', + did, + consumerAccount, + undefined, + undefined, + jobId + ) + + return computeJobsList[0] as ComputeJob + } + + /** + * Returns information about the status of all compute jobs, or a single compute job. + * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {string} did Decentralized identifier. + * @param {string} jobId The ID of the compute job to be stopped + * @return {Promise} Returns the status + */ + public async status( + consumerAccount: Account, + did?: string, + jobId?: string + ): Promise { + const computeJobsList = await this.ocean.provider.compute( + 'get', + did, + consumerAccount, + undefined, + undefined, + jobId + ) + + return computeJobsList as ComputeJob[] + } + + /** + * Returns the final result of a specific compute job published as an asset. + * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {string} did Decentralized identifier. + * @param {string} jobId The ID of the compute job to be stopped. + * @return {Promise} Returns the DDO of the result asset. + */ + public async result( + consumerAccount: Account, + did: string, + jobId: string + ): Promise { + const computeJobsList = await this.ocean.provider.compute( + 'get', + did, + consumerAccount, + undefined, + undefined, + jobId + ) + + return computeJobsList[0] as ComputeJob + } + + public createServerAttributes( + serverId: string, serverType: string, cost: string, + cpu: string, gpu: string, memory: string, + disk: string, maxExecutionTime: number + ): object { + return { + serverId, serverType, cost, cpu, gpu, memory, disk, maxExecutionTime + } + } + + public createContainerAttributes(image: string, tag: string, checksum: string): object { + return {image, tag, checksum} + } + + public createClusterAttributes(type: string, url: string): object { + return {type, url} + } + + public createProviderAttributes( + type: string, description: string, cluster: object, containers: object[], servers: object[] + ): object { + return { + type, + description, + environment: { + cluster: cluster, + supportedServers: containers, + supportedContainers: servers + } + } + } + + public createComputeService( + consumerAccount: Account, + cost: string, + datePublished: string, + providerAttributes: object, + computePrivacy?: ServiceComputePrivacy, + timeout?: number, + ): ServiceCompute { + const name = 'dataAssetComputingService' + if (!timeout) timeout = 3600 + + const service = { + type: 'compute', + index: 3, + serviceEndpoint: this.ocean.provider.getComputeEndpoint(), + attributes: { + main: { + name, + creator: consumerAccount.getId(), + datePublished, + cost, + timeout: timeout, + provider: providerAttributes, + privacy: {} + } + } + } + + if (computePrivacy) + service.attributes.main.privacy = computePrivacy + + return service as ServiceCompute + } + + /** + * Check the output object and add default properties if needed + * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {Output} output Output section used for publishing the result. + * @return {Promise} Returns output object + */ + private checkOutput(consumerAccount: Account, output?: Output): Output { + const isDefault = + !output || (!output.publishAlgorithmLog && !output.publishOutput) + + if (isDefault) { + return { + publishAlgorithmLog: false, + publishOutput: false + } + } + // 'signature': signature, + // 'documentId': did, + // 'serviceId': sa.index, + // 'serviceType': sa.type, + // 'consumerAddress': cons_acc.address, + // 'transferTxId': Web3.toHex(tx_id), + // 'dataToken': data_token, + // 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc), + // 'algorithmDid': alg_ddo.did, + // 'algorithmMeta': {}, + // 'algorithmDataToken': alg_data_token + + return { + publishAlgorithmLog: output.publishAlgorithmLog, + publishOutput: output.publishOutput, + providerAddress: output.providerAddress || this.config.providerAddress, + providerUri: output.providerUri || this.config.providerUri, + metadataUri: output.metadataUri || this.config.metadataStoreUri, + nodeUri: output.nodeUri || this.config.nodeUri, + owner: output.owner || consumerAccount.getId() + } + } + +} +// "creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", + // "datePublished": "2019-04-09T19:02:11Z", + // "cost": "10", + // "timeout": 86400, + // "provider": { + // "type": "Azure", + // "description": "", + // "environment": { + // "cluster": { + // "type": "Kubernetes", + // "url": "http://10.0.0.17/xxx" + // }, + // "supportedContainers": [ + // { + // "image": "tensorflow/tensorflow", + // "tag": "latest", + // "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" + // }, + // { + // "image": "tensorflow/tensorflow", + // "tag": "latest", + // "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" + // } + // ], + // "supportedServers": [ + // { + // "serverId": "1", + // "serverType": "xlsize", + // "cost": "50", + // "cpu": "16", + // "gpu": "0", + // "memory": "128gb", + // "disk": "160gb", + // "maxExecutionTime": 86400 + // }, + // { + // "serverId": "2", + // "serverType": "medium", + // "cost": "10", + // "cpu": "2", + // "gpu": "0", + // "memory": "8gb", + // "disk": "80gb", + // "maxExecutionTime": 86400 + // } + // ] + // } diff --git a/src/ocean/Ocean.ts b/src/ocean/Ocean.ts index e52f95a8..ac18104b 100644 --- a/src/ocean/Ocean.ts +++ b/src/ocean/Ocean.ts @@ -17,6 +17,7 @@ import { Instantiable, generateIntantiableConfigFromConfig } from '../Instantiable.abstract' +import {Compute} from "./Compute"; /** * Main interface for Ocean Protocol. @@ -47,7 +48,7 @@ export class Ocean extends Instantiable { instance.accounts = await Accounts.getInstance(instanceConfig) // instance.auth = await Auth.getInstance(instanceConfig) instance.assets = await Assets.getInstance(instanceConfig) - // instance.compute = await Compute.getInstance(instanceConfig) + instance.compute = await Compute.getInstance(instanceConfig) instance.datatokens = new DataTokens( instanceConfig.config.factoryAddress, instanceConfig.config.factoryABI, @@ -105,9 +106,8 @@ export class Ocean extends Instantiable { /** * Ocean compute submodule * @type {Compute} - + */ public compute: Compute - */ /** * Ocean secretStore submodule diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index 9c14a763..263f06ad 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -174,6 +174,17 @@ export class Provider extends Instantiable { url += `&serviceType=${serviceType}` || '' url += `&dataToken=${tokenAddress}` || '' url += `&consumerAddress=${consumerAccount.getId()}` || '' + // 'signature': signature, + // 'documentId': did, + // 'serviceId': sa.index, + // 'serviceType': sa.type, + // 'consumerAddress': cons_acc.address, + // 'transferTxId': Web3.toHex(tx_id), + // 'dataToken': data_token, + // 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc), + // 'algorithmDid': alg_ddo.did, + // 'algorithmMeta': {}, + // 'algorithmDataToken': alg_data_token // switch fetch method let fetch diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 33bd3ed8..c13ddbca 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -22,9 +22,10 @@ describe('Marketplace flow', () => { let service1 let price let ocean - let accessService + let computeService let data let blob + const dateCreated = new Date(Date.now()).toISOString().split('.')[0] + 'Z' // remove milliseconds const marketplaceAllowance = 20 const tokenAmount = 100 @@ -60,13 +61,75 @@ describe('Marketplace flow', () => { assert(tokenAddress != null) }) - // it('Alice publishes dataset with a compute service', async () => {}) + it('Generates metadata', async () => { + asset = { + main: { + type: 'dataset', + name: 'UK Weather information 2011', + dateCreated: dateCreated, + author: 'Met Office', + license: 'CC-BY', + files: [ + { + url:'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + checksum: 'efb2c764274b745f5fc37f97c6b0e764', + contentLength: '4535431', + contentType: 'text/csv', + encoding: 'UTF-8', + compression: 'zip' + } + ] + } + } + }) - // it('Alice mints 100 DTs and tranfers them to the compute marketplace', async () => {}) + it('Alice publishes dataset with a compute service', async () => { + price = 10 // in datatoken + const timeout = 86400 + const cluster = ocean.compute.createClusterAttributes('Kubernetes', 'http://10.0.0.17/xxx') + const servers = [ + ocean.compute.createServerAttributes('1', 'xlsize', '50', '16', '0', '128gb', '160gb', timeout) + ] + const containers = [ + ocean.compute.createContainerAttributes( + 'tensorflow/tensorflow', + 'latest', + 'sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc' + ) + ] + const provider = ocean.compute.createProviderAttributes( + 'Azure', + 'Compute service with 16gb ram for each node.', + cluster, + containers, + servers + ) + const computeService = ocean.compute.createComputeService( + alice, price, dateCreated, provider + ) + ddo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) + assert(ddo.dataToken === tokenAddress) - // it('Markeplace post compute service for sale', async () => {}) + }) - // it('Bob buys datatokens from open market and order a compute service', async () => {}) + it('Alice mints 100 DTs and tranfers them to the compute marketplace', async () => { + await datatoken.mint(tokenAddress, alice.getId(), tokenAmount) + }) + + it('Marketplace posts compute service for sale', async () => { + computeService = await ocean.assets.getServiceByType(ddo.id, 'compute') + assert(computeService.attributes.main.cost === price) + }) + + it('Bob buys datatokens from open market and order a compute service', async () => { + const dTamount = 20 + await datatoken + .transfer(tokenAddress, bob.getId(), dTamount, alice.getId()) + .then(async () => { + const balance = await datatoken.balance(tokenAddress, bob.getId()) + assert(balance.toString() === dTamount.toString()) + }) + }) // it('Bob starts compute job', async () => {}) diff --git a/test/integration/Computeflow.test.ts b/test/integration/Computeflow.test.ts new file mode 100644 index 00000000..00ddf9bf --- /dev/null +++ b/test/integration/Computeflow.test.ts @@ -0,0 +1,188 @@ +import { TestContractHandler } from '../TestContractHandler' +import { DataTokens } from '../../src/datatokens/Datatokens' +import { Ocean } from '../../src/ocean/Ocean' +import config from './config' +import { assert } from 'console' + +const Web3 = require('web3') +const web3 = new Web3('http://127.0.0.1:8545') +const factory = require('@oceanprotocol/contracts/artifacts/development/Factory.json') +const datatokensTemplate = require('@oceanprotocol/contracts/artifacts/development/DataTokenTemplate.json') + +describe('Compute-2-Data flow', () => { + let owner + let bob + let ddo + let alice + let asset + let marketplace + let contracts + let datatoken + let tokenAddress + let service1 + let price + let ocean + let accessService + let data + let blob + + const marketplaceAllowance = 20 + const tokenAmount = 100 + + describe('#test', () => { + it('Initialize Ocean contracts v3', async () => { + contracts = new TestContractHandler( + factory.abi, + datatokensTemplate.abi, + datatokensTemplate.bytecode, + factory.bytecode, + web3 + ) + + ocean = await Ocean.getInstance(config) + owner = (await ocean.accounts.list())[0] + alice = (await ocean.accounts.list())[1] + bob = (await ocean.accounts.list())[2] + marketplace = (await ocean.accounts.list())[3] + data = { t: 1, url: ocean.config.metadataStoreUri } + blob = JSON.stringify(data) + await contracts.deployContracts(owner.getId()) + }) + + it('Alice publishes a datatoken contract', async () => { + datatoken = new DataTokens( + contracts.factoryAddress, + factory.abi, + datatokensTemplate.abi, + web3 + ) + tokenAddress = await datatoken.create(blob, alice.getId()) + assert(tokenAddress != null) + }) + + it('Generates metadata', async () => { + asset = { + main: { + type: 'dataset', + name: 'UK Weather information 2011', + dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds + author: 'Met Office', + license: 'CC-BY', + files: [ + { + url:'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + checksum: 'efb2c764274b745f5fc37f97c6b0e764', + contentLength: '4535431', + contentType: 'text/csv', + encoding: 'UTF-8', + compression: 'zip' + } + ] + } + } + }) + + it('Alice publishes a dataset', async () => { + price = 10 // in datatoken + const publishedDate = new Date(Date.now()).toISOString().split('.')[0] + 'Z' + const timeout = 0 + service1 = await ocean.assets.createAccessServiceAttributes( + alice, + price, + publishedDate, + timeout + ) + ddo = await ocean.assets.create(asset, alice, [service1], tokenAddress) + assert(ddo.dataToken === tokenAddress) + }) + + it('Alice mints 100 tokens', async () => { + await datatoken.mint(tokenAddress, alice.getId(), tokenAmount) + }) + + it('Alice allows marketplace to sell her datatokens', async () => { + await datatoken + .approve( + tokenAddress, + marketplace.getId(), + marketplaceAllowance, + alice.getId() + ) + .then(async () => { + const allowance = await datatoken.allowance( + tokenAddress, + alice.getId(), + marketplace.getId() + ) + assert(allowance.toString() === marketplaceAllowance.toString()) + }) + }) + + it('Marketplace withdraw Alice tokens from allowance', async () => { + const allowance = await datatoken.allowance( + tokenAddress, + alice.getId(), + marketplace.getId() + ) + await datatoken + .transferFrom(tokenAddress, alice.getId(), allowance, marketplace.getId()) + .then(async () => { + const marketplaceBalance = await datatoken.balance( + tokenAddress, + marketplace.getId() + ) + assert( + marketplaceBalance.toString() === marketplaceAllowance.toString() + ) + }) + }) + it('Marketplace should resolve asset using DID', async () => { + await ocean.assets.resolve(ddo.id).then((newDDO) => { + assert(newDDO.id === ddo.id) + }) + }) + + it('Marketplace posts asset for sale', async () => { + accessService = await ocean.assets.getServiceByType(ddo.id, 'access') + price = 20 + assert(accessService.attributes.main.cost * price === 200) + }) + + it('Bob gets datatokens', async () => { + const dTamount = 20 + await datatoken + .transfer(tokenAddress, bob.getId(), dTamount, alice.getId()) + .then(async () => { + const balance = await datatoken.balance(tokenAddress, bob.getId()) + assert(balance.toString() === dTamount.toString()) + }) + }) + + it('Bob consumes asset 1', async () => { + await ocean.assets + .order(ddo.id, accessService.type, bob.getId()) + .then(async (res: string) => { + res = JSON.parse(res) + return await datatoken.transfer( + res['dataToken'], + res['to'], + res['numTokens'], + res['from'] + ) + }) + .then(async (tx) => { + await ocean.assets.download( + ddo.id, + tx.transactionHash, + tokenAddress, + bob, + './node_modules/my-datasets' + ) + }) + }) + it('owner can list there assets', async () => { + const assets = await ocean.assets.ownerAssets(alice.getId()) + assert(assets.length > 0) + }) + }) +}) From 3179a233757a5c0c5d9ded53f1bd9fb181b1d696 Mon Sep 17 00:00:00 2001 From: ssallam Date: Fri, 10 Jul 2020 16:24:53 +0200 Subject: [PATCH 02/14] Remove extra test. --- test/integration/Computeflow.test.ts | 188 --------------------------- 1 file changed, 188 deletions(-) delete mode 100644 test/integration/Computeflow.test.ts diff --git a/test/integration/Computeflow.test.ts b/test/integration/Computeflow.test.ts deleted file mode 100644 index 00ddf9bf..00000000 --- a/test/integration/Computeflow.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { TestContractHandler } from '../TestContractHandler' -import { DataTokens } from '../../src/datatokens/Datatokens' -import { Ocean } from '../../src/ocean/Ocean' -import config from './config' -import { assert } from 'console' - -const Web3 = require('web3') -const web3 = new Web3('http://127.0.0.1:8545') -const factory = require('@oceanprotocol/contracts/artifacts/development/Factory.json') -const datatokensTemplate = require('@oceanprotocol/contracts/artifacts/development/DataTokenTemplate.json') - -describe('Compute-2-Data flow', () => { - let owner - let bob - let ddo - let alice - let asset - let marketplace - let contracts - let datatoken - let tokenAddress - let service1 - let price - let ocean - let accessService - let data - let blob - - const marketplaceAllowance = 20 - const tokenAmount = 100 - - describe('#test', () => { - it('Initialize Ocean contracts v3', async () => { - contracts = new TestContractHandler( - factory.abi, - datatokensTemplate.abi, - datatokensTemplate.bytecode, - factory.bytecode, - web3 - ) - - ocean = await Ocean.getInstance(config) - owner = (await ocean.accounts.list())[0] - alice = (await ocean.accounts.list())[1] - bob = (await ocean.accounts.list())[2] - marketplace = (await ocean.accounts.list())[3] - data = { t: 1, url: ocean.config.metadataStoreUri } - blob = JSON.stringify(data) - await contracts.deployContracts(owner.getId()) - }) - - it('Alice publishes a datatoken contract', async () => { - datatoken = new DataTokens( - contracts.factoryAddress, - factory.abi, - datatokensTemplate.abi, - web3 - ) - tokenAddress = await datatoken.create(blob, alice.getId()) - assert(tokenAddress != null) - }) - - it('Generates metadata', async () => { - asset = { - main: { - type: 'dataset', - name: 'UK Weather information 2011', - dateCreated: new Date(Date.now()).toISOString().split('.')[0] + 'Z', // remove milliseconds - author: 'Met Office', - license: 'CC-BY', - files: [ - { - url:'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', - checksum: 'efb2c764274b745f5fc37f97c6b0e764', - contentLength: '4535431', - contentType: 'text/csv', - encoding: 'UTF-8', - compression: 'zip' - } - ] - } - } - }) - - it('Alice publishes a dataset', async () => { - price = 10 // in datatoken - const publishedDate = new Date(Date.now()).toISOString().split('.')[0] + 'Z' - const timeout = 0 - service1 = await ocean.assets.createAccessServiceAttributes( - alice, - price, - publishedDate, - timeout - ) - ddo = await ocean.assets.create(asset, alice, [service1], tokenAddress) - assert(ddo.dataToken === tokenAddress) - }) - - it('Alice mints 100 tokens', async () => { - await datatoken.mint(tokenAddress, alice.getId(), tokenAmount) - }) - - it('Alice allows marketplace to sell her datatokens', async () => { - await datatoken - .approve( - tokenAddress, - marketplace.getId(), - marketplaceAllowance, - alice.getId() - ) - .then(async () => { - const allowance = await datatoken.allowance( - tokenAddress, - alice.getId(), - marketplace.getId() - ) - assert(allowance.toString() === marketplaceAllowance.toString()) - }) - }) - - it('Marketplace withdraw Alice tokens from allowance', async () => { - const allowance = await datatoken.allowance( - tokenAddress, - alice.getId(), - marketplace.getId() - ) - await datatoken - .transferFrom(tokenAddress, alice.getId(), allowance, marketplace.getId()) - .then(async () => { - const marketplaceBalance = await datatoken.balance( - tokenAddress, - marketplace.getId() - ) - assert( - marketplaceBalance.toString() === marketplaceAllowance.toString() - ) - }) - }) - it('Marketplace should resolve asset using DID', async () => { - await ocean.assets.resolve(ddo.id).then((newDDO) => { - assert(newDDO.id === ddo.id) - }) - }) - - it('Marketplace posts asset for sale', async () => { - accessService = await ocean.assets.getServiceByType(ddo.id, 'access') - price = 20 - assert(accessService.attributes.main.cost * price === 200) - }) - - it('Bob gets datatokens', async () => { - const dTamount = 20 - await datatoken - .transfer(tokenAddress, bob.getId(), dTamount, alice.getId()) - .then(async () => { - const balance = await datatoken.balance(tokenAddress, bob.getId()) - assert(balance.toString() === dTamount.toString()) - }) - }) - - it('Bob consumes asset 1', async () => { - await ocean.assets - .order(ddo.id, accessService.type, bob.getId()) - .then(async (res: string) => { - res = JSON.parse(res) - return await datatoken.transfer( - res['dataToken'], - res['to'], - res['numTokens'], - res['from'] - ) - }) - .then(async (tx) => { - await ocean.assets.download( - ddo.id, - tx.transactionHash, - tokenAddress, - bob, - './node_modules/my-datasets' - ) - }) - }) - it('owner can list there assets', async () => { - const assets = await ocean.assets.ownerAssets(alice.getId()) - assert(assets.length > 0) - }) - }) -}) From 7a47201ca465ed266d278812d52f10cfb5b22c26 Mon Sep 17 00:00:00 2001 From: ssallam Date: Fri, 10 Jul 2020 16:28:39 +0200 Subject: [PATCH 03/14] update compute.start interface. --- src/ocean/Compute.ts | 9 +++++-- test/integration/ComputeFlow.test.ts | 37 ++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 2b62c5e4..ae05e0a0 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -53,17 +53,22 @@ export class Compute extends Instantiable { /** * Start the execution of a compute job. - * @param {Account} consumerAccount The account of the consumer ordering the service. * @param {string} did Decentralized identifer for the asset + * @param {string} txId + * @param {string} tokenAddress + * @param {Account} consumerAccount The account of the consumer ordering the service. * @param {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset. * @param {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified. * @param {Output} output Define algorithm output publishing. Publishing the result of a compute job is turned off by default. * @return {Promise} Returns compute job ID under status.jobId */ public async start( - consumerAccount: Account, did: string, + txId: string, + tokenAddress: string, + consumerAccount: Account, algorithmDid?: string, + algorithmTokenAddress?: string, algorithmMeta?: MetadataAlgorithm, output?: Output ): Promise { diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index c13ddbca..ccc0b9df 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -121,7 +121,7 @@ describe('Marketplace flow', () => { assert(computeService.attributes.main.cost === price) }) - it('Bob buys datatokens from open market and order a compute service', async () => { + it('Bob gets datatokens from Alice to be able to try the compute service', async () => { const dTamount = 20 await datatoken .transfer(tokenAddress, bob.getId(), dTamount, alice.getId()) @@ -131,7 +131,40 @@ describe('Marketplace flow', () => { }) }) - // it('Bob starts compute job', async () => {}) + it('Bob starts compute job', async () => { + // TODO + const algorithmMeta = { + + } + // TODO + const output = { + + } + + await ocean.assets + .order(ddo.id, computeService.type, bob.getId()) + .then(async (res: string) => { + res = JSON.parse(res) + return await datatoken.transfer( + res['dataToken'], + res['to'], + res['numTokens'], + res['from'] + ) + }) + .then(async (tx) => { + await ocean.compute.start( + ddo.id, + tx.transactionHash, + tokenAddress, + bob, + algorithmMeta, + output + + ) + }) + + }) // it('Bob gets the compute job status', async () => {}) From 52962cc54d65c71ac1ddd1c54c322cef8d87353a Mon Sep 17 00:00:00 2001 From: ssallam Date: Fri, 10 Jul 2020 16:36:03 +0200 Subject: [PATCH 04/14] add algorithm meta in test. --- test/integration/ComputeFlow.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index ccc0b9df..2dd469df 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -132,11 +132,18 @@ describe('Marketplace flow', () => { }) it('Bob starts compute job', async () => { - // TODO const algorithmMeta = { - + 'language': 'scala', + 'format': 'docker-image', + 'version': '0.1', + 'url': 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', + 'container': { + 'entrypoint': 'node $ALGO', + 'image': 'node', + 'tag': '10' + } } - // TODO + const output = { } From 3071dc1de72d766c30f8334d5eadce6e2ca4c705 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 00:08:27 -0700 Subject: [PATCH 05/14] add more tests --- .travis.yml | 4 +- src/ocean/Compute.ts | 11 +- src/provider/Provider.ts | 2 +- test/integration/ComputeFlow.test.ts | 165 ++++++++++++++++++++++----- 4 files changed, 146 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 63b820c1..0e670ddc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_script: - git clone https://github.com/oceanprotocol/barge - cd barge - git checkout v3 - - export PROVIDER_VERSION=alex + - export PROVIDER_VERSION=phase2 - bash -x start_ocean.sh --no-dashboard 2>&1 > start_ocean.log & - cd .. - sleep 300 @@ -39,4 +39,4 @@ deploy: api_key: ${NPM_TOKEN} skip_cleanup: true on: - tags: true \ No newline at end of file + tags: true diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index ae05e0a0..98659140 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -70,7 +70,9 @@ export class Compute extends Instantiable { algorithmDid?: string, algorithmTokenAddress?: string, algorithmMeta?: MetadataAlgorithm, - output?: Output + output?: Output, + serviceIndex?: string, + serviceType?: string ): Promise { output = this.checkOutput(consumerAccount, output) if (did) { @@ -81,7 +83,12 @@ export class Compute extends Instantiable { algorithmDid, algorithmMeta, undefined, - output + output, + txId, + serviceIndex, + serviceType, + tokenAddress + ) return computeJobsList[0] as ComputeJob } else return null diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index 263f06ad..d02c7f90 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -87,7 +87,6 @@ export class Provider extends Instantiable { initializeUrl += `&serviceType=${serviceType}` initializeUrl += `&dataToken=${DDO.dataToken}` initializeUrl += `&consumerAddress=${consumerAddress}` - try { const response = await this.ocean.utils.fetch.get(initializeUrl) return await response.text() @@ -187,6 +186,7 @@ export class Provider extends Instantiable { // 'algorithmDataToken': alg_data_token // switch fetch method + let fetch switch (method) { diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 2dd469df..711ee119 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -3,7 +3,12 @@ import { DataTokens } from '../../src/datatokens/Datatokens' import { Ocean } from '../../src/ocean/Ocean' import config from './config' import { assert } from 'console' - +import {ComputeJob} from "../../src/ocean/interfaces/ComputeJob"; +import { + Service, + ServiceComputePrivacy, + ServiceCompute +} from '../../src/ddo/interfaces/Service' const Web3 = require('web3') const web3 = new Web3('http://127.0.0.1:8545') const factory = require('@oceanprotocol/contracts/artifacts/development/Factory.json') @@ -15,6 +20,9 @@ describe('Marketplace flow', () => { let ddo let alice let asset + let datasetNoRawAlgo + let datasetWithTrustedAlgo + let algorithmAsset let marketplace let contracts let datatoken @@ -25,11 +33,20 @@ describe('Marketplace flow', () => { let computeService let data let blob + let jobId + + let cluster + let servers + let containers + let provider + const dateCreated = new Date(Date.now()).toISOString().split('.')[0] + 'Z' // remove milliseconds const marketplaceAllowance = 20 const tokenAmount = 100 + const timeout = 86400 + describe('#MarketplaceComputeFlow-Test', () => { it('Initialize Ocean contracts v3', async () => { contracts = new TestContractHandler( @@ -85,25 +102,25 @@ describe('Marketplace flow', () => { it('Alice publishes dataset with a compute service', async () => { price = 10 // in datatoken - const timeout = 86400 - const cluster = ocean.compute.createClusterAttributes('Kubernetes', 'http://10.0.0.17/xxx') - const servers = [ + cluster = ocean.compute.createClusterAttributes('Kubernetes', 'http://10.0.0.17/xxx') + servers = [ ocean.compute.createServerAttributes('1', 'xlsize', '50', '16', '0', '128gb', '160gb', timeout) ] - const containers = [ + containers = [ ocean.compute.createContainerAttributes( 'tensorflow/tensorflow', 'latest', 'sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc' ) ] - const provider = ocean.compute.createProviderAttributes( + provider = ocean.compute.createProviderAttributes( 'Azure', 'Compute service with 16gb ram for each node.', cluster, containers, servers ) + const computeService = ocean.compute.createComputeService( alice, price, dateCreated, provider ) @@ -112,6 +129,83 @@ describe('Marketplace flow', () => { }) + //alex + it('should publish a dataset with a compute service object that does not allow rawAlgo', async () => { + const origComputePrivacy = { + allowRawAlgorithm: false, + allowNetworkAccess: false, + trustedAlgorithms: [] + } + + const computeService = ocean.compute.createComputeService( + alice, + '1000', + dateCreated, + provider, + origComputePrivacy as ServiceComputePrivacy + ) + datasetNoRawAlgo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) + assert(datasetNoRawAlgo.dataToken === tokenAddress) + }) + + it('should publish a dataset with a compute service object that allows only algo with did:op:1234', async () => { + const origComputePrivacy = { + allowRawAlgorithm: false, + allowNetworkAccess: false, + trustedAlgorithms: ['did:op:1234'] + } + + const computeService = ocean.compute.createComputeService( + alice, + '1000', + dateCreated, + provider, + origComputePrivacy as ServiceComputePrivacy + ) + datasetWithTrustedAlgo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) + assert(datasetWithTrustedAlgo.dataToken === tokenAddress) + }) + + it('should publish an algorithm', async () => { + const algoAsset = { + main: { + type: 'algorithm', + name: 'Test Algo', + dateCreated: dateCreated, + author: 'DevOps', + license: 'CC-BY', + files: [ + { + url:'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', + contentType: 'text/js', + encoding: 'UTF-8' + + } + ], + "algorithm": { + "language": "js", + "format": "docker-image", + "version": "0.1", + "container": { + "entrypoint": "node $ALGO", + "image": "node", + "tag": "10" + } + }, + } + } + const service1 = await ocean.assets.createAccessServiceAttributes( + alice, + price, + dateCreated, + 0 + ) + algorithmAsset = await ocean.assets.create(algoAsset, alice,[service1],tokenAddress) + assert(algorithmAsset.dataToken === tokenAddress) + + }) + + it('Alice mints 100 DTs and tranfers them to the compute marketplace', async () => { await datatoken.mint(tokenAddress, alice.getId(), tokenAmount) }) @@ -147,33 +241,42 @@ describe('Marketplace flow', () => { const output = { } - - await ocean.assets - .order(ddo.id, computeService.type, bob.getId()) - .then(async (res: string) => { - res = JSON.parse(res) - return await datatoken.transfer( - res['dataToken'], - res['to'], - res['numTokens'], - res['from'] - ) - }) - .then(async (tx) => { - await ocean.compute.start( - ddo.id, - tx.transactionHash, - tokenAddress, - bob, - algorithmMeta, - output - - ) - }) - + let order = await ocean.assets.order(ddo.id, computeService.type, bob.getId()) + let computeOrder=JSON.parse(order) + let tx=await datatoken.transfer( + computeOrder["dataToken"], + computeOrder["to"], + computeOrder["numTokens"], + computeOrder["from"] + ) + const response = await ocean.compute.start( + ddo.id, + tx.transactionHash, + tokenAddress, + bob, + undefined, + undefined, + algorithmMeta, + output, + computeService.index, + computeService.type + ) + jobId = response.jobId + assert(response.status>=10) + }) - // it('Bob gets the compute job status', async () => {}) + it('Bob should get status of a compute job', async () => { + const response = await ocean.compute.status(bob, ddo.id, jobId) + assert(response[0].jobId==jobId) + }) + + it('should get status of all compute jobs for an address', async () => { + const response = await ocean.compute.status(bob, undefined, undefined) + assert(response.length>0) + + }) + // it('Bob restarts compute job', async () => {}) From 8174709bbf3d89def8b102d34b0d5ee4c3584f6b Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 00:10:47 -0700 Subject: [PATCH 06/14] add more tests --- src/datatokens/Datatokens.ts | 6 +- src/ocean/Compute.ts | 149 +++++++++++++++------------ src/ocean/Ocean.ts | 2 +- src/provider/Provider.ts | 2 +- test/integration/ComputeFlow.test.ts | 142 +++++++++++++++---------- 5 files changed, 172 insertions(+), 129 deletions(-) diff --git a/src/datatokens/Datatokens.ts b/src/datatokens/Datatokens.ts index b8f90f99..d8d1201d 100644 --- a/src/datatokens/Datatokens.ts +++ b/src/datatokens/Datatokens.ts @@ -40,9 +40,9 @@ export class DataTokens { */ public async create(metaDataStoreURI: string, account: Account): Promise { // Create factory contract object - const factory = new this.web3.eth.Contract( - this.factoryABI, this.factoryAddress, {from: account} - ) + const factory = new this.web3.eth.Contract(this.factoryABI, this.factoryAddress, { + from: account + }) const estGas = await factory.methods .createToken(metaDataStoreURI) .estimateGas(function (err, estGas) { diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 98659140..7a132d1a 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -2,18 +2,14 @@ import { SearchQuery } from '../metadatastore/MetadataStore' import { DDO } from '../ddo/DDO' import { Metadata } from '../ddo/interfaces/Metadata' import { MetadataAlgorithm } from '../ddo/interfaces/MetadataAlgorithm' -import { - Service, - ServiceComputePrivacy, - ServiceCompute -} from '../ddo/interfaces/Service' +import { Service, ServiceComputePrivacy, ServiceCompute } from '../ddo/interfaces/Service' import { EditableMetadata } from '../ddo/interfaces/EditableMetadata' import Account from './Account' import DID from './DID' import { SubscribablePromise } from '../utils' import { Instantiable, InstantiableConfig } from '../Instantiable.abstract' -import {Output} from "./interfaces/ComputeOutput"; -import {ComputeJob} from "./interfaces/ComputeJob"; +import { Output } from './interfaces/ComputeOutput' +import { ComputeJob } from './interfaces/ComputeJob' // import { WebServiceConnector } from './utils/WebServiceConnector' // import { Output } from './interfaces/ComputeOutput' // import { ComputeJob } from './interfaces/ComputeJob' @@ -72,7 +68,7 @@ export class Compute extends Instantiable { algorithmMeta?: MetadataAlgorithm, output?: Output, serviceIndex?: string, - serviceType?: string + serviceType?: string ): Promise { output = this.checkOutput(consumerAccount, output) if (did) { @@ -88,7 +84,6 @@ export class Compute extends Instantiable { serviceIndex, serviceType, tokenAddress - ) return computeJobsList[0] as ComputeJob } else return null @@ -167,25 +162,45 @@ export class Compute extends Instantiable { } public createServerAttributes( - serverId: string, serverType: string, cost: string, - cpu: string, gpu: string, memory: string, - disk: string, maxExecutionTime: number + serverId: string, + serverType: string, + cost: string, + cpu: string, + gpu: string, + memory: string, + disk: string, + maxExecutionTime: number ): object { return { - serverId, serverType, cost, cpu, gpu, memory, disk, maxExecutionTime + serverId, + serverType, + cost, + cpu, + gpu, + memory, + disk, + maxExecutionTime } } - public createContainerAttributes(image: string, tag: string, checksum: string): object { - return {image, tag, checksum} + public createContainerAttributes( + image: string, + tag: string, + checksum: string + ): object { + return { image, tag, checksum } } public createClusterAttributes(type: string, url: string): object { - return {type, url} + return { type, url } } public createProviderAttributes( - type: string, description: string, cluster: object, containers: object[], servers: object[] + type: string, + description: string, + cluster: object, + containers: object[], + servers: object[] ): object { return { type, @@ -204,7 +219,7 @@ export class Compute extends Instantiable { datePublished: string, providerAttributes: object, computePrivacy?: ServiceComputePrivacy, - timeout?: number, + timeout?: number ): ServiceCompute { const name = 'dataAssetComputingService' if (!timeout) timeout = 3600 @@ -226,13 +241,12 @@ export class Compute extends Instantiable { } } - if (computePrivacy) - service.attributes.main.privacy = computePrivacy + if (computePrivacy) service.attributes.main.privacy = computePrivacy return service as ServiceCompute } - /** + /** * Check the output object and add default properties if needed * @param {Account} consumerAccount The account of the consumer ordering the service. * @param {Output} output Output section used for publishing the result. @@ -270,52 +284,51 @@ export class Compute extends Instantiable { owner: output.owner || consumerAccount.getId() } } - } // "creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", - // "datePublished": "2019-04-09T19:02:11Z", - // "cost": "10", - // "timeout": 86400, - // "provider": { - // "type": "Azure", - // "description": "", - // "environment": { - // "cluster": { - // "type": "Kubernetes", - // "url": "http://10.0.0.17/xxx" - // }, - // "supportedContainers": [ - // { - // "image": "tensorflow/tensorflow", - // "tag": "latest", - // "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" - // }, - // { - // "image": "tensorflow/tensorflow", - // "tag": "latest", - // "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" - // } - // ], - // "supportedServers": [ - // { - // "serverId": "1", - // "serverType": "xlsize", - // "cost": "50", - // "cpu": "16", - // "gpu": "0", - // "memory": "128gb", - // "disk": "160gb", - // "maxExecutionTime": 86400 - // }, - // { - // "serverId": "2", - // "serverType": "medium", - // "cost": "10", - // "cpu": "2", - // "gpu": "0", - // "memory": "8gb", - // "disk": "80gb", - // "maxExecutionTime": 86400 - // } - // ] - // } +// "datePublished": "2019-04-09T19:02:11Z", +// "cost": "10", +// "timeout": 86400, +// "provider": { +// "type": "Azure", +// "description": "", +// "environment": { +// "cluster": { +// "type": "Kubernetes", +// "url": "http://10.0.0.17/xxx" +// }, +// "supportedContainers": [ +// { +// "image": "tensorflow/tensorflow", +// "tag": "latest", +// "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" +// }, +// { +// "image": "tensorflow/tensorflow", +// "tag": "latest", +// "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" +// } +// ], +// "supportedServers": [ +// { +// "serverId": "1", +// "serverType": "xlsize", +// "cost": "50", +// "cpu": "16", +// "gpu": "0", +// "memory": "128gb", +// "disk": "160gb", +// "maxExecutionTime": 86400 +// }, +// { +// "serverId": "2", +// "serverType": "medium", +// "cost": "10", +// "cpu": "2", +// "gpu": "0", +// "memory": "8gb", +// "disk": "80gb", +// "maxExecutionTime": 86400 +// } +// ] +// } diff --git a/src/ocean/Ocean.ts b/src/ocean/Ocean.ts index ac18104b..6a2cafc9 100644 --- a/src/ocean/Ocean.ts +++ b/src/ocean/Ocean.ts @@ -17,7 +17,7 @@ import { Instantiable, generateIntantiableConfigFromConfig } from '../Instantiable.abstract' -import {Compute} from "./Compute"; +import { Compute } from './Compute' /** * Main interface for Ocean Protocol. diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index d02c7f90..9c94eafa 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -186,7 +186,7 @@ export class Provider extends Instantiable { // 'algorithmDataToken': alg_data_token // switch fetch method - + let fetch switch (method) { diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 711ee119..86d314b4 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -3,7 +3,7 @@ import { DataTokens } from '../../src/datatokens/Datatokens' import { Ocean } from '../../src/ocean/Ocean' import config from './config' import { assert } from 'console' -import {ComputeJob} from "../../src/ocean/interfaces/ComputeJob"; +import { ComputeJob } from '../../src/ocean/interfaces/ComputeJob' import { Service, ServiceComputePrivacy, @@ -38,7 +38,7 @@ describe('Marketplace flow', () => { let cluster let servers let containers - let provider + let provider const dateCreated = new Date(Date.now()).toISOString().split('.')[0] + 'Z' // remove milliseconds @@ -88,7 +88,8 @@ describe('Marketplace flow', () => { license: 'CC-BY', files: [ { - url:'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + url: + 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', checksum: 'efb2c764274b745f5fc37f97c6b0e764', contentLength: '4535431', contentType: 'text/csv', @@ -102,9 +103,21 @@ describe('Marketplace flow', () => { it('Alice publishes dataset with a compute service', async () => { price = 10 // in datatoken - cluster = ocean.compute.createClusterAttributes('Kubernetes', 'http://10.0.0.17/xxx') + cluster = ocean.compute.createClusterAttributes( + 'Kubernetes', + 'http://10.0.0.17/xxx' + ) servers = [ - ocean.compute.createServerAttributes('1', 'xlsize', '50', '16', '0', '128gb', '160gb', timeout) + ocean.compute.createServerAttributes( + '1', + 'xlsize', + '50', + '16', + '0', + '128gb', + '160gb', + timeout + ) ] containers = [ ocean.compute.createContainerAttributes( @@ -120,23 +133,25 @@ describe('Marketplace flow', () => { containers, servers ) - + const computeService = ocean.compute.createComputeService( - alice, price, dateCreated, provider + alice, + price, + dateCreated, + provider ) ddo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) assert(ddo.dataToken === tokenAddress) - }) - //alex + // alex it('should publish a dataset with a compute service object that does not allow rawAlgo', async () => { const origComputePrivacy = { allowRawAlgorithm: false, allowNetworkAccess: false, trustedAlgorithms: [] } - + const computeService = ocean.compute.createComputeService( alice, '1000', @@ -144,17 +159,22 @@ describe('Marketplace flow', () => { provider, origComputePrivacy as ServiceComputePrivacy ) - datasetNoRawAlgo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) + datasetNoRawAlgo = await ocean.assets.create( + asset, + alice, + [computeService], + tokenAddress + ) assert(datasetNoRawAlgo.dataToken === tokenAddress) }) - + it('should publish a dataset with a compute service object that allows only algo with did:op:1234', async () => { const origComputePrivacy = { allowRawAlgorithm: false, allowNetworkAccess: false, trustedAlgorithms: ['did:op:1234'] } - + const computeService = ocean.compute.createComputeService( alice, '1000', @@ -162,10 +182,15 @@ describe('Marketplace flow', () => { provider, origComputePrivacy as ServiceComputePrivacy ) - datasetWithTrustedAlgo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) + datasetWithTrustedAlgo = await ocean.assets.create( + asset, + alice, + [computeService], + tokenAddress + ) assert(datasetWithTrustedAlgo.dataToken === tokenAddress) }) - + it('should publish an algorithm', async () => { const algoAsset = { main: { @@ -176,22 +201,22 @@ describe('Marketplace flow', () => { license: 'CC-BY', files: [ { - url:'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', + url: + 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', contentType: 'text/js', encoding: 'UTF-8' - } ], - "algorithm": { - "language": "js", - "format": "docker-image", - "version": "0.1", - "container": { - "entrypoint": "node $ALGO", - "image": "node", - "tag": "10" + algorithm: { + language: 'js', + format: 'docker-image', + version: '0.1', + container: { + entrypoint: 'node $ALGO', + image: 'node', + tag: '10' } - }, + } } } const service1 = await ocean.assets.createAccessServiceAttributes( @@ -200,12 +225,15 @@ describe('Marketplace flow', () => { dateCreated, 0 ) - algorithmAsset = await ocean.assets.create(algoAsset, alice,[service1],tokenAddress) + algorithmAsset = await ocean.assets.create( + algoAsset, + alice, + [service1], + tokenAddress + ) assert(algorithmAsset.dataToken === tokenAddress) - }) - it('Alice mints 100 DTs and tranfers them to the compute marketplace', async () => { await datatoken.mint(tokenAddress, alice.getId(), tokenAmount) }) @@ -225,29 +253,32 @@ describe('Marketplace flow', () => { }) }) - it('Bob starts compute job', async () => { + it('Bob starts compute job with a raw Algo', async () => { const algorithmMeta = { - 'language': 'scala', - 'format': 'docker-image', - 'version': '0.1', - 'url': 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', - 'container': { - 'entrypoint': 'node $ALGO', - 'image': 'node', - 'tag': '10' + language: 'js', + format: 'docker-image', + version: '0.1', + url: + 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', + container: { + entrypoint: 'node $ALGO', + image: 'node', + tag: '10' } } - const output = { - - } - let order = await ocean.assets.order(ddo.id, computeService.type, bob.getId()) - let computeOrder=JSON.parse(order) - let tx=await datatoken.transfer( - computeOrder["dataToken"], - computeOrder["to"], - computeOrder["numTokens"], - computeOrder["from"] + const output = {} + const order = await ocean.assets.order( + ddo.id, + computeService.type, + bob.getId() + ) + const computeOrder = JSON.parse(order) + const tx = await datatoken.transfer( + computeOrder['dataToken'], + computeOrder['to'], + computeOrder['numTokens'], + computeOrder['from'] ) const response = await ocean.compute.start( ddo.id, @@ -262,24 +293,23 @@ describe('Marketplace flow', () => { computeService.type ) jobId = response.jobId - assert(response.status>=10) - + assert(response.status >= 10) }) it('Bob should get status of a compute job', async () => { const response = await ocean.compute.status(bob, ddo.id, jobId) - assert(response[0].jobId==jobId) + assert(response[0].jobId === jobId) }) - + it('should get status of all compute jobs for an address', async () => { const response = await ocean.compute.status(bob, undefined, undefined) - assert(response.length>0) - + assert(response.length > 0) }) - + // it('should not allow order the compute service with raw algo for dataset that does not allow raw algo', async () => {}) + // it('should not allow order the compute service with did != did:op:1234 for dataset that allows only did:op:1234 as algo', async () => {}) + // it('should start a compute job with a published algo', async () => { // it('Bob restarts compute job', async () => {}) - // it('Bob gets outputs', async () => {}) }) }) From 4e45956edac9769ea856ca1f2352031657723303 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 01:11:53 -0700 Subject: [PATCH 07/14] add more complex tests --- src/ocean/Compute.ts | 56 ++++++++++++++++++++++ test/integration/ComputeFlow.test.ts | 71 ++++++++++++++++++++-------- 2 files changed, 106 insertions(+), 21 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 7a132d1a..5796bd8f 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -284,6 +284,62 @@ export class Compute extends Instantiable { owner: output.owner || consumerAccount.getId() } } + + /** + * Starts an order of a compute service that is defined in an asset's services. + * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {string} datasetDid The DID of the dataset asset (of type `dataset`) to run the algorithm on. + * @param {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset. + * @param {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified. + * @return {Promise} Returns the Service Agreement ID, representation of `bytes32` ID. + * + * Note: algorithmDid and algorithmMeta are optional, but if they are not passed, + * you can end up in the situation that you are ordering and paying for your agreement, + * but brizo will not allow the compute, due to privacy settings of the ddo + */ + public order( + consumerAccount: string, + datasetDid: string, + serviceIndex: number, + algorithmDid?: string, + algorithmMeta?: MetadataAlgorithm + ): SubscribablePromise { + return new SubscribablePromise(async (observer) => { + const ddo: DDO = await this.ocean.assets.resolve(datasetDid) + // const service: Service = ddo.findServiceByType('compute') + const service: Service = ddo.findServiceById(serviceIndex) + if (!service) return null + if (service.type !== 'compute') return null + if (algorithmMeta) { + // check if raw algo is allowed + if (service.attributes.main.privacy) + if (!service.attributes.main.privacy.allowRawAlgorithm) { + console.error('This service does not allow Raw Algo') + return null + } + } + if (algorithmDid) { + // check if did is in trusted list + if (service.attributes.main.privacy) + if (service.attributes.main.privacy.trustedAlgorithms) + if (service.attributes.main.privacy.trustedAlgorithms.length > 0) + if ( + !service.attributes.main.privacy.trustedAlgorithms.includes( + algorithmDid + ) + ) { + console.error('This service does not allow this Algo') + return null + } + } + const order = await this.ocean.assets.order( + datasetDid, + service.type, + consumerAccount + ) + return order + }) + } } // "creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", // "datePublished": "2019-04-09T19:02:11Z", diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 86d314b4..bf5520fb 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -46,6 +46,18 @@ describe('Marketplace flow', () => { const tokenAmount = 100 const timeout = 86400 + const algorithmMeta = { + language: 'js', + format: 'docker-image', + version: '0.1', + url: + 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', + container: { + entrypoint: 'node $ALGO', + image: 'node', + tag: '10' + } + } describe('#MarketplaceComputeFlow-Test', () => { it('Initialize Ocean contracts v3', async () => { @@ -101,7 +113,7 @@ describe('Marketplace flow', () => { } }) - it('Alice publishes dataset with a compute service', async () => { + it('Alice publishes dataset with a compute service that allows Raw Algo', async () => { price = 10 // in datatoken cluster = ocean.compute.createClusterAttributes( 'Kubernetes', @@ -133,12 +145,17 @@ describe('Marketplace flow', () => { containers, servers ) - + const origComputePrivacy = { + allowRawAlgorithm: true, + allowNetworkAccess: false, + trustedAlgorithms: [] + } const computeService = ocean.compute.createComputeService( alice, price, dateCreated, - provider + provider, + origComputePrivacy as ServiceComputePrivacy ) ddo = await ocean.assets.create(asset, alice, [computeService], tokenAddress) assert(ddo.dataToken === tokenAddress) @@ -254,25 +271,15 @@ describe('Marketplace flow', () => { }) it('Bob starts compute job with a raw Algo', async () => { - const algorithmMeta = { - language: 'js', - format: 'docker-image', - version: '0.1', - url: - 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', - container: { - entrypoint: 'node $ALGO', - image: 'node', - tag: '10' - } - } - const output = {} - const order = await ocean.assets.order( + const order = await ocean.compute.order( + bob.getId(), ddo.id, - computeService.type, - bob.getId() + computeService.index, + undefined, + algorithmMeta ) + assert(order != null) const computeOrder = JSON.parse(order) const tx = await datatoken.transfer( computeOrder['dataToken'], @@ -306,8 +313,30 @@ describe('Marketplace flow', () => { assert(response.length > 0) }) - // it('should not allow order the compute service with raw algo for dataset that does not allow raw algo', async () => {}) - // it('should not allow order the compute service with did != did:op:1234 for dataset that allows only did:op:1234 as algo', async () => {}) + it('should not allow order the compute service with raw algo for dataset that does not allow raw algo', async () => { + const service1 = datasetNoRawAlgo.findServiceByType('compute') + assert(service1 !== null) + const order = await ocean.compute.order( + bob.getId(), + datasetNoRawAlgo.id, + service1.index, + undefined, + algorithmMeta + ) + assert(order === null) + }) + it('should not allow order the compute service with algoDid != "did:op:1234" for dataset that allows only "did:op:1234" as algo', async () => { + const service1 = datasetWithTrustedAlgo.findServiceByType('compute') + assert(service1 !== null) + const order = await ocean.compute.order( + bob.getId(), + datasetWithTrustedAlgo.id, + service1.index, + 'did:op:77777', + undefined + ) + assert(order === null) + }) // it('should start a compute job with a published algo', async () => { // it('Bob restarts compute job', async () => {}) // it('Bob gets outputs', async () => {}) From 2bf04170d7777e1cc1364acae23226d3b8ad3dcf Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 01:14:52 -0700 Subject: [PATCH 08/14] minor doc fix --- src/ocean/Compute.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 5796bd8f..6a7e8a9d 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -287,15 +287,16 @@ export class Compute extends Instantiable { /** * Starts an order of a compute service that is defined in an asset's services. - * @param {Account} consumerAccount The account of the consumer ordering the service. + * @param {String} consumerAccount The account of the consumer ordering the service. * @param {string} datasetDid The DID of the dataset asset (of type `dataset`) to run the algorithm on. + * @param {string} serviceIndex The Service index * @param {string} algorithmDid The DID of the algorithm asset (of type `algorithm`) to run on the asset. * @param {MetaData} algorithmMeta Metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithmDid` is specified. - * @return {Promise} Returns the Service Agreement ID, representation of `bytes32` ID. + * @return {Promise} Returns the transaction details * * Note: algorithmDid and algorithmMeta are optional, but if they are not passed, - * you can end up in the situation that you are ordering and paying for your agreement, - * but brizo will not allow the compute, due to privacy settings of the ddo + * you can end up in the situation that you are ordering and paying for your compute job, + * but provider will not allow the compute, due to privacy settings of the ddo */ public order( consumerAccount: string, From 7ef5c6b7b0fffc704c9133c3fcd489030399c47d Mon Sep 17 00:00:00 2001 From: ssallam Date: Mon, 13 Jul 2020 10:48:25 +0200 Subject: [PATCH 09/14] Fix storing encrypted files in new ddo. --- src/provider/Provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index 9c94eafa..cc78ab53 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -60,7 +60,7 @@ export class Provider extends Instantiable { this.getEncryptEndpoint(), decodeURI(JSON.stringify(args)) ) - return await response.text() + return (await response.json()).encryptedDocument } catch (e) { this.logger.error(e) throw new Error('HTTP request failed') From fdbc69206469af5ac5cb30bee5a364d35609d7b9 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 02:12:34 -0700 Subject: [PATCH 10/14] add compute test with published algo --- src/ocean/Compute.ts | 8 +++-- src/provider/Provider.ts | 10 ++++-- test/integration/ComputeFlow.test.ts | 50 ++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 6a7e8a9d..bf5893dd 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -68,7 +68,9 @@ export class Compute extends Instantiable { algorithmMeta?: MetadataAlgorithm, output?: Output, serviceIndex?: string, - serviceType?: string + serviceType?: string, + algorithmTransferTxId?: string, + algorithmDataToken?: string ): Promise { output = this.checkOutput(consumerAccount, output) if (did) { @@ -83,7 +85,9 @@ export class Compute extends Instantiable { txId, serviceIndex, serviceType, - tokenAddress + tokenAddress, + algorithmTransferTxId, + algorithmDataToken ) return computeJobsList[0] as ComputeJob } else return null diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index 9c94eafa..2ef4cb36 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -144,10 +144,11 @@ export class Provider extends Instantiable { txId?: string, serviceIndex?: string, serviceType?: string, - tokenAddress?: string + tokenAddress?: string, + algorithmTransferTxId?: string, + algorithmDataToken?: string ): Promise { const address = consumerAccount.getId() - let signatureMessage = address signatureMessage += jobId || '' signatureMessage += (did && `${noZeroX(did)}`) || '' @@ -169,6 +170,11 @@ export class Provider extends Instantiable { url += (jobId && `&jobId=${jobId}`) || '' url += `&consumerAddress=${address}` url += `&transferTxId=${txId}` || '' + url += + (algorithmTransferTxId && + `&algorithmTransferTxId=${algorithmTransferTxId}`) || + '' + url += (algorithmDataToken && `&algorithmDataToken=${algorithmDataToken}`) || '' url += `&serviceId=${serviceIndex}` || '' url += `&serviceType=${serviceType}` || '' url += `&dataToken=${tokenAddress}` || '' diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index bf5520fb..36cc4ec2 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -114,7 +114,7 @@ describe('Marketplace flow', () => { }) it('Alice publishes dataset with a compute service that allows Raw Algo', async () => { - price = 10 // in datatoken + price = 2 // in datatoken cluster = ocean.compute.createClusterAttributes( 'Kubernetes', 'http://10.0.0.17/xxx' @@ -337,7 +337,53 @@ describe('Marketplace flow', () => { ) assert(order === null) }) - // it('should start a compute job with a published algo', async () => { + it('should start a compute job with a published algo', async () => { + const output = {} + const serviceAlgo = algorithmAsset.findServiceByType('access') + const orderalgo = await ocean.assets.order( + algorithmAsset.id, + serviceAlgo.type, + bob.getId() + ) + const algoOrder = JSON.parse(orderalgo) + const algoTx = await datatoken.transfer( + algoOrder['dataToken'], + algoOrder['to'], + algoOrder['numTokens'], + algoOrder['from'] + ) + const order = await ocean.compute.order( + bob.getId(), + ddo.id, + computeService.index, + algorithmAsset.id, + undefined + ) + assert(order != null) + const computeOrder = JSON.parse(order) + const tx = await datatoken.transfer( + computeOrder['dataToken'], + computeOrder['to'], + computeOrder['numTokens'], + computeOrder['from'] + ) + const response = await ocean.compute.start( + ddo.id, + tx.transactionHash, + tokenAddress, + bob, + undefined, + undefined, + algorithmMeta, + output, + computeService.index, + computeService.type, + algoTx, + algorithmAsset.datatoken + ) + jobId = response.jobId + assert(response.status >= 10) + }) // it('Bob restarts compute job', async () => {}) // it('Bob gets outputs', async () => {}) }) From 61786ce532e8077e25153473aa747a129080e4be Mon Sep 17 00:00:00 2001 From: Ahmed Ali Date: Mon, 13 Jul 2020 11:24:05 +0200 Subject: [PATCH 11/14] comment restart for now --- src/ocean/Compute.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index f68524bd..173e71ad 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -141,22 +141,22 @@ export class Compute extends Instantiable { return computeJobsList[0] as ComputeJob } - /** - * Ends a running compute job and starts it again. - * @param {Account} consumerAccount The account of the consumer ordering the service. - * @param {string} did Decentralized identifier. - * @param {string} jobId The ID of the compute job to be stopped - * @return {Promise} Returns the new status of a job - */ - public async restart( - consumerAccount: Account, - did: string, - jobId: string - ): Promise { - await this.stop(consumerAccount, did, jobId) - const result = await this.start(consumerAccount, did, jobId) - return result - } + // /** + // * Ends a running compute job and starts it again. + // * @param {Account} consumerAccount The account of the consumer ordering the service. + // * @param {string} did Decentralized identifier. + // * @param {string} jobId The ID of the compute job to be stopped + // * @return {Promise} Returns the new status of a job + // */ + // public async restart( + // consumerAccount: Account, + // did: string, + // jobId: string + // ): Promise { + // await this.stop(consumerAccount, did, jobId) + // const result = await this.start(consumerAccount, did, jobId) + // return result + // } /** * Returns information about the status of all compute jobs, or a single compute job. From 2deca53cc76d9d24d0b26932cee2c4092d3735d0 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 02:34:31 -0700 Subject: [PATCH 12/14] add assets.order by index --- src/ocean/Assets.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/ocean/Assets.ts b/src/ocean/Assets.ts index 6dd9ab40..daaa29fe 100644 --- a/src/ocean/Assets.ts +++ b/src/ocean/Assets.ts @@ -347,6 +347,20 @@ export class Assets extends Instantiable { return service } + public async getServiceByIndex( + did: string, + serviceIndex: number + ): Promise { + const services: ServiceCommon[] = (await this.resolve(did)).service + let service + services.forEach((serv) => { + if (serv.index === serviceIndex) { + service = serv + } + }) + return service + } + public async createAccessServiceAttributes( creator: Account, dtCost: number, @@ -372,12 +386,19 @@ export class Assets extends Instantiable { public async order( did: string, serviceType: string, - consumerAddress: string + consumerAddress: string, + serviceIndex: number = -1 ): Promise { - const service = await this.getServiceByType(did, serviceType) + if (serviceIndex === -1) { + const service = await this.getServiceByType(did, serviceType) + serviceIndex = service.index + } else { + const service = await this.getServiceByIndex(did, serviceIndex) + serviceType = service.type + } return await this.ocean.provider.initialize( did, - service.index, + serviceIndex, serviceType, consumerAddress ) From f8ef12f8dcd3ce6580160a7c7af0347d32cd022f Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Mon, 13 Jul 2020 02:51:17 -0700 Subject: [PATCH 13/14] add compute stop and fix job with algodid --- src/ocean/Compute.ts | 1 - src/provider/Provider.ts | 1 - test/integration/ComputeFlow.test.ts | 15 +++++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 173e71ad..0db5caf1 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -64,7 +64,6 @@ export class Compute extends Instantiable { tokenAddress: string, consumerAccount: Account, algorithmDid?: string, - algorithmTokenAddress?: string, algorithmMeta?: MetadataAlgorithm, output?: Output, serviceIndex?: string, diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index 957d9c8f..ddbec903 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -192,7 +192,6 @@ export class Provider extends Instantiable { // 'algorithmDataToken': alg_data_token // switch fetch method - let fetch switch (method) { diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 36cc4ec2..f48b5707 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -293,7 +293,6 @@ describe('Marketplace flow', () => { tokenAddress, bob, undefined, - undefined, algorithmMeta, output, computeService.index, @@ -312,7 +311,11 @@ describe('Marketplace flow', () => { const response = await ocean.compute.status(bob, undefined, undefined) assert(response.length > 0) }) - + it('Bob should stop compute job', async () => { + await ocean.compute.stop(bob, ddo.id, jobId) + const response = await ocean.compute.status(bob, ddo.id, jobId) + assert(response[0].stopreq === 1) + }) it('should not allow order the compute service with raw algo for dataset that does not allow raw algo', async () => { const service1 = datasetNoRawAlgo.findServiceByType('compute') assert(service1 !== null) @@ -372,18 +375,18 @@ describe('Marketplace flow', () => { tx.transactionHash, tokenAddress, bob, + algorithmAsset.id, undefined, - undefined, - algorithmMeta, output, computeService.index, computeService.type, - algoTx, - algorithmAsset.datatoken + algoTx.transactionHash, + algorithmAsset.dataToken ) jobId = response.jobId assert(response.status >= 10) }) + // it('Bob restarts compute job', async () => {}) // it('Bob gets outputs', async () => {}) }) From 85776919ec401b7617410b124f4705a220de972b Mon Sep 17 00:00:00 2001 From: Ahmed Ali Date: Mon, 13 Jul 2020 11:59:22 +0200 Subject: [PATCH 14/14] minor cleaning --- src/ocean/Compute.ts | 82 -------------------------------------------- 1 file changed, 82 deletions(-) diff --git a/src/ocean/Compute.ts b/src/ocean/Compute.ts index 0db5caf1..d3836477 100644 --- a/src/ocean/Compute.ts +++ b/src/ocean/Compute.ts @@ -1,18 +1,11 @@ -import { SearchQuery } from '../metadatastore/MetadataStore' import { DDO } from '../ddo/DDO' -import { Metadata } from '../ddo/interfaces/Metadata' import { MetadataAlgorithm } from '../ddo/interfaces/MetadataAlgorithm' import { Service, ServiceComputePrivacy, ServiceCompute } from '../ddo/interfaces/Service' -import { EditableMetadata } from '../ddo/interfaces/EditableMetadata' import Account from './Account' -import DID from './DID' import { SubscribablePromise } from '../utils' import { Instantiable, InstantiableConfig } from '../Instantiable.abstract' import { Output } from './interfaces/ComputeOutput' import { ComputeJob } from './interfaces/ComputeJob' -// import { WebServiceConnector } from './utils/WebServiceConnector' -// import { Output } from './interfaces/ComputeOutput' -// import { ComputeJob } from './interfaces/ComputeJob' export enum OrderProgressStep { TransferDataToken @@ -140,23 +133,6 @@ export class Compute extends Instantiable { return computeJobsList[0] as ComputeJob } - // /** - // * Ends a running compute job and starts it again. - // * @param {Account} consumerAccount The account of the consumer ordering the service. - // * @param {string} did Decentralized identifier. - // * @param {string} jobId The ID of the compute job to be stopped - // * @return {Promise} Returns the new status of a job - // */ - // public async restart( - // consumerAccount: Account, - // did: string, - // jobId: string - // ): Promise { - // await this.stop(consumerAccount, did, jobId) - // const result = await this.start(consumerAccount, did, jobId) - // return result - // } - /** * Returns information about the status of all compute jobs, or a single compute job. * @param {Account} consumerAccount The account of the consumer ordering the service. @@ -304,17 +280,6 @@ export class Compute extends Instantiable { publishOutput: false } } - // 'signature': signature, - // 'documentId': did, - // 'serviceId': sa.index, - // 'serviceType': sa.type, - // 'consumerAddress': cons_acc.address, - // 'transferTxId': Web3.toHex(tx_id), - // 'dataToken': data_token, - // 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc), - // 'algorithmDid': alg_ddo.did, - // 'algorithmMeta': {}, - // 'algorithmDataToken': alg_data_token return { publishAlgorithmLog: output.publishAlgorithmLog, @@ -384,50 +349,3 @@ export class Compute extends Instantiable { }) } } -// "creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", -// "datePublished": "2019-04-09T19:02:11Z", -// "cost": "10", -// "timeout": 86400, -// "provider": { -// "type": "Azure", -// "description": "", -// "environment": { -// "cluster": { -// "type": "Kubernetes", -// "url": "http://10.0.0.17/xxx" -// }, -// "supportedContainers": [ -// { -// "image": "tensorflow/tensorflow", -// "tag": "latest", -// "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" -// }, -// { -// "image": "tensorflow/tensorflow", -// "tag": "latest", -// "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" -// } -// ], -// "supportedServers": [ -// { -// "serverId": "1", -// "serverType": "xlsize", -// "cost": "50", -// "cpu": "16", -// "gpu": "0", -// "memory": "128gb", -// "disk": "160gb", -// "maxExecutionTime": 86400 -// }, -// { -// "serverId": "2", -// "serverType": "medium", -// "cost": "10", -// "cpu": "2", -// "gpu": "0", -// "memory": "8gb", -// "disk": "80gb", -// "maxExecutionTime": 86400 -// } -// ] -// }