oceanprotocol/ocean.js
1
0
Fork 0
mirror of https://github.com/oceanprotocol/ocean.js.git synced 2024-11-26 20:39:05 +01:00
Code Releases Activity

Merge pull request #1212 from oceanprotocol/feature/default-http-fixes

Browse Source 
Add default http function and fixes provider class
This commit is contained in:
Bogdan Fazakas 2022-01-20 09:04:36 +02:00 committed by GitHub
commit 7c6ce6368a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 158 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -39,6 +39,7 @@
"test:unit": "npm run mocha -- 'test/unit/**/*.test.ts'",
"test:unit:cover": "nyc --report-dir coverage/unit npm run test:unit",
"test:integration": "npm run mocha -- 'test/integration/**/*.test.ts'",
"test:provider": "npm run mocha -- 'test/integration/Provider.test.ts'",
"test:integration:cover": "nyc --report-dir coverage/integration --no-clean npm run test:integration"
},
"repository": {

6
src/@types/FileMetadata.ts
View File

@ -35,4 +35,10 @@ export interface FileMetadata {
* @type {string}
*/
method?: string
/**
* check if file exists
* @type {boolean}
*/
valid?: boolean
}

38
src/aquarius/Aquarius.ts
View File

@ -1,4 +1,4 @@
import { LoggerInstance } from '../utils'
import { LoggerInstance, crossFetchGeneric } from '../utils'
import { Asset, DDO, Metadata, ValidateMetadata } from '../@types/'
import { json } from 'stream/consumers'
@ -17,12 +17,15 @@ export class Aquarius {
* @param {string} fetchMethod fetch client instance
* @return {Promise<DDO>} DDO
*/
public async resolve(did: string, fetchMethod: any): Promise<DDO> {
public async resolve(did: string, fetchMethod?: any): Promise<DDO> {
const preferedFetch = fetchMethod || crossFetchGeneric
const path = this.aquariusURL + '/api/aquarius/assets/ddo/' + did
try {
const response = await fetchMethod('GET', path)
const response = await preferedFetch('GET', path, null, {
'Content-Type': 'application/json'
})
if (response.ok) {
const raw = await response.json()
const raw = response.data ? response.data : await response.json()
return raw as DDO
} else {
throw new Error('HTTP request failed with status ' + response.status)
@ -44,19 +47,27 @@ export class Aquarius {
/**
* Blocks until Aqua will cache the did (or the update for that did) or timeouts
* @param {string} did DID of the asset.
* @param {string} txid used when the did exists and we expect an update with that txid.
* @param {string} fetchMethod fetch client instance
* @param {string} did DID of the asset.
* @param {string} txid used when the did exists and we expect an update with that txid.
* @return {Promise<DDO>} DDO of the asset.
*/
public async waitForAqua(fetchMethod: any, did: string, txid?: string): Promise<Asset> {
public async waitForAqua(
did: string,
txid?: string,
fetchMethod?: any
): Promise<Asset> {
let tries = 0
do {
try {
const preferedFetch = fetchMethod || crossFetchGeneric
const path = this.aquariusURL + '/api/aquarius/assets/ddo/' + did
const response = await fetchMethod('GET', path)
const response = await preferedFetch('GET', path, null, {
'Content-Type': 'application/json'
})
if (response.ok) {
const ddo = await response.json()
const ddo = response.data ? response.data : await response.json()
if (txid) {
// check tx
if (ddo.event && ddo.event.txid === txid) return ddo as Asset
@ -73,21 +84,22 @@ export class Aquarius {
/**
* Validate DDO content
* @param {DDO} ddo DID Descriptor Object content.
* @param {string} fetchMethod fetch client instance
* @param {DDO} ddo DID Descriptor Object content.
* @return {Promise<ValidateMetadata>}.
*/
public async validate(fetchMethod: any, ddo: DDO): Promise<ValidateMetadata> {
public async validate(ddo: DDO, fetchMethod: any): Promise<ValidateMetadata> {
const preferedFetch = fetchMethod || crossFetchGeneric
const status: ValidateMetadata = {
valid: false
}
let jsonResponse
try {
const path = this.aquariusURL + '/api/aquarius/assets/ddo/validate'
const response = await fetchMethod('POST', path, JSON.stringify(ddo), {
const response = await preferedFetch('POST', path, JSON.stringify(ddo), {
'Content-Type': 'application/octet-stream'
})
jsonResponse = await response.json()
jsonResponse = response.data ? response.data : await response.json()
if (response.status === 200) {
status.valid = true
status.hash = jsonResponse.hash

116
src/provider/Provider.ts
View File

@ -1,7 +1,6 @@
import Web3 from 'web3'
import { LoggerInstance, getData } from '../utils'
import { LoggerInstance, getData, crossFetchGeneric } from '../utils'
import {
Asset,
FileMetadata,
ComputeJob,
ComputeOutput,
@ -74,10 +73,11 @@ export class Provider {
public async getNonce(
providerUri: string,
consumerAddress: string,
fetchMethod: any,
fetchMethod?: any,
providerEndpoints?: any,
serviceEndpoints?: ServiceEndpoint[]
): Promise<string> {
const preferedFetch = fetchMethod || crossFetchGeneric
if (!providerEndpoints) {
providerEndpoints = await this.getEndpoints(providerUri)
}
@ -89,7 +89,14 @@ export class Provider {
: null
if (!path) return null
try {
const response = await fetchMethod(path + `?userAddress=${consumerAddress}`)
const response = await preferedFetch(
'GET',
path + `?userAddress=${consumerAddress}`,
null,
{
'Content-Type': 'application/json'
}
)
return String((await response.json()).nonce)
} catch (e) {
LoggerInstance.error(e)
@ -121,7 +128,8 @@ export class Provider {
* @param {string} postMethod http post method
* @return {Promise<string>} urlDetails
*/
public async encrypt(data: any, providerUri: string, postMethod: any): Promise<any> {
public async encrypt(data: any, providerUri: string, postMethod?: any): Promise<any> {
const preferedFetch = postMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -133,9 +141,14 @@ export class Provider {
if (!path) return null
try {
const response = await postMethod('POST', path, decodeURI(JSON.stringify(data)), {
'Content-Type': 'application/octet-stream'
})
const response = await preferedFetch(
'POST',
path,
decodeURI(JSON.stringify(data)),
{
'Content-Type': 'application/octet-stream'
}
)
return response
} catch (e) {
LoggerInstance.error(e)
@ -154,8 +167,9 @@ export class Provider {
did: string,
serviceId: number,
providerUri: string,
fetchMethod: any
fetchMethod?: any
): Promise<FileMetadata[]> {
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -168,8 +182,12 @@ export class Provider {
: null
if (!path) return null
try {
const response = await fetchMethod(path, JSON.stringify(args))
const results: FileMetadata[] = await response.json()
const response = await preferedFetch('POST', path, JSON.stringify(args), {
'Content-Type': 'application/json'
})
const results: FileMetadata[] = response.data
? response.data
: await response.json()
for (const result of results) {
files.push(result)
}
@ -188,8 +206,9 @@ export class Provider {
public async checkFileUrl(
url: string,
providerUri: string,
fetchMethod: any
fetchMethod?: any
): Promise<FileMetadata[]> {
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -202,8 +221,12 @@ export class Provider {
: null
if (!path) return null
try {
const response = await fetchMethod('POST', path, JSON.stringify(args))
const results: FileMetadata[] = await response.json()
const response = await preferedFetch('POST', path, JSON.stringify(args), {
'Content-Type': 'application/json'
})
const results: FileMetadata[] = response.data
? response.data
: await response.json()
for (const result of results) {
files.push(result)
}
@ -229,11 +252,12 @@ export class Provider {
fileIndex: number,
consumerAddress: string,
providerUri: string,
getMethod: any,
fetchMethod?: any,
userCustomParameters?: UserCustomParameters,
computeEnv?: string,
validUntil?: number
): Promise<ProviderInitialize> {
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -253,8 +277,12 @@ export class Provider {
if (computeEnv) initializeUrl += '&computeEnv=' + encodeURI(computeEnv)
if (validUntil) initializeUrl += '&validUntil=' + validUntil
try {
const response = await getMethod('GET', initializeUrl)
const results: ProviderInitialize = await response.json()
const response = await preferedFetch('GET', initializeUrl, null, {
'Content-Type': 'application/json'
})
const results: ProviderInitialize = response.data
? response.data
: await response.json()
return results
} catch (e) {
LoggerInstance.error(e)
@ -321,14 +349,15 @@ export class Provider {
public async computeStart(
providerUri: string,
web3: Web3,
fetchMethod: any,
consumerAddress: string,
computeEnv: string,
dataset: ComputeAsset,
algorithm: ComputeAlgorithm,
fetchMethod?: any,
additionalDatasets?: ComputeAsset[],
output?: ComputeOutput
): Promise<ComputeJob | ComputeJob[]> {
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -359,7 +388,7 @@ export class Provider {
if (output) payload.output = output
if (!computeStartUrl) return null
try {
const response = await fetchMethod(
const response = await preferedFetch(
'POST',
computeStartUrl,
JSON.stringify(payload),
@ -368,7 +397,7 @@ export class Provider {
}
)
if (response?.ok) {
const params = await response.json()
const params = response.data ? response.data : await response.json()
return params
}
console.error('Compute start failed:', response.status, response.statusText)
@ -397,8 +426,9 @@ export class Provider {
jobId: string,
providerUri: string,
web3: Web3,
fetchMethod: any
fetchMethod?: any
): Promise<ComputeJob | ComputeJob[]> {
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -434,9 +464,16 @@ export class Provider {
if (!computeStopUrl) return null
try {
const response = await fetchMethod(computeStopUrl, JSON.stringify(payload))
const response = await preferedFetch(
'PUT',
computeStopUrl,
JSON.stringify(payload),
{
'Content-Type': 'application/json'
}
)
if (response?.ok) {
const params = await response.json()
const params = response.data ? response.data : await response.json()
return params
}
LoggerInstance.error('Compute stop failed:', response.status, response.statusText)
@ -461,7 +498,7 @@ export class Provider {
*/
public async computeStatus(
providerUri: string,
fetchMethod: any,
fetchMethod?: any,
jobId?: string,
did?: string,
consumerAddress?: string
@ -469,6 +506,8 @@ export class Provider {
if (!jobId && !did && !consumerAddress) {
throw new Error('You need at least one of jobId, did, consumerAddress')
}
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -484,9 +523,11 @@ export class Provider {
if (!computeStatusUrl) return null
try {
const response = await fetchMethod('GET', computeStatusUrl + url)
const response = await preferedFetch('GET', computeStatusUrl + url, null, {
'Content-Type': 'application/json'
})
if (response?.ok) {
const params = await response.json()
const params = response.data ? response.data : await response.json()
return params
}
LoggerInstance.error(
@ -577,8 +618,9 @@ export class Provider {
jobId: string,
providerUri: string,
web3: Web3,
fetchMethod: any
fetchMethod?: any
): Promise<ComputeJob | ComputeJob[]> {
const preferedFetch = fetchMethod || crossFetchGeneric
const providerEndpoints = await this.getEndpoints(providerUri)
const serviceEndpoints = await this.getServiceEndpoints(
providerUri,
@ -614,9 +656,16 @@ export class Provider {
if (!computeDeleteUrl) return null
try {
const response = await fetchMethod(computeDeleteUrl, JSON.stringify(payload))
const response = await preferedFetch(
'DELETE',
computeDeleteUrl,
JSON.stringify(payload),
{
'Content-Type': 'application/json'
}
)
if (response?.ok) {
const params = await response.json()
const params = response.data ? response.data : await response.json()
return params
}
LoggerInstance.error(
@ -639,11 +688,14 @@ export class Provider {
* @param {String} fetchMethod fetch client instance
* @return {Promise<boolean>} string
*/
public async isValidProvider(url: string, fetchMethod: any): Promise<boolean> {
public async isValidProvider(url: string, fetchMethod?: any): Promise<boolean> {
try {
const response = await fetchMethod(url)
const preferedFetch = fetchMethod || crossFetchGeneric
const response = await preferedFetch('GET', url, null, {
'Content-Type': 'application/json'
})
if (response?.ok) {
const params = await response.json()
const params = response.data ? response.data : await response.json()
if (params && params.providerAddress) return true
}
return false

12
test/integration/ComputeFlow.test.ts
View File

@ -242,9 +242,13 @@ describe('Simple compute tests', async () => {
'0x' + metadataHash
)
// let's wait
const resolvedDDOAsset = await aquarius.waitForAqua(crossFetchGeneric, ddo.id)
const resolvedDDOAsset = await aquarius.waitForAqua(ddo.id, null, crossFetchGeneric)
assert(resolvedDDOAsset, 'Cannot fetch DDO from Aquarius')
const resolvedDDOAlgo = await aquarius.waitForAqua(crossFetchGeneric, algoDdo.id)
const resolvedDDOAlgo = await aquarius.waitForAqua(
algoDdo.id,
null,
crossFetchGeneric
)
assert(resolvedDDOAlgo, 'Cannot fetch DDO from Aquarius')
// mint 1 ERC20 and send it to the consumer
await datatoken.mint(datatokenAddressAsset, publisherAccount, '1', consumerAccount)
@ -318,7 +322,6 @@ describe('Simple compute tests', async () => {
const computeJobs = await ProviderInstance.computeStart(
providerUrl,
web3,
crossFetchGeneric,
consumerAccount,
'env1',
{
@ -330,7 +333,8 @@ describe('Simple compute tests', async () => {
documentId: resolvedDDOAlgo.id,
serviceId: resolvedDDOAlgo.services[0].id,
transferTxId: txidAlgo.transactionHash
}
},
crossFetchGeneric
)
assert(computeJobs, 'Cannot start compute job')
const jobStatus = await ProviderInstance.computeStatus(

19
test/integration/Provider.test.ts
View File

@ -1,6 +1,7 @@
import { Provider } from '../../src/provider/Provider'
import { assert } from 'chai'
import { fetchData } from '../../src/utils'
import { fetchData, crossFetchGeneric } from '../../src/utils'
import { FileMetadata } from '../../src/@types'
describe('Provider tests', () => {
let providerInstance: Provider
@ -10,15 +11,27 @@ describe('Provider tests', () => {
})
it('Alice tests invalid provider', async () => {
const valid = await providerInstance.isValidProvider('http://example.net', fetchData)
const valid = await providerInstance.isValidProvider(
'http://example.net',
crossFetchGeneric
)
assert(valid === false)
})
it('Alice tests valid provider', async () => {
const valid = await providerInstance.isValidProvider(
'http://127.0.0.1:8030',
fetchData
crossFetchGeneric
)
assert(valid === true)
})
it('Alice checks fileinfo', async () => {
const fileinfo: FileMetadata[] = await providerInstance.checkFileUrl(
'https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-abstract.xml.gz-rss.xml',
'http://127.0.0.1:8030',
crossFetchGeneric
)
assert(fileinfo[0].valid === true, 'Sent file is not valid')
})
})

26
test/integration/PublishFlows.test.ts
View File

@ -147,8 +147,8 @@ describe('Publish tests', async () => {
'did:op:' + SHA256(web3.utils.toChecksumAddress(nftAddress) + chain.toString(10))
const AssetValidation: ValidateMetadata = await aquarius.validate(
crossFetchGeneric,
poolDdo
poolDdo,
crossFetchGeneric
)
assert(AssetValidation.valid === true, 'Published asset is not valid')
@ -173,7 +173,7 @@ describe('Publish tests', async () => {
[AssetValidation.proof]
)
const resolvedDDO = await aquarius.waitForAqua(crossFetchGeneric, poolDdo.id)
const resolvedDDO = await aquarius.waitForAqua(poolDdo.id, null, crossFetchGeneric)
assert(resolvedDDO, 'Cannot fetch DDO from Aquarius')
})
@ -236,8 +236,8 @@ describe('Publish tests', async () => {
'did:op:' + SHA256(web3.utils.toChecksumAddress(nftAddress) + chain.toString(10))
const isAssetValid: ValidateMetadata = await aquarius.validate(
crossFetchGeneric,
fixedPriceDdo
fixedPriceDdo,
crossFetchGeneric
)
assert(isAssetValid.valid === true, 'Published asset is not valid')
@ -260,7 +260,11 @@ describe('Publish tests', async () => {
'0x' + metadataHash,
[]
)
const resolvedDDO = await aquarius.waitForAqua(crossFetchGeneric, fixedPriceDdo.id)
const resolvedDDO = await aquarius.waitForAqua(
fixedPriceDdo.id,
null,
crossFetchGeneric
)
assert(resolvedDDO, 'Cannot fetch DDO from Aquarius')
})
@ -317,8 +321,8 @@ describe('Publish tests', async () => {
'did:op:' + SHA256(web3.utils.toChecksumAddress(nftAddress) + chain.toString(10))
const isAssetValid: ValidateMetadata = await aquarius.validate(
crossFetchGeneric,
dispenserDdo
dispenserDdo,
crossFetchGeneric
)
assert(isAssetValid.valid === true, 'Published asset is not valid')
@ -340,7 +344,11 @@ describe('Publish tests', async () => {
encryptedResponse,
'0x' + metadataHash
)
const resolvedDDO = await aquarius.waitForAqua(crossFetchGeneric, dispenserDdo.id)
const resolvedDDO = await aquarius.waitForAqua(
dispenserDdo.id,
null,
crossFetchGeneric
)
assert(resolvedDDO, 'Cannot fetch DDO from Aquarius')
})
})

2
test/integration/SimplePublishConsumeFlow.test.ts
View File

@ -123,7 +123,7 @@ describe('Simple Publish & consume test', async () => {
encryptedResponse,
'0x' + metadataHash
)
const resolvedDDO = await aquarius.waitForAqua(crossFetchGeneric, ddo.id)
const resolvedDDO = await aquarius.waitForAqua(ddo.id, null, crossFetchGeneric)
assert(resolvedDDO, 'Cannot fetch DDO from Aquarius')
// mint 1 ERC20 and send it to the consumer
await datatoken.mint(datatokenAddress, publisherAccount, '1', consumerAccount)