oceanprotocol/squid-js
1
0
Fork 0
mirror of https://github.com/oceanprotocol-archive/squid-js.git synced 2024-02-02 15:31:51 +01:00
Code Releases Activity

update metadata to OEP-8 v0.4

Browse Source
This commit is contained in:
Matthias Kretschmann 2019-08-15 13:23:56 +02:00
parent ba542f299a
commit 07d17fe32e
Signed by: m
GPG Key ID: 606EEEF3C479A91F
15 changed files with 221 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
integration/ocean/AssetOwners.test.ts
View File

@ -72,7 +72,7 @@ describe('Asset Owners', () => {
// Granting access
try {
await account2.requestTokens(
+metadata.base.price *
+metadata.main.price *
10 ** -(await ocean.keeper.token.decimals())
)
} catch {}

8
integration/ocean/ConsumeAsset.test.ts
View File

@ -31,7 +31,7 @@ describe('Consume Asset', () => {
}
})
it('should regiester a asset', async () => {
it('should register an asset', async () => {
ddo = await ocean.assets.create(metadata as any, publisher)
assert.isDefined(ddo, 'Register has not returned a DDO')
@ -50,7 +50,7 @@ describe('Consume Asset', () => {
it('should be able to request tokens for consumer', async () => {
const initialBalance = (await consumer.getBalance()).ocn
const claimedTokens =
+metadata.base.price * 10 ** -(await ocean.keeper.token.decimals())
+metadata.main.price * 10 ** -(await ocean.keeper.token.decimals())
try {
await consumer.requestTokens(claimedTokens)
@ -115,7 +115,7 @@ describe('Consume Asset', () => {
it('should lock the payment by the consumer', async () => {
const paid = await ocean.agreements.conditions.lockReward(
serviceAgreementSignatureResult.agreementId,
ddo.findServiceByType('Metadata').metadata.base.price,
ddo.findServiceByType('Metadata').metadata.main.price,
consumer
)
@ -182,7 +182,7 @@ describe('Consume Asset', () => {
)
})
it('should consume and store one assets', async () => {
it('should consume and store one asset', async () => {
const accessService = ddo.findServiceByType('Access')
const folder = '/tmp/ocean/squid-js-2'

2
integration/ocean/ConsumeAssetBrizo.test.ts
View File

@ -54,7 +54,7 @@ describe('Consume Asset (Brizo)', () => {
try {
await consumer.requestTokens(
+metadata.base.price *
+metadata.main.price *
10 ** -(await ocean.keeper.token.decimals())
)
} catch {}

6
integration/ocean/ConsumeBigAsset.test.ts
View File

@ -30,8 +30,8 @@ xdescribe('Consume Asset (Large size)', () => {
}
metadata = {
...baseMetadata,
base: {
...baseMetadata.base,
main: {
...baseMetadata.main,
files: [
{
url: 'https://speed.hetzner.de/1GB.bin'
@ -52,7 +52,7 @@ xdescribe('Consume Asset (Large size)', () => {
try {
await consumer.requestTokens(
+metadata.base.price *
+metadata.main.price *
10 ** -(await ocean.keeper.token.decimals())
)
} catch {}

2
integration/ocean/SearchAsset.test.ts
View File

@ -45,7 +45,7 @@ describe('Search Asset', () => {
}
})
it('should regiester some a asset', async () => {
it('should register an asset', async () => {
assert.instanceOf(
await ocean.assets.create(
metadataGenerator('Test1') as any,

2
integration/ocean/Signature.test.ts
View File

@ -64,7 +64,7 @@ describe('Signature', () => {
{
type: 'Metadata',
metadata: {
base: {
main: {
price: 10
}
}

2
integration/ocean/Versions.test.ts
View File

@ -11,7 +11,7 @@ describe('Versions', () => {
ocean = await Ocean.getInstance(config)
})
it('should returns the versions', async () => {
it('should return the versions', async () => {
const versions = await ocean.versions.get()
assert.equal(versions.aquarius.status, OceanPlatformTechStatus.Working)

43
integration/utils/ddo-metadata-generator.ts
View File

@ -1,15 +1,30 @@
import { MetaData } from '../../src' // @oceanprotocol/squid
const metadata: Partial<MetaData> = {
base: {
main: {
name: undefined,
type: 'dataset',
description:
'Weather information of UK including temperature and humidity',
dateCreated: '2012-10-10T17:00:00Z',
datePublished: '2012-10-10T17:00:00Z',
author: 'Met Office',
license: 'CC-BY',
price: '21' + '0'.repeat(18),
files: [
{
index: 0,
url:
'https://raw.githubusercontent.com/oceanprotocol/squid-js/master/package.json'
},
{
index: 1,
url:
'https://raw.githubusercontent.com/oceanprotocol/squid-js/master/README.md'
}
]
},
additionalInformation: {
description:
'Weather information of UK including temperature and humidity',
copyrightHolder: 'Met Office',
workExample:
'423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68',
@ -27,20 +42,7 @@ const metadata: Partial<MetaData> = {
],
inLanguage: 'en',
categories: ['Economy', 'Data Science'],
tags: ['weather', 'uk', '2011', 'temperature', 'humidity'],
price: '21' + '0'.repeat(18),
files: [
{
index: 0,
url:
'https://raw.githubusercontent.com/oceanprotocol/squid-js/master/package.json'
},
{
index: 1,
url:
'https://raw.githubusercontent.com/oceanprotocol/squid-js/master/README.md'
}
]
tags: ['weather', 'uk', '2011', 'temperature', 'humidity']
}
}
@ -49,10 +51,13 @@ export const generateMetadata = (
price?: number
): Partial<MetaData> => ({
...metadata,
base: {
...metadata.base,
main: {
...metadata.main,
name,
price: (price || 21) + '0'.repeat(18)
},
additionalInformation: {
...metadata.additionalInformation
}
})

8
src/ddo/DDO.ts
View File

@ -102,7 +102,7 @@ export class DDO {
*/
public getChecksum(): string {
const { metadata } = this.findServiceByType('Metadata')
const { files, name, author, license } = metadata.base
const { files, name, author, license } = metadata.main
const values = [
...(files || []).map(({ checksum }) => checksum).filter(_ => !!_),
@ -121,7 +121,7 @@ export class DDO {
* Generates proof using personal sing.
* @param {Web3} web3 Web3 instance.
* @param {string} publicKey Public key to be used on personal sign.
* @param {string} password Password if it's requirted.
* @param {string} password Password if it's required.
* @return {Promise<Proof>} Proof object.
*/
public async generateProof(
@ -150,11 +150,11 @@ export class DDO {
*/
public addChecksum(): void {
const metadataService = this.findServiceByType('Metadata')
if (metadataService.metadata.base.checksum) {
if (metadataService.metadata.main.checksum) {
LoggerInstance.log('Checksum already exists')
return
}
metadataService.metadata.base.checksum = this.getChecksum()
metadataService.metadata.main.checksum = this.getChecksum()
}
/**

198
src/ddo/MetaData.ts
View File

@ -1,8 +1,4 @@
export interface StageRequirements {
computeServiceId?: string
serviceDefinitionId?: string
serverId?: string
serverInstances?: string
container: {
image: string
tag: string
@ -23,7 +19,7 @@ export interface StageOutput {
metadataUrl: string
secretStoreUrl: string
accessProxyUrl: string
metadata: MetaDataBase
metadata: MetaDataMain
}
export interface Stage {
@ -39,6 +35,38 @@ export interface Workflow {
stages: Stage[]
}
export interface Algorithm {
language: string
format?: string
version?: string
entrypoint: string
requirements: {
requirement: string
version: string
}
}
export interface ServiceDefinition {
auth: {
type: string
user?: string
password?: string
token?: string
}
endpoints: {
index: number
url: string
method: string
contentTypes: string[]
}
}
export interface Service {
spec?: string
specChecksum?: string
definition: ServiceDefinition
}
export interface File {
/**
* File name.
@ -105,10 +133,10 @@ export interface File {
}
/**
* Base attributes of Assets Metadata.
* @see https://github.com/oceanprotocol/OEPs/tree/master/8#base-attributes
* Main attributes of assets metadata.
* @see https://github.com/oceanprotocol/OEPs/tree/master/8
*/
export interface MetaDataBase {
export interface MetaDataMain {
/**
* Descriptive name of the Asset.
* @type {string}
@ -124,14 +152,6 @@ export interface MetaDataBase {
*/
type: 'dataset' | 'algorithm' | 'container' | 'workflow' | 'other'
/**
* Details of what the resource is. For a dataset, this attribute
* explains what the data represents and what it can be used for.
* @type {string}
* @example "Weather information of UK including temperature and humidity"
*/
description?: string
/**
* The date on which the asset was created by the originator in
* ISO 8601 format, Coordinated Universal Time.
@ -164,6 +184,83 @@ export interface MetaDataBase {
*/
license: string
/**
* Price of the asset.
* @type {string}
* @example "1000000000000000000"
*/
price: string
/**
* Array of File objects including the encrypted file urls and some additional information.
* @type {File[]}
*/
files: File[]
/**
* SHA3 hash of concatenated values: [list of all file checksums] + name + author + license + did
* @type {string}
*/
checksum?: string
encryptedFiles?: any
encryptedService?: any
workflow?: Workflow
algorithm?: Algorithm
service?: Service
}
/**
* Curation attributes of Assets Metadata.
* @see https://github.com/oceanprotocol/OEPs/tree/master/8
*/
export interface Curation {
/**
* Decimal value between 0 and 1. 0 is the default value.
* @type {number}
* @example 0.93
*/
rating: number
/**
* Number of votes. 0 is the default value.
* @type {number}
* @example 123
*/
numVotes: number
/**
* Schema applied to calculate the rating.
* @type {string}
* @example "Binary Voting"
*/
schema?: string
/**
* Flag unsuitable content.
* @type {boolean}
* @example true
*/
isListed?: boolean
}
/**
* Additional Information of Assets Metadata.
* @see https://github.com/oceanprotocol/OEPs/tree/master/8#additional-information
*/
export interface AdditionalInformation {
/**
* Details of what the resource is. For a dataset, this attribute
* explains what the data represents and what it can be used for.
* @type {string}
* @example "Weather information of UK including temperature and humidity"
*/
description?: string
/**
* The party holding the legal copyright. Empty by default.
* @type {string}
@ -220,62 +317,6 @@ export interface MetaDataBase {
*/
tags?: string[]
/**
* Price of the asset.
* @type {string}
* @example "1000000000000000000"
*/
price: string
/**
* Array of File objects including the encrypted file urls and some additional information.
* @type {File[]}
*/
files: File[]
/**
* SHA3 hash of concatenated values: [list of all file checksums] + name + author + license + did
* @type {string}
*/
checksum?: string
encryptedFiles?: any
workflow?: Workflow
}
/**
* Curation attributes of Assets Metadata.
* @see https://github.com/oceanprotocol/OEPs/tree/master/8#curation-attributes
*/
export interface Curation {
/**
* Decimal value between 0 and 1. 0 is the default value.
* @type {number}
* @example 0.93
*/
rating: number
/**
* Number of votes. 0 is the default value.
* @type {number}
* @example 123
*/
numVotes: number
/**
* Schema applied to calculate the rating.
* @type {string}
* @example "Binary Voting"
*/
schema?: string
}
/**
* Additional Information of Assets Metadata.
* @see https://github.com/oceanprotocol/OEPs/tree/master/8#additional-information
*/
export interface AdditionalInformation {
/**
* An indication of update latency - i.e. How often are updates expected (seldom,
* annually, quarterly, etc.), or is the resource static that is never expected
@ -283,28 +324,21 @@ export interface AdditionalInformation {
* @type {string}
* @example "yearly"
*/
updateFrequency: string
updateFrequency?: string
/**
* A link to machine-readable structured markup (such as ttl/json-ld/rdf)
* describing the dataset.
* @type {StructuredMarkup[]}
*/
structuredMarkup: {
structuredMarkup?: {
uri: string
mediaType: string
}[]
/**
* Checksum of attributes to be able to compare if there are changes in
* the asset that you are purchasing.
* @type {string}
*/
checksum: string
}
export interface MetaData {
main: MetaDataMain
additionalInformation?: AdditionalInformation
base: MetaDataBase
curation?: Curation
}

4
src/keeper/contracts/templates/EscrowAccessSecretStoreTemplate.ts
View File

@ -59,7 +59,7 @@ export class EscrowAccessSecretStoreTemplate extends AgreementTemplate {
) {
return !!(await this.createFullAgreement(
ddo.shortId(),
ddo.findServiceByType('Metadata').metadata.base.price,
ddo.findServiceByType('Metadata').metadata.main.price,
consumer,
from,
agreementId
@ -79,7 +79,7 @@ export class EscrowAccessSecretStoreTemplate extends AgreementTemplate {
} = await this.createFullAgreementData(
agreementId,
ddo.shortId(),
ddo.findServiceByType('Metadata').metadata.base.price,
ddo.findServiceByType('Metadata').metadata.main.price,
consumer
)
return [

16
src/ocean/OceanAssets.ts
View File

@ -81,7 +81,7 @@ export class OceanAssets extends Instantiable {
observer.next(CreateProgressStep.EncryptingFiles)
const encryptedFiles = await this.ocean.secretStore.encrypt(
did.getId(),
metadata.base.files,
metadata.main.files,
publisher
)
this.logger.log('Files encrypted')
@ -135,11 +135,11 @@ export class OceanAssets extends Instantiable {
// Overwrites defaults
...metadata,
// Cleaning not needed information
base: {
...metadata.base,
main: {
...metadata.main,
contentUrls: undefined,
encryptedFiles,
files: metadata.base.files.map(
files: metadata.main.files.map(
(file, index) => ({
...file,
index,
@ -165,7 +165,7 @@ export class OceanAssets extends Instantiable {
})) as Service[]
})
// Overwritte initial service agreement conditions
// Overwrite initial service agreement conditions
const rawConditions = await templates.escrowAccessSecretStoreTemplate.getServiceAgreementTemplateConditions()
const conditions = fillConditionsWithDDO(rawConditions, ddo)
serviceAgreementTemplate.conditions = conditions
@ -237,7 +237,7 @@ export class OceanAssets extends Instantiable {
const accessService = ddo.findServiceById(serviceDefinitionId)
const { files } = metadata.base
const { files } = metadata.main
const { serviceEndpoint } = accessService
@ -265,7 +265,7 @@ export class OceanAssets extends Instantiable {
} else {
const files = await this.ocean.secretStore.decrypt(
did,
ddo.findServiceByType('Metadata').metadata.base.encryptedFiles,
ddo.findServiceByType('Metadata').metadata.main.encryptedFiles,
consumerAccount,
ddo.findServiceByType('Authorization').serviceEndpoint
)
@ -327,7 +327,7 @@ export class OceanAssets extends Instantiable {
observer.next(OrderProgressStep.LockingPayment)
const paid = await oceanAgreements.conditions.lockReward(
agreementId,
metadata.base.price,
metadata.main.price,
consumer
)
observer.next(OrderProgressStep.LockedPayment)

2
src/utils/DDOHelpers.ts
View File

@ -13,7 +13,7 @@ function fillParameterWithDDO(
case 'amount':
case 'price':
return String(
ddo.findServiceByType('Metadata').metadata.base.price
ddo.findServiceByType('Metadata').metadata.main.price
)
case 'assetId':
case 'documentId':

60
test/ddo/DDO.test.ts
View File

@ -91,41 +91,13 @@ describe('DDO', () => {
serviceEndpoint:
'http://myaquarius.org/api/v1/provider/assets/metadata/{did}',
metadata: {
base: {
main: {
name: 'UK Weather information 2011',
type: 'dataset',
description:
'Weather information of UK including temperature and humidity',
dateCreated: '2012-10-10T17:00:000Z',
datePublished: '2012-10-10T17:00:000Z',
author: 'Met Office',
license: 'CC-BY',
copyrightHolder: 'Met Office',
workExample:
'423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68',
links: [
{
sample1:
'http://data.ceda.ac.uk/badc/ukcp09/data/gridded-land-obs/gridded-land-obs-daily/'
},
{
sample2:
'http://data.ceda.ac.uk/badc/ukcp09/data/gridded-land-obs/gridded-land-obs-averages-25km/'
},
{
fieldsDescription:
'http://data.ceda.ac.uk/badc/ukcp09/'
}
],
inLanguage: 'en',
categories: ['Economy', 'Data Science'],
tags: [
'weather',
'uk',
'2011',
'temperature',
'humidity'
],
price: 10,
files: [
{
@ -152,7 +124,35 @@ describe('DDO', () => {
schema: 'Binary Voting'
},
additionalInformation: {
updateFrecuency: 'yearly',
description:
'Weather information of UK including temperature and humidity',
copyrightHolder: 'Met Office',
workExample:
'423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68',
links: [
{
sample1:
'http://data.ceda.ac.uk/badc/ukcp09/data/gridded-land-obs/gridded-land-obs-daily/'
},
{
sample2:
'http://data.ceda.ac.uk/badc/ukcp09/data/gridded-land-obs/gridded-land-obs-averages-25km/'
},
{
fieldsDescription:
'http://data.ceda.ac.uk/badc/ukcp09/'
}
],
inLanguage: 'en',
categories: ['Economy', 'Data Science'],
tags: [
'weather',
'uk',
'2011',
'temperature',
'humidity'
],
updateFrequency: 'yearly',
structuredMarkup: [
{
uri:

56
test/testdata/ddo.json vendored
View File

@ -58,9 +58,7 @@
},
"events": {
"PaymentLocked": {
"actorType": [
"publisher"
],
"actorType": ["publisher"],
"handlers": [
{
"moduleName": "accessControl",
@ -84,9 +82,7 @@
},
"events": {
"PaymentReleased": {
"actorType": [
"publisher"
],
"actorType": ["publisher"],
"handlers": [
{
"moduleName": "serviceAgreement",
@ -110,9 +106,7 @@
},
"events": {
"AccessGranted": {
"actorType": [
"consumer"
],
"actorType": ["consumer"],
"handlers": [
{
"moduleName": "asset",
@ -136,9 +130,7 @@
},
"events": {
"PaymentRefund": {
"actorType": [
"consumer"
],
"actorType": ["consumer"],
"handlers": [
{
"moduleName": "serviceAgreement",
@ -165,26 +157,34 @@
"base": {
"name": "UK Weather information 2011",
"type": "dataset",
"description": "Weather information of UK including temperature and humidity",
"dateCreated": "2012-10-10T17:00:000Z",
"author": "Met Office",
"license": "CC-BY",
"price": 10,
"files": [
{
"index": 0,
"url": "https://testocnfiles.blob.core.windows.net/testfiles/testzkp.zip",
"checksum": "085340abffh21495345af97c6b0e761",
"contentLength": 12324
},
{
"url": "https://testocnfiles.blob.core.windows.net/testfiles/testzkp2.zip"
}
]
},
"curation": {
"rating": 0.93,
"numVotes": 123,
"schema": "Binary Voting"
},
"additionalInformation": {
"description": "Weather information of UK including temperature and humidity",
"copyrightHolder": "Met Office",
"workExample": "423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68",
"contentUrls": [
"https://testocnfiles.blob.core.windows.net/testfiles/testzkp.zip"
],
"files": [
{
"index": 0,
"url": "https://testocnfiles.blob.core.windows.net/testfiles/testzkp.zip",
"checksum": "085340abffh21495345af97c6b0e761",
"contentLength": 12324
},
{
"url": "https://testocnfiles.blob.core.windows.net/testfiles/testzkp2.zip"
}
],
"links": [
{
"name": "Sample of Asset Data",
@ -200,14 +200,6 @@
"inLanguage": "en",
"categories": ["Economy", "Data Science"],
"tags": ["weather", "uk", "2011", "temperature", "humidity"],
"price": 10
},
"curation": {
"rating": 0.93,
"numVotes": 123,
"schema": "Binary Voting"
},
"additionalInformation": {
"updateFrequency": "yearly",
"structuredMarkup": [
{