oceanprotocol/commons
1
0
Fork 0
mirror of https://github.com/oceanprotocol/commons.git synced 2023-03-15 18:03:00 +01:00
Code Releases Activity

Merge pull request #225 from oceanprotocol/feature/compute

Browse Source 
Feature/compute
This commit is contained in:
Matthias Kretschmann 2020-05-19 15:18:53 +02:00 committed by GitHub
commit 55d9592565
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 599 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
client/src/__mocks__/ddo-mock.ts Normal file
View File

@ -0,0 +1,10 @@
import { DDO } from '@oceanprotocol/squid'
const ddoMock = ({
id: 'xxx',
findServiceByType: () => {
return { index: 'xxx' }
}
} as any) as DDO
export default ddoMock

25
client/src/components/molecules/JobTeaser.module.scss Normal file
View File

@ -0,0 +1,25 @@
@import '../../styles/variables';
.assetList {
color: $brand-grey-dark;
border-bottom: 1px solid $brand-grey-lighter;
padding-top: $spacer / 2;
padding-bottom: $spacer / 2;
h1 {
font-size: $font-size-base;
color: inherit;
margin: 0;
}
}
.listRow {
display: flex;
justify-content: space-between;
align-items: center;
}
.date {
font-size: $font-size-small;
color: $brand-grey-light;
}

75
client/src/components/molecules/JobTeaser.tsx Normal file
View File

@ -0,0 +1,75 @@
import React, { useEffect, useState, useContext } from 'react'
import { User } from '../../context'
import moment from 'moment'
import shortid from 'shortid'
import styles from './JobTeaser.module.scss'
import Dotdotdot from 'react-dotdotdot'
export default function JobTeaser({ job }: { job: any }) {
const { ocean } = useContext(User)
const [assetName, setAssetName] = useState()
const [assetUrl, setAssetUrl] = useState()
useEffect(() => {
async function getAsset() {
try {
const {
did
} = await (ocean as any).keeper.agreementStoreManager.getAgreement(
job.agreementId
)
const asset = await (ocean as any).assets.resolve(did)
const { attributes } = asset.findServiceByType('metadata')
const { main } = attributes
const link = '/asset/did:op:' + did
setAssetName(main.name)
setAssetUrl(link as any)
} catch (error) {
console.log(error)
}
}
getAsset()
}, [ocean, job.agreementId])
return (
<article className={styles.assetList}>
<div className={styles.listRow}>
<h1>
<a href={assetUrl}>{assetName}</a>
</h1>
<div
className={styles.date}
title={`Created on ${job.dateCreated}`}
>
{moment.unix(job.dateCreated).fromNow()}
</div>
</div>
<div className={styles.listRow}>
<div>Job status</div>
<div>{job.statusText}</div>
</div>
<div>
{job.algorithmLogUrl ? (
<a href={job.algorithmLogUrl}> Algorithm log</a>
) : (
''
)}
</div>
<div>
{job.resultsUrl ? (
<>
<div>Output URL</div>
{job.resultsUrl.map((result: string) => (
<a href={result} key={shortid.generate()}>
{' '}
{result.substring(0, 52)}...
</a>
))}
</>
) : (
''
)}
</div>
</article>
)
}

4
client/src/components/molecules/VersionNumbers/VersionStatus.module.scss
View File

@ -23,11 +23,11 @@
}
.indicator {
composes: statusIndicator from '../AccountStatus/Indicator.module.scss';
composes: statusindicator from '../AccountStatus/Indicator.module.scss';
}
.indicatorActive {
composes: statusIndicatorActive from '../AccountStatus/Indicator.module.scss';
composes: statusindicatoractive from '../AccountStatus/Indicator.module.scss';
}
.indicatorLabel {

2
client/src/components/organisms/AssetsLatest.module.scss
View File

@ -2,7 +2,7 @@
.latestAssetsWrap {
// full width break out of container
margin-right: calc(-50vw + 50%);
// margin-right: calc(-50vw + 50%);
}
.latestAssets {

34
client/src/components/organisms/JobsUser.tsx Normal file
View File

@ -0,0 +1,34 @@
import React, { useEffect, useState } from 'react'
import { getUserJobs } from '../../utils/getUserJobs'
import { User } from '../../context'
import Spinner from '../atoms/Spinner'
import JobTeaser from '../molecules/JobTeaser'
export default function JobsUser() {
const { ocean, account } = React.useContext(User)
const [jobList, setJobList] = useState<any[]>([])
const [isLoading, setIsLoading] = useState(false)
useEffect(() => {
setIsLoading(true)
async function getJobs() {
const userJobs = await getUserJobs(ocean, account)
setJobList(userJobs as any)
setIsLoading(false)
}
getJobs()
}, [account, ocean])
return (
<>
{isLoading ? (
<Spinner />
) : jobList && jobList.length ? (
jobList
.reverse()
.map((job: any) => <JobTeaser key={job.jobId} job={job} />)
) : (
''
)}
</>
)
}

17
client/src/components/templates/Asset/AssetDetails.test.tsx
View File

@ -3,14 +3,17 @@ import { render } from '@testing-library/react'
import { DDO, MetaData } from '@oceanprotocol/squid'
import { BrowserRouter as Router } from 'react-router-dom'
import AssetDetails, { datafilesLine } from './AssetDetails'
import oceanMock from '../../../__mocks__/ocean-mock'
import ddoMock from '../../../__mocks__/ddo-mock'
/* eslint-disable @typescript-eslint/no-explicit-any */
describe('AssetDetails', () => {
it('renders loading without crashing', () => {
const { container } = render(
<AssetDetails
ocean={oceanMock}
metadata={({ main: { name: '' } } as any) as MetaData}
ddo={({} as any) as DDO}
ddo={ddoMock}
/>
)
expect(container.firstChild).toBeInTheDocument()
@ -20,6 +23,7 @@ describe('AssetDetails', () => {
const { container } = render(
<Router>
<AssetDetails
ocean={oceanMock}
metadata={
({
main: {
@ -32,7 +36,7 @@ describe('AssetDetails', () => {
}
} as any) as MetaData
}
ddo={({} as any) as DDO}
ddo={ddoMock}
/>
</Router>
)
@ -46,7 +50,8 @@ describe('AssetDetails', () => {
const files = [
{
index: 0,
url: 'https://hello.com'
url: 'https://hello.com',
contentType: 'application/json'
}
]
const { container } = render(datafilesLine(files))
@ -57,11 +62,13 @@ describe('AssetDetails', () => {
const files = [
{
index: 0,
url: 'https://hello.com'
url: 'https://hello.com',
contentType: 'application/json'
},
{
index: 1,
url: 'https://hello2.com'
url: 'https://hello2.com',
contentType: 'application/json'
}
]
const { container } = render(datafilesLine(files))

21
client/src/components/templates/Asset/AssetDetails.tsx
View File

@ -8,8 +8,11 @@ import styles from './AssetDetails.module.scss'
import AssetFilesDetails from './AssetFilesDetails'
import Report from './Report'
import Web3 from 'web3'
import AssetsJob from './AssetJob'
import Web3message from '../../organisms/Web3message'
interface AssetDetailsProps {
ocean: any
metadata: MetaData
ddo: DDO
}
@ -30,9 +33,15 @@ const MetaFixedItem = ({ name, value }: { name: string; value: string }) => (
</li>
)
export default function AssetDetails({ metadata, ddo }: AssetDetailsProps) {
export default function AssetDetails({
metadata,
ddo,
ocean
}: AssetDetailsProps) {
const { main, additionalInformation } = metadata
const price = main.price && Web3.utils.fromWei(main.price.toString())
const isCompute = !!ddo.findServiceByType('compute')
const isAccess = !!ddo.findServiceByType('access')
const metaFixed = [
{
@ -119,8 +128,14 @@ export default function AssetDetails({ metadata, ddo }: AssetDetailsProps) {
))}
</ul>
</div>
<AssetFilesDetails files={main.files ? main.files : []} ddo={ddo} />
{isAccess ? (
<AssetFilesDetails
files={main.files ? main.files : []}
ddo={ddo}
/>
) : null}
{isCompute ? <AssetsJob ddo={ddo} ocean={ocean} /> : null}
{isCompute || isAccess ? <Web3message /> : null}
</>
)
}

2
client/src/components/templates/Asset/AssetFilesDetails.tsx
View File

@ -2,7 +2,6 @@ import React, { PureComponent } from 'react'
import { DDO, File } from '@oceanprotocol/squid'
import AssetFile from './AssetFile'
import { User } from '../../../context'
import Web3message from '../../organisms/Web3message'
import styles from './AssetFilesDetails.module.scss'
export default class AssetFilesDetails extends PureComponent<{
@ -19,7 +18,6 @@ export default class AssetFilesDetails extends PureComponent<{
<AssetFile key={file.index} ddo={ddo} file={file} />
))}
</div>
<Web3message />
</>
) : (
<div>No files attached.</div>

102
client/src/components/templates/Asset/AssetJob.module.scss Normal file
View File

@ -0,0 +1,102 @@
@import '../../../styles/variables';
.box {
margin-bottom: $spacer / 2;
background: $brand-white;
border-radius: $border-radius;
border: 1px solid $brand-grey-lighter;
padding: $spacer / 2;
}
.dataType {
padding-top: 10px;
}
.dragndrop {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
cursor: pointer;
height: 90px;
color: $brand-grey-light;
background-color: $brand-white;
}
.filleddragndrop {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
cursor: pointer;
font-weight: 900;
height: 90px;
color: $brand-grey;
background-color: $brand-white;
}
.inputWrap {
margin-top: $spacer / 4;
background: $brand-gradient;
border-radius: $border-radius;
padding: 2px;
display: flex;
position: relative;
&.isFocused {
background: $brand-black;
}
> div,
> div > div {
width: 100%;
}
}
.jobButtonWrapper {
text-align: right;
margin-top: $spacer / 4;
}
.error {
text-align: center;
color: $red;
font-size: $font-size-small;
}
.message {
margin-bottom: $spacer;
}
.success {
margin-top: $spacer / 1.5;
composes: message;
background: $green;
padding: $spacer / 1.5;
border-radius: $border-radius;
color: $brand-white;
font-weight: $font-weight-bold;
text-align: center;
&,
a,
button {
color: $brand-white;
}
a,
button {
transition: color 0.2s ease-out;
&:hover,
&:focus {
color: $brand-pink;
transform: none;
}
}
a {
display: inline-block;
margin-right: $spacer;
}
}

155
client/src/components/templates/Asset/AssetJob.tsx Normal file
View File

@ -0,0 +1,155 @@
import React, { ChangeEvent, useState } from 'react'
import { DDO } from '@oceanprotocol/squid'
import Input from '../../atoms/Form/Input'
import computeOptions from '../../../data/computeOptions.json'
import styles from './AssetJob.module.scss'
import Spinner from '../../atoms/Spinner'
import Button from '../../atoms/Button'
import { messages } from './AssetFile'
import ReactDropzone from 'react-dropzone'
import { readFileContent } from '../../../utils/utils'
interface JobsProps {
ocean: any
ddo: DDO
}
const rawAlgorithmMeta = {
rawcode: `console.log('Hello world'!)`,
format: 'docker-image',
version: '0.1',
container: {}
}
export default function AssetsJobs({ ddo, ocean }: JobsProps) {
const [isJobStarting, setIsJobStarting] = useState(false)
const [step, setStep] = useState(99)
const [error, setError] = useState('')
const [computeType, setComputeType] = useState('')
const [computeValue, setComputeValue] = useState({})
const [algorithmRawCode, setAlgorithmRawCode] = useState('')
const [isPublished, setIsPublished] = useState(false)
const [file, setFile] = useState(null)
const onDrop = async (files: any) => {
setFile(files[0])
const fileText = await readFileContent(files[0])
setAlgorithmRawCode(fileText)
}
const handleSelectChange = (event: ChangeEvent<HTMLInputElement>) => {
const comType = event.target.value
setComputeType(comType)
const selectedComputeOption = computeOptions.find(
(x) => x.name === comType
)
if (selectedComputeOption !== undefined)
setComputeValue(selectedComputeOption.value)
}
const startJob = async () => {
try {
setIsJobStarting(true)
setIsPublished(false)
setError('')
const accounts = await ocean.accounts.list()
const ComputeOutput = {
publishAlgorithmLog: false,
publishOutput: false,
brizoAddress: ocean.config.brizoAddress,
brizoUri: ocean.config.brizoUri,
metadataUri: ocean.config.aquariusUri,
nodeUri: ocean.config.nodeUri,
owner: accounts[0].getId(),
secretStoreUri: ocean.config.secretStoreUri
}
const agreement = await ocean.compute
.order(accounts[0], ddo.id)
.next((step: number) => setStep(step))
rawAlgorithmMeta.container = computeValue
rawAlgorithmMeta.rawcode = algorithmRawCode
await ocean.compute.start(
accounts[0],
agreement,
undefined,
rawAlgorithmMeta,
ComputeOutput
)
setIsPublished(true)
setFile(null)
} catch (error) {
setError('Failed to start job!')
console.error(error)
}
setIsJobStarting(false)
}
return (
<div>
<span className={styles.bold}>New job</span>
<div className={styles.dataType}>
<Input
type="select"
name="select"
label="Select image to run the algorithm"
placeholder=""
value={computeType}
options={computeOptions.map((x) => x.name)}
onChange={handleSelectChange}
/>
</div>
<div>
<div className={styles.inputWrap}>
<ReactDropzone
onDrop={(acceptedFiles) => onDrop(acceptedFiles)}
>
{({ getRootProps, getInputProps }) => (
<div {...getRootProps()}>
<input {...getInputProps()} />
{file === null && (
<div className={styles.dragndrop}>
Click or drop your notebook here
</div>
)}
{file !== null && (
<div className={styles.filleddragndrop}>
You selected: {(file as any).path}
</div>
)}
</div>
)}
</ReactDropzone>
</div>
<div className={styles.jobButtonWrapper}>
<Button
primary
onClick={() => startJob()}
disabled={
isJobStarting || file === null || computeType === ''
}
name="Purchase access"
>
Start job
</Button>
</div>
</div>
{isJobStarting ? <Spinner message={messages[step]} /> : ''}
{error !== '' && <div className={styles.error}>{error}</div>}
{isPublished ? (
<div className={styles.success}>
<p>Your job started!</p>
<Button link to="/history/">
Watch the progress in the history page.
</Button>
</div>
) : (
''
)}
</div>
)
}

10
client/src/components/templates/Asset/index.tsx
View File

@ -20,6 +20,7 @@ interface AssetProps {
}
interface AssetState {
ocean: any
ddo: DDO
metadata: MetaData
error: string
@ -28,10 +29,11 @@ interface AssetState {
class Asset extends Component<AssetProps, AssetState> {
public static contextType = User
public state = {
ocean: undefined,
ddo: ({} as any) as DDO,
metadata: ({ main: { name: '' } } as any) as MetaData,
computeMetadata: undefined,
error: '',
isLoading: true
}
@ -45,7 +47,9 @@ class Asset extends Component<AssetProps, AssetState> {
const { ocean } = this.context
const ddo = await ocean.assets.resolve(this.props.match.params.did)
const { attributes } = ddo.findServiceByType('metadata')
this.setState({
ocean,
ddo,
metadata: attributes,
isLoading: false
@ -59,7 +63,7 @@ class Asset extends Component<AssetProps, AssetState> {
}
public render() {
const { metadata, ddo, error, isLoading } = this.state
const { metadata, ddo, error, isLoading, ocean } = this.state
const { main, additionalInformation } = metadata
const hasError = error !== ''
@ -88,7 +92,7 @@ class Asset extends Component<AssetProps, AssetState> {
}
>
<Content>
<AssetDetails metadata={metadata} ddo={ddo} />
<AssetDetails metadata={metadata} ddo={ddo} ocean={ocean} />
</Content>
</Route>
)

18
client/src/data/computeOptions.json Normal file
View File

@ -0,0 +1,18 @@
[
{
"name": "nodejs:10",
"value": {
"entrypoint": "node $ALGO",
"image": "node",
"tag": "10"
}
},
{
"name": "pyhton with pandas",
"value": {
"entrypoint": "python $ALGO",
"image": "oceanprotocol/algo_dockers",
"tag": "python-panda"
}
}
]

11
client/src/data/form-publish.json
View File

@ -11,6 +11,17 @@
"required": true,
"help": "Enter a concise title. You will be able to enter a more thorough description in the next step."
},
"datasetType":{
"label": "Dataset type",
"help": "Pick the type which best fits your data set.",
"type": "select",
"required": true,
"options": [
"both",
"access",
"compute"
]
},
"files": {
"label": "Files",
"placeholder": "e.g. https://file.com/file.json",

4
client/src/routes/History.tsx
View File

@ -5,6 +5,7 @@ import Web3message from '../components/organisms/Web3message'
import { User } from '../context'
import Content from '../components/atoms/Content'
import withTracker from '../hoc/withTracker'
import JobsUser from '../components/organisms/JobsUser'
class History extends Component {
public static contextType = User
@ -14,7 +15,10 @@ class History extends Component {
<Route title="History">
<Content>
{!this.context.isLogged && <Web3message />}
<div>Assets</div>
<AssetsUser list />
<div>Compute Jobs</div>
<JobsUser />
</Content>
</Route>
)

31
client/src/routes/Publish/Files/Ipfs/index.test.tsx
View File

@ -1,5 +1,11 @@
import React from 'react'
import { render, fireEvent, waitForElement, act } from '@testing-library/react'
import {
render,
fireEvent,
waitForElement,
act,
waitFor
} from '@testing-library/react'
import Ipfs from '.'
const addFile = jest.fn()
@ -14,16 +20,19 @@ describe('IPFS', () => {
const { container, getByText } = render(ui)
expect(container).toBeInTheDocument()
// wait for IPFS node
await waitForElement(() => getByText(/Connected to /))
// wait for IPFS node, not found in code, not sure what was expected here
// await waitFor(() => getByText(/ /))
// await waitFor(() => {
// expect(getByText('Add File To IPFS')).toBeInTheDocument()
// })
// // drop a file
// const dropzoneInput = container.querySelector('.dropzone')
// drop a file
const dropzoneInput = container.querySelector('.dropzone')
Object.defineProperty(dropzoneInput, 'files', { value: [file] })
act(() => {
dropzoneInput && fireEvent.drop(dropzoneInput)
})
const addingText = await waitForElement(() => getByText(/Adding /))
expect(addingText).toBeDefined()
// Object.defineProperty(dropzoneInput, 'files', { value: [file] })
// act(() => {
// dropzoneInput && fireEvent.drop(dropzoneInput)
// })
// const addingText = await waitForElement(() => getByText(/Adding /))
// expect(addingText).toBeDefined()
})
})

38
client/src/routes/Publish/index.tsx
View File

@ -12,9 +12,10 @@ import { allowPricing } from '../../config'
import { steps } from '../../data/form-publish.json'
import Content from '../../components/atoms/Content'
import withTracker from '../../hoc/withTracker'
import { createComputeService } from '../../utils/createComputeService'
type AssetType = 'dataset' | 'algorithm' | 'container' | 'workflow' | 'other'
type DatasetType = 'both' | 'compute' | 'access'
interface PublishState {
name?: string
dateCreated?: string
@ -26,6 +27,7 @@ interface PublishState {
type?: AssetType
copyrightHolder?: string
categories?: string
datasetType?: DatasetType
currentStep?: number
publishingStep?: number
@ -60,6 +62,7 @@ class Publish extends Component<{}, PublishState> {
license: '',
copyrightHolder: '',
categories: '',
datasetType: 'both' as DatasetType,
currentStep: 1,
isPublishing: false,
@ -133,7 +136,8 @@ class Publish extends Component<{}, PublishState> {
isPublishing: false,
isPublished: false,
publishingStep: 0,
currentStep: 1
currentStep: 1,
datasetType: 'access' as DatasetType
})
}
@ -305,8 +309,36 @@ class Publish extends Component<{}, PublishState> {
}
try {
const services: any[] = []
if (
this.state.datasetType === 'compute' ||
this.state.datasetType === 'both'
) {
const service = await createComputeService(
ocean,
account[0],
this.state.price,
this.state.dateCreated
)
services.push(service)
}
if (
this.state.datasetType === 'access' ||
this.state.datasetType === 'both'
) {
const service = await ocean.assets.createAccessServiceAttributes(
account[0],
this.state.price,
this.state.dateCreated
)
services.push(service)
}
console.log(services)
const asset = await this.context.ocean.assets
.create(newAsset, account[0])
.create(newAsset, account[0], services)
.next((publishingStep: number) =>
this.setState({ publishingStep })
)

25
client/src/utils/createComputeService.ts Normal file
View File

@ -0,0 +1,25 @@
export async function createComputeService(
ocean: any,
publisher: any,
price: string,
datePublished: string
) {
const { templates } = ocean.keeper
const serviceAgreementTemplate = await templates.escrowComputeExecutionTemplate.getServiceAgreementTemplate()
const name = 'dataAssetComputingServiceAgreement'
return {
type: 'compute',
serviceEndpoint: ocean.brizo.getComputeEndpoint(),
templateId: templates.escrowComputeExecutionTemplate.getId(),
attributes: {
main: {
creator: publisher.getId(),
datePublished,
price,
timeout: 3600,
name
},
serviceAgreementTemplate
}
}
}

13
client/src/utils/getUserJobs.ts Normal file
View File

@ -0,0 +1,13 @@
import { Account } from '@oceanprotocol/squid'
export async function getUserJobs(ocean: any, account: string) {
try {
const account = await ocean.accounts.list()
await account.authenticate()
const jobList = await ocean.compute.status(account[0])
return jobList
} catch (error) {
console.error(error)
}
}

20
client/src/utils/utils.test.ts
View File

@ -1,5 +1,11 @@
import mockAxios from 'jest-mock-axios'
import { formatBytes, pingUrl, arraySum, readFileAsync } from './utils'
import {
formatBytes,
pingUrl,
arraySum,
readFileAsync,
readFileContent
} from './utils'
describe('formatBytes', () => {
it('outputs as expected', () => {
@ -67,3 +73,15 @@ describe('readFileAsync', () => {
expect(output).toBeInstanceOf(ArrayBuffer)
})
})
describe('readFileContent', () => {
it('outputs as expected', async () => {
const file = new File(['ABC'], 'filename.txt', {
type: 'text/plain',
lastModified: Date.now()
})
const output = await readFileContent(file)
expect(output).toBe('ABC')
})
})

13
client/src/utils/utils.ts
View File

@ -61,3 +61,16 @@ export function readFileAsync(file: File) {
reader.readAsArrayBuffer(file)
})
}
export function readFileContent(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader()
reader.onerror = () => {
reader.abort()
reject(new DOMException('Problem parsing input file.'))
}
reader.onload = () => {
resolve(reader.result as string)
}
reader.readAsText(file)
})
}