From a83841e4282893d610f53fbe363fd306f506267d Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sun, 24 May 2020 01:06:08 -0700 Subject: [PATCH 01/28] setup CtD --- content/setup/ctd.md | 134 ++++++++++++++++++++++++++++++++++++++++ data/sidebars/setup.yml | 4 ++ 2 files changed, 138 insertions(+) create mode 100644 content/setup/ctd.md diff --git a/content/setup/ctd.md b/content/setup/ctd.md new file mode 100644 index 00000000..79179372 --- /dev/null +++ b/content/setup/ctd.md @@ -0,0 +1,134 @@ +--- +title: Set Up a Compute-to-Data enviroment +description: Set Up a Compute-to-Data enviroment. +--- + + +## Requirments + +At the time of writing, we need the following: + +* working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ +* a working Kubernetes cluster (minikube is a good start) +* a working kubectl connected to the k8 cluster +* one folder (/ocean/operator-service/), in which we will download the following: + * [postgres-configmap.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-configmap.yaml) + * [postgres-storage.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-storage.yaml) + * [postgres-deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-deployment.yaml) + * [postgres-service.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-service.yaml) + * [deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/deployment.yaml) + * [role_binding.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/role_binding.yaml) + * [service_account.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/service_account.yaml) +* one folder (/ocean/operator-engine/), in which we will download the following: + * [sa.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/sa.yml) + * [binding.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/binding.yml) + * [operator.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/operator.yml) + * [computejob-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/computejob-crd.yml) + * [workflow-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/workflow-crd.yml) + +## Customize your Operator Service deployment + +The following resources needs attention: + + * postgres-configmap.yaml - contains secrets for the postgresql deployment + * deployment.yaml - ALGO_POD_TIMEOUT = Allowed time for a algorithm to run. If it exceeded this value (in minutes), it's going to get killed + +## Customize your Operator Engine deployment + +The following resource needs attention: + +* operator.yaml - contains configuration variabiles + * ACCOUNT_JSON , ACCOUNT_PASSWORD = Defines the account that is going to be used when publishing results back to OceanProtocol + * AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID , AWS_REGION = S3 credidentials for the logs and output buckets + * AWS_BUCKET_OUTPUT = Bucket that will hold the output data (algorithm logs & algorithm output) + * AWS_BUCKET_ADMINLOGS = Bucket that will hold the admin logs (logs from pod-configure & pod-publish) + * STORAGE_CLASS = Storage class to use (see next section) + +## Storage class + +For minikube, you can use 'standard' class. + +For AWS , please make sure that your class allocates volumes in the same region and zone in which you are running your pods. + +We created our own 'standard' class in AWS: +``` + kubectl get storageclass standard -o yaml +``` + +``` + allowedTopologies: + - matchLabelExpressions: + - key: failure-domain.beta.kubernetes.io/zone + values: + - us-east-1a + apiVersion: storage.k8s.io/v1 + kind: StorageClass + parameters: + fsType: ext4 + type: gp2 + provisioner: kubernetes.io/aws-ebs + reclaimPolicy: Delete + volumeBindingMode: Immediate +``` + +## Create namespaces + +``` +kubectl create ns ocean-operator +kubectl create ns ocean-compute +``` + +## Deploy Operator-Service + +``` +kubectl config set-context --current --namespace ocean-operator +kubectl create -f /ocean/operator-service/postgres-configmap.yaml +kubectl create -f /ocean/operator-service/postgres-storage.yaml +kubectl create -f /ocean/operator-service/postgres-deployment.yaml +kubectl create -f /ocean/operator-service/postgresql-service.yaml +kubectl apply -f /ocean/operator-service/deployment.yaml +kubectl apply -f /ocean/operator-service/role_binding.yaml +kubectl apply -f /ocean/operator-service/service_account.yaml +``` + +## Deploy Operator-Engine + +``` +kubectl config set-context --current --namespace ocean-compute +kubectl apply -f /ocean/operator-engine/sa.yml +kubectl apply -f /ocean/operator-engine/binding.yml +kubectl apply -f /ocean/operator-engine/operator.yml +kubectl apply -f /ocean/operator-engine/computejob-crd.yaml +kubectl apply -f /ocean/operator-engine/workflow-crd.yaml +kubectl create -f /ocean/operator-service/postgres-configmap.yaml +``` + +## Expose Operator - Service + +``` +kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 +``` + +Run a port forward or create your ingress service (not covered here): + +``` +kubectl -n ocean-operator port-forward svc/operator-api 8050 +``` + +## Initialize database + +If your cluster is running on example.com: + +``` +curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: application/json" +``` + +## Update Brizo + +Update Brizo by adding or updating the OPERATOR_SERVICE_URL env (in /ocean/barge/compose-files/brizo.yaml) + +``` +OPERATOR_SERVICE_URL: http://example.com:8050/ +``` + +## Restart Barge with updated Brizo configuration diff --git a/data/sidebars/setup.yml b/data/sidebars/setup.yml index 76b843a2..5d791811 100644 --- a/data/sidebars/setup.yml +++ b/data/sidebars/setup.yml @@ -17,3 +17,7 @@ items: - title: Run a Keeper link: /setup/keeper/ +- group: Compute-to-Data Enviroment + items: + - title: Run a Compute-to-Data Enviroment + link: /setup/ctd/ From 5ab37492e4262532df128cacce8b38162a7455c7 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sun, 24 May 2020 01:25:58 -0700 Subject: [PATCH 02/28] fix lint --- content/setup/ctd.md | 73 ++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 79179372..8fa510fb 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -8,41 +8,41 @@ description: Set Up a Compute-to-Data enviroment. At the time of writing, we need the following: -* working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ -* a working Kubernetes cluster (minikube is a good start) -* a working kubectl connected to the k8 cluster -* one folder (/ocean/operator-service/), in which we will download the following: - * [postgres-configmap.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-configmap.yaml) - * [postgres-storage.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-storage.yaml) - * [postgres-deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-deployment.yaml) - * [postgres-service.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-service.yaml) - * [deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/deployment.yaml) - * [role_binding.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/role_binding.yaml) - * [service_account.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/service_account.yaml) -* one folder (/ocean/operator-engine/), in which we will download the following: - * [sa.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/sa.yml) - * [binding.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/binding.yml) - * [operator.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/operator.yml) - * [computejob-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/computejob-crd.yml) - * [workflow-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/workflow-crd.yml) +- working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ +- a working Kubernetes cluster (minikube is a good start) +- a working kubectl connected to the k8 cluster +- one folder (/ocean/operator-service/), in which we will download the following: + - [postgres-configmap.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-configmap.yaml) + - [postgres-storage.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-storage.yaml) + - [postgres-deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-deployment.yaml) + - [postgres-service.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-service.yaml) + - [deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/deployment.yaml) + - [role_binding.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/role_binding.yaml) + - [service_account.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/service_account.yaml) +- one folder (/ocean/operator-engine/), in which we will download the following: + - [sa.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/sa.yml) + - [binding.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/binding.yml) + - [operator.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/operator.yml) + - [computejob-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/computejob-crd.yml) + - [workflow-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/workflow-crd.yml) ## Customize your Operator Service deployment The following resources needs attention: - * postgres-configmap.yaml - contains secrets for the postgresql deployment - * deployment.yaml - ALGO_POD_TIMEOUT = Allowed time for a algorithm to run. If it exceeded this value (in minutes), it's going to get killed + - postgres-configmap.yaml - contains secrets for the postgresql deployment + - deployment.yaml - ALGO_POD_TIMEOUT = Allowed time for a algorithm to run. If it exceeded this value (in minutes), it's going to get killed ## Customize your Operator Engine deployment The following resource needs attention: -* operator.yaml - contains configuration variabiles - * ACCOUNT_JSON , ACCOUNT_PASSWORD = Defines the account that is going to be used when publishing results back to OceanProtocol - * AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID , AWS_REGION = S3 credidentials for the logs and output buckets - * AWS_BUCKET_OUTPUT = Bucket that will hold the output data (algorithm logs & algorithm output) - * AWS_BUCKET_ADMINLOGS = Bucket that will hold the admin logs (logs from pod-configure & pod-publish) - * STORAGE_CLASS = Storage class to use (see next section) +- operator.yaml - contains configuration variabiles + - ACCOUNT_JSON , ACCOUNT_PASSWORD = Defines the account that is going to be used when publishing results back to OceanProtocol + - AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID , AWS_REGION = S3 credidentials for the logs and output buckets + - AWS_BUCKET_OUTPUT = Bucket that will hold the output data (algorithm logs & algorithm output) + - AWS_BUCKET_ADMINLOGS = Bucket that will hold the admin logs (logs from pod-configure & pod-publish) + - STORAGE_CLASS = Storage class to use (see next section) ## Storage class @@ -51,11 +51,11 @@ For minikube, you can use 'standard' class. For AWS , please make sure that your class allocates volumes in the same region and zone in which you are running your pods. We created our own 'standard' class in AWS: -``` - kubectl get storageclass standard -o yaml -``` -``` +```bash + kubectl get storageclass standard -o yaml + + allowedTopologies: - matchLabelExpressions: - key: failure-domain.beta.kubernetes.io/zone @@ -73,14 +73,14 @@ We created our own 'standard' class in AWS: ## Create namespaces -``` +```bash kubectl create ns ocean-operator kubectl create ns ocean-compute ``` ## Deploy Operator-Service -``` +```bash kubectl config set-context --current --namespace ocean-operator kubectl create -f /ocean/operator-service/postgres-configmap.yaml kubectl create -f /ocean/operator-service/postgres-storage.yaml @@ -93,7 +93,7 @@ kubectl apply -f /ocean/operator-service/service_account.yaml ## Deploy Operator-Engine -``` +```bash kubectl config set-context --current --namespace ocean-compute kubectl apply -f /ocean/operator-engine/sa.yml kubectl apply -f /ocean/operator-engine/binding.yml @@ -105,13 +105,13 @@ kubectl create -f /ocean/operator-service/postgres-configmap.yaml ## Expose Operator - Service -``` +```bash kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 ``` Run a port forward or create your ingress service (not covered here): -``` +```bash kubectl -n ocean-operator port-forward svc/operator-api 8050 ``` @@ -119,7 +119,7 @@ kubectl -n ocean-operator port-forward svc/operator-api 8050 If your cluster is running on example.com: -``` +```bash curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: application/json" ``` @@ -127,8 +127,7 @@ curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: ap Update Brizo by adding or updating the OPERATOR_SERVICE_URL env (in /ocean/barge/compose-files/brizo.yaml) -``` OPERATOR_SERVICE_URL: http://example.com:8050/ -``` + ## Restart Barge with updated Brizo configuration From c28956207f5a002fee1677700c3213fa84c80366 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Sun, 24 May 2020 02:09:14 -0700 Subject: [PATCH 03/28] fix lint --- content/setup/ctd.md | 72 +++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 8fa510fb..ecfabc7b 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -52,8 +52,8 @@ For AWS , please make sure that your class allocates volumes in the same region We created our own 'standard' class in AWS: -```bash - kubectl get storageclass standard -o yaml + +kubectl get storageclass standard -o yaml allowedTopologies: @@ -69,59 +69,61 @@ We created our own 'standard' class in AWS: provisioner: kubernetes.io/aws-ebs reclaimPolicy: Delete volumeBindingMode: Immediate -``` + + ## Create namespaces -```bash -kubectl create ns ocean-operator -kubectl create ns ocean-compute -``` + kubectl create ns ocean-operator + kubectl create ns ocean-compute + + ## Deploy Operator-Service -```bash -kubectl config set-context --current --namespace ocean-operator -kubectl create -f /ocean/operator-service/postgres-configmap.yaml -kubectl create -f /ocean/operator-service/postgres-storage.yaml -kubectl create -f /ocean/operator-service/postgres-deployment.yaml -kubectl create -f /ocean/operator-service/postgresql-service.yaml -kubectl apply -f /ocean/operator-service/deployment.yaml -kubectl apply -f /ocean/operator-service/role_binding.yaml -kubectl apply -f /ocean/operator-service/service_account.yaml -``` + + kubectl config set-context --current --namespace ocean-operator + kubectl create -f /ocean/operator-service/postgres-configmap.yaml + kubectl create -f /ocean/operator-service/postgres-storage.yaml + kubectl create -f /ocean/operator-service/postgres-deployment.yaml + kubectl create -f /ocean/operator-service/postgresql-service.yaml + kubectl apply -f /ocean/operator-service/deployment.yaml + kubectl apply -f /ocean/operator-service/role_binding.yaml + kubectl apply -f /ocean/operator-service/service_account.yaml + + ## Deploy Operator-Engine -```bash -kubectl config set-context --current --namespace ocean-compute -kubectl apply -f /ocean/operator-engine/sa.yml -kubectl apply -f /ocean/operator-engine/binding.yml -kubectl apply -f /ocean/operator-engine/operator.yml -kubectl apply -f /ocean/operator-engine/computejob-crd.yaml -kubectl apply -f /ocean/operator-engine/workflow-crd.yaml -kubectl create -f /ocean/operator-service/postgres-configmap.yaml -``` + + kubectl config set-context --current --namespace ocean-compute + kubectl apply -f /ocean/operator-engine/sa.yml + kubectl apply -f /ocean/operator-engine/binding.yml + kubectl apply -f /ocean/operator-engine/operator.yml + kubectl apply -f /ocean/operator-engine/computejob-crd.yaml + kubectl apply -f /ocean/operator-engine/workflow-crd.yaml + kubectl create -f /ocean/operator-service/postgres-configmap.yaml + + ## Expose Operator - Service -```bash -kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 -``` + + kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 + Run a port forward or create your ingress service (not covered here): -```bash -kubectl -n ocean-operator port-forward svc/operator-api 8050 -``` + + kubectl -n ocean-operator port-forward svc/operator-api 8050 + ## Initialize database If your cluster is running on example.com: -```bash -curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: application/json" -``` + curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: application/json" + ## Update Brizo From 65dc844617e7e5a2ec8b6dbce5b7a923e8de0815 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 27 May 2020 22:34:08 -0700 Subject: [PATCH 04/28] add minikube standard storage class --- content/setup/ctd.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index ecfabc7b..cde836e5 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -70,7 +70,14 @@ kubectl get storageclass standard -o yaml reclaimPolicy: Delete volumeBindingMode: Immediate +Or we can use this for minikube: + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: standard + provisioner: docker.io/hostpath + reclaimPolicy: Retain ## Create namespaces From dce872f9742a69bc40db67b191a75719efd10725 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 08:20:53 +0300 Subject: [PATCH 05/28] fix links --- content/setup/ctd.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index cde836e5..2317bfa0 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -15,7 +15,7 @@ At the time of writing, we need the following: - [postgres-configmap.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-configmap.yaml) - [postgres-storage.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-storage.yaml) - [postgres-deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-deployment.yaml) - - [postgres-service.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-service.yaml) + - [postgres-service.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgresql-service.yaml) - [deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/deployment.yaml) - [role_binding.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/role_binding.yaml) - [service_account.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/service_account.yaml) @@ -23,8 +23,8 @@ At the time of writing, we need the following: - [sa.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/sa.yml) - [binding.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/binding.yml) - [operator.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/operator.yml) - - [computejob-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/computejob-crd.yml) - - [workflow-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/workflow-crd.yml) + - [computejob-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/computejob-crd.yaml) + - [workflow-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/workflow-crd.yaml) ## Customize your Operator Service deployment From acc64014ac4b002989029c56fe216136225b8ee7 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 08:37:44 +0300 Subject: [PATCH 06/28] fix raw links --- content/setup/ctd.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 2317bfa0..b8f11416 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -12,19 +12,19 @@ At the time of writing, we need the following: - a working Kubernetes cluster (minikube is a good start) - a working kubectl connected to the k8 cluster - one folder (/ocean/operator-service/), in which we will download the following: - - [postgres-configmap.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-configmap.yaml) - - [postgres-storage.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-storage.yaml) - - [postgres-deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgres-deployment.yaml) - - [postgres-service.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/postgresql-service.yaml) - - [deployment.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/deployment.yaml) - - [role_binding.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/role_binding.yaml) - - [service_account.yaml](https://github.com/oceanprotocol/operator-service/blob/develop/deploy_on_k8s/service_account.yaml) + - [postgres-configmap.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgres-configmap.yaml) + - [postgres-storage.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgres-storage.yaml) + - [postgres-deployment.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgres-deployment.yaml) + - [postgres-service.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgresql-service.yaml) + - [deployment.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/deployment.yaml) + - [role_binding.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/role_binding.yaml) + - [service_account.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/service_account.yaml) - one folder (/ocean/operator-engine/), in which we will download the following: - - [sa.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/sa.yml) - - [binding.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/binding.yml) - - [operator.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/operator.yml) - - [computejob-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/computejob-crd.yaml) - - [workflow-crd.yaml](https://github.com/oceanprotocol/operator-engine/blob/develop/k8s_install/workflow-crd.yaml) + - [sa.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-engine/develop/k8s_install/sa.yml) + - [binding.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-engine/develop/k8s_install/binding.yml) + - [operator.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-engine/develop/k8s_install/operator.yml) + - [computejob-crd.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-engine/develop/k8s_install/computejob-crd.yaml) + - [workflow-crd.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-engine/develop/k8s_install/workflow-crd.yaml) ## Customize your Operator Service deployment From 6379709c00da6d158f9e75db57e5037fef74f27d Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 09:46:39 +0300 Subject: [PATCH 07/28] add storage-clases docs --- content/setup/ctd.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index b8f11416..f5e263a8 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -79,6 +79,9 @@ Or we can use this for minikube: provisioner: docker.io/hostpath reclaimPolicy: Retain + + For more information, please visit https://kubernetes.io/docs/concepts/storage/storage-classes/ + ## Create namespaces kubectl create ns ocean-operator From aba3ebe5f8d3debf5fe1ffced276efbc31663097 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:18:24 +0300 Subject: [PATCH 08/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index f5e263a8..6a1c25ef 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -4,7 +4,7 @@ description: Set Up a Compute-to-Data enviroment. --- -## Requirments +## Requirements At the time of writing, we need the following: From 6a2d077ab5f6841af1a5c0326c026cf1dde7dfda Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:18:36 +0300 Subject: [PATCH 09/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 6a1c25ef..e1e93f22 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -28,7 +28,7 @@ At the time of writing, we need the following: ## Customize your Operator Service deployment -The following resources needs attention: +The following resources need attention: - postgres-configmap.yaml - contains secrets for the postgresql deployment - deployment.yaml - ALGO_POD_TIMEOUT = Allowed time for a algorithm to run. If it exceeded this value (in minutes), it's going to get killed From 672dfcf5a6dccb7368cb245d9b8f996c13072578 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:18:47 +0300 Subject: [PATCH 10/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index e1e93f22..fc1297c6 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -35,7 +35,7 @@ The following resources need attention: ## Customize your Operator Engine deployment -The following resource needs attention: +The following resources need attention: - operator.yaml - contains configuration variabiles - ACCOUNT_JSON , ACCOUNT_PASSWORD = Defines the account that is going to be used when publishing results back to OceanProtocol From cbe3feb7ea6c6fc19bc3f750e37468687a546213 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:19:03 +0300 Subject: [PATCH 11/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index fc1297c6..11b04a17 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -80,7 +80,7 @@ Or we can use this for minikube: reclaimPolicy: Retain - For more information, please visit https://kubernetes.io/docs/concepts/storage/storage-classes/ +For more information, please visit https://kubernetes.io/docs/concepts/storage/storage-classes/ ## Create namespaces From 38fcaf9bab78b89c56151e0901c33aa2e5045dd1 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:19:15 +0300 Subject: [PATCH 12/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 11b04a17..941ed3a5 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -137,7 +137,7 @@ If your cluster is running on example.com: ## Update Brizo -Update Brizo by adding or updating the OPERATOR_SERVICE_URL env (in /ocean/barge/compose-files/brizo.yaml) +Update Brizo by adding or updating the `OPERATOR_SERVICE_URL` env in `/ocean/barge/compose-files/brizo.yaml` OPERATOR_SERVICE_URL: http://example.com:8050/ From 1db59d5b887e9e4ca3cf875b7d748136dd7b4e29 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:19:30 +0300 Subject: [PATCH 13/28] Update data/sidebars/setup.yml Co-authored-by: Matthias Kretschmann --- data/sidebars/setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/sidebars/setup.yml b/data/sidebars/setup.yml index 5d791811..415ab84c 100644 --- a/data/sidebars/setup.yml +++ b/data/sidebars/setup.yml @@ -17,7 +17,7 @@ items: - title: Run a Keeper link: /setup/keeper/ -- group: Compute-to-Data Enviroment +- group: Compute-to-Data items: - title: Run a Compute-to-Data Enviroment link: /setup/ctd/ From 3f58af2e853474b78eb47ba359bb1ca210273995 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:19:40 +0300 Subject: [PATCH 14/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 941ed3a5..6e0c1b68 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -1,5 +1,5 @@ --- -title: Set Up a Compute-to-Data enviroment +title: Set Up a Compute-to-Data Environment description: Set Up a Compute-to-Data enviroment. --- From 268d519d9c05c06c9dd64e1b920fd615eb991a01 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:19:49 +0300 Subject: [PATCH 15/28] Update data/sidebars/setup.yml Co-authored-by: Matthias Kretschmann --- data/sidebars/setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/sidebars/setup.yml b/data/sidebars/setup.yml index 415ab84c..b66ca6ca 100644 --- a/data/sidebars/setup.yml +++ b/data/sidebars/setup.yml @@ -19,5 +19,5 @@ link: /setup/keeper/ - group: Compute-to-Data items: - - title: Run a Compute-to-Data Enviroment + - title: Run a Compute-to-Data Environment link: /setup/ctd/ From 88c2833099fe427f598fc5cbdf14a68404b195f4 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:19:58 +0300 Subject: [PATCH 16/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 6e0c1b68..3b61e476 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -139,6 +139,7 @@ If your cluster is running on example.com: Update Brizo by adding or updating the `OPERATOR_SERVICE_URL` env in `/ocean/barge/compose-files/brizo.yaml` +```yaml OPERATOR_SERVICE_URL: http://example.com:8050/ From 701d2357085e61fda634664d455b1f0fa83e3f7b Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:20:41 +0300 Subject: [PATCH 17/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 3b61e476..c0cf8057 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -89,7 +89,7 @@ For more information, please visit https://kubernetes.io/docs/concepts/storage/s -## Deploy Operator-Service +## Deploy Operator Service kubectl config set-context --current --namespace ocean-operator From b5a748005d307e5ad469d48d6af51f26e981acb5 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:21:12 +0300 Subject: [PATCH 18/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index c0cf8057..30fa37db 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -9,7 +9,7 @@ description: Set Up a Compute-to-Data enviroment. At the time of writing, we need the following: - working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ -- a working Kubernetes cluster (minikube is a good start) +- a working Kubernetes (K8s) cluster (Minikube is a good start) - a working kubectl connected to the k8 cluster - one folder (/ocean/operator-service/), in which we will download the following: - [postgres-configmap.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgres-configmap.yaml) From cf389e18638aa231d03ada511020812fe9f7c9c6 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:21:24 +0300 Subject: [PATCH 19/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 30fa37db..13acab13 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -10,7 +10,7 @@ At the time of writing, we need the following: - working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ - a working Kubernetes (K8s) cluster (Minikube is a good start) -- a working kubectl connected to the k8 cluster +- a working `kubectl` connected to the K8s cluster - one folder (/ocean/operator-service/), in which we will download the following: - [postgres-configmap.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgres-configmap.yaml) - [postgres-storage.yaml](https://raw.githubusercontent.com/oceanprotocol/operator-service/develop/deploy_on_k8s/postgres-storage.yaml) From 14f56e872ecbfca55cfb9619678d94d05dd073e3 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:21:39 +0300 Subject: [PATCH 20/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 13acab13..f1bf24be 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -103,7 +103,7 @@ For more information, please visit https://kubernetes.io/docs/concepts/storage/s -## Deploy Operator-Engine +## Deploy Operator Engine kubectl config set-context --current --namespace ocean-compute From d8a2bcbbb9ca2a44383afed670b33455140b9f40 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:21:52 +0300 Subject: [PATCH 21/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index f1bf24be..836eb465 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -7,6 +7,7 @@ description: Set Up a Compute-to-Data enviroment. ## Requirements At the time of writing, we need the following: +First, create a folder with the following structure: - working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ - a working Kubernetes (K8s) cluster (Minikube is a good start) From c4aa957c713a83873e184392f54df8b38636d470 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:22:00 +0300 Subject: [PATCH 22/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 836eb465..c6bc7bfc 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -117,7 +117,7 @@ For more information, please visit https://kubernetes.io/docs/concepts/storage/s -## Expose Operator - Service +## Expose Operator Service kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 From df65b0988fde4ef49c6e502d59f99fcfd06573b0 Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 3 Jun 2020 13:22:41 +0300 Subject: [PATCH 23/28] Update content/setup/ctd.md Co-authored-by: Matthias Kretschmann --- content/setup/ctd.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index c6bc7bfc..eedde603 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -54,6 +54,7 @@ For AWS , please make sure that your class allocates volumes in the same region We created our own 'standard' class in AWS: +```bash kubectl get storageclass standard -o yaml From ed53707e21c94162de3b2e045aea622b3ee5e614 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 3 Jun 2020 03:29:37 -0700 Subject: [PATCH 24/28] fixes --- content/setup/ctd.md | 120 ++++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 52 deletions(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index eedde603..4e67b781 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -29,15 +29,24 @@ First, create a folder with the following structure: ## Customize your Operator Service deployment -The following resources need attention: - - postgres-configmap.yaml - contains secrets for the postgresql deployment - - deployment.yaml - ALGO_POD_TIMEOUT = Allowed time for a algorithm to run. If it exceeded this value (in minutes), it's going to get killed +The following resources need attention: +| Resource | Variable | Description | +| ------------------------- | ------------------ | ------------------------------------------------------------------------------------------------------ | +| `postgres-configmap.yaml` | | Contains secrets for the PostgreSQL deployment. | +| `deployment.yaml` | `ALGO_POD_TIMEOUT` | Allowed time for a algorithm to run. If it exceeded this value (in minutes), it's going to get killed. | ## Customize your Operator Engine deployment The following resources need attention: +| Resource | Variable | Description | +| --------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------- | +| `operator.yaml` | `ACCOUNT_JSON`, `ACCOUNT_PASSWORD` | Defines the account that is going to be used when publishing results back to OceanProtocol. | +| | `AWS_ACCESS_KEY_ID`, `AWS_ACCESS_KEY_ID`, `AWS_REGION` | S3 credentials for the logs and output buckets. | +| | `AWS_BUCKET_OUTPUT` | Bucket that will hold the output data (algorithm logs & algorithm output). | +| | `AWS_BUCKET_ADMINLOGS` | Bucket that will hold the admin logs (logs from pod-configure & pod-publish). | +| | `STORAGE_CLASS` | Storage class to use (see next section). | - operator.yaml - contains configuration variabiles - ACCOUNT_JSON , ACCOUNT_PASSWORD = Defines the account that is going to be used when publishing results back to OceanProtocol - AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID , AWS_REGION = S3 credidentials for the logs and output buckets @@ -56,85 +65,92 @@ We created our own 'standard' class in AWS: ```bash kubectl get storageclass standard -o yaml +``` - - allowedTopologies: - - matchLabelExpressions: - - key: failure-domain.beta.kubernetes.io/zone +```yaml +allowedTopologies: +- matchLabelExpressions: + - key: failure-domain.beta.kubernetes.io/zone values: - us-east-1a - apiVersion: storage.k8s.io/v1 - kind: StorageClass - parameters: - fsType: ext4 - type: gp2 - provisioner: kubernetes.io/aws-ebs - reclaimPolicy: Delete - volumeBindingMode: Immediate +apiVersion: storage.k8s.io/v1 +kind: StorageClass +parameters: + fsType: ext4 + type: gp2 +provisioner: kubernetes.io/aws-ebs +reclaimPolicy: Delete +volumeBindingMode: Immediate +``` Or we can use this for minikube: - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: standard - provisioner: docker.io/hostpath - reclaimPolicy: Retain - +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: standard +provisioner: docker.io/hostpath +reclaimPolicy: Retain +``` For more information, please visit https://kubernetes.io/docs/concepts/storage/storage-classes/ ## Create namespaces - kubectl create ns ocean-operator - kubectl create ns ocean-compute - +```bash +kubectl create ns ocean-operator +kubectl create ns ocean-compute +``` ## Deploy Operator Service - - kubectl config set-context --current --namespace ocean-operator - kubectl create -f /ocean/operator-service/postgres-configmap.yaml - kubectl create -f /ocean/operator-service/postgres-storage.yaml - kubectl create -f /ocean/operator-service/postgres-deployment.yaml - kubectl create -f /ocean/operator-service/postgresql-service.yaml - kubectl apply -f /ocean/operator-service/deployment.yaml - kubectl apply -f /ocean/operator-service/role_binding.yaml - kubectl apply -f /ocean/operator-service/service_account.yaml - +```bash +kubectl config set-context --current --namespace ocean-operator +kubectl create -f /ocean/operator-service/postgres-configmap.yaml +kubectl create -f /ocean/operator-service/postgres-storage.yaml +kubectl create -f /ocean/operator-service/postgres-deployment.yaml +kubectl create -f /ocean/operator-service/postgresql-service.yaml +kubectl apply -f /ocean/operator-service/deployment.yaml +kubectl apply -f /ocean/operator-service/role_binding.yaml +kubectl apply -f /ocean/operator-service/service_account.yaml +``` ## Deploy Operator Engine - - kubectl config set-context --current --namespace ocean-compute - kubectl apply -f /ocean/operator-engine/sa.yml - kubectl apply -f /ocean/operator-engine/binding.yml - kubectl apply -f /ocean/operator-engine/operator.yml - kubectl apply -f /ocean/operator-engine/computejob-crd.yaml - kubectl apply -f /ocean/operator-engine/workflow-crd.yaml - kubectl create -f /ocean/operator-service/postgres-configmap.yaml - +```bash +kubectl config set-context --current --namespace ocean-compute +kubectl apply -f /ocean/operator-engine/sa.yml +kubectl apply -f /ocean/operator-engine/binding.yml +kubectl apply -f /ocean/operator-engine/operator.yml +kubectl apply -f /ocean/operator-engine/computejob-crd.yaml +kubectl apply -f /ocean/operator-engine/workflow-crd.yaml +kubectl create -f /ocean/operator-service/postgres-configmap.yaml +``` ## Expose Operator Service - - kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 +```bash +kubectl expose deployment operator-api --namespace=ocean-operator --port=8050 +``` Run a port forward or create your ingress service (not covered here): - - kubectl -n ocean-operator port-forward svc/operator-api 8050 - +```bash +kubectl -n ocean-operator port-forward svc/operator-api 8050 +``` ## Initialize database If your cluster is running on example.com: - curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: application/json" +```bash +curl -X POST "http://example.com:8050/api/v1/operator/pgsqlinit" -H "accept: application/json" +``` ## Update Brizo @@ -143,6 +159,6 @@ Update Brizo by adding or updating the `OPERATOR_SERVICE_URL` env in `/ocean/ba ```yaml OPERATOR_SERVICE_URL: http://example.com:8050/ +``` - -## Restart Barge with updated Brizo configuration +Restart Barge with updated Brizo configuration From 263a9077d2ac8a665ffce318646499d07681f3f6 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 3 Jun 2020 03:32:04 -0700 Subject: [PATCH 25/28] fixes --- content/setup/ctd.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/content/setup/ctd.md b/content/setup/ctd.md index 4e67b781..66ab480f 100644 --- a/content/setup/ctd.md +++ b/content/setup/ctd.md @@ -47,12 +47,6 @@ The following resources need attention: | | `AWS_BUCKET_OUTPUT` | Bucket that will hold the output data (algorithm logs & algorithm output). | | | `AWS_BUCKET_ADMINLOGS` | Bucket that will hold the admin logs (logs from pod-configure & pod-publish). | | | `STORAGE_CLASS` | Storage class to use (see next section). | -- operator.yaml - contains configuration variabiles - - ACCOUNT_JSON , ACCOUNT_PASSWORD = Defines the account that is going to be used when publishing results back to OceanProtocol - - AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID , AWS_REGION = S3 credidentials for the logs and output buckets - - AWS_BUCKET_OUTPUT = Bucket that will hold the output data (algorithm logs & algorithm output) - - AWS_BUCKET_ADMINLOGS = Bucket that will hold the admin logs (logs from pod-configure & pod-publish) - - STORAGE_CLASS = Storage class to use (see next section) ## Storage class From a6fd389d89c08ad59ac05c418239fcb7c0604836 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 3 Jun 2020 03:40:32 -0700 Subject: [PATCH 26/28] move ctd to compute-to-data --- content/setup/{ctd.md => compute-to-data.md} | 0 data/sidebars/setup.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename content/setup/{ctd.md => compute-to-data.md} (100%) diff --git a/content/setup/ctd.md b/content/setup/compute-to-data.md similarity index 100% rename from content/setup/ctd.md rename to content/setup/compute-to-data.md diff --git a/data/sidebars/setup.yml b/data/sidebars/setup.yml index b66ca6ca..90cdd018 100644 --- a/data/sidebars/setup.yml +++ b/data/sidebars/setup.yml @@ -20,4 +20,4 @@ - group: Compute-to-Data items: - title: Run a Compute-to-Data Environment - link: /setup/ctd/ + link: /setup/compute-to-data/ From 2d942591f4e0093521ababfb57b53fdbefdddd3f Mon Sep 17 00:00:00 2001 From: Matthias Kretschmann Date: Wed, 3 Jun 2020 12:44:01 +0200 Subject: [PATCH 27/28] add actual folder structure --- content/setup/compute-to-data.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/content/setup/compute-to-data.md b/content/setup/compute-to-data.md index 66ab480f..4a4db489 100644 --- a/content/setup/compute-to-data.md +++ b/content/setup/compute-to-data.md @@ -1,14 +1,23 @@ --- title: Set Up a Compute-to-Data Environment -description: Set Up a Compute-to-Data enviroment. +description: Set Up a Compute-to-Data environment. --- ## Requirements -At the time of writing, we need the following: + First, create a folder with the following structure: +``` +ocean/ + barge/ + operator-service/ + operator-engine/ +``` + +Then you need the following parts: + - working [Barge](https://github.com/oceanprotocol/barge). For this setup, we will asume the Barge is installed in /ocean/barge/ - a working Kubernetes (K8s) cluster (Minikube is a good start) - a working `kubectl` connected to the K8s cluster From 41ab9a3c6fff4feaedccd4eeadbe38e835a2508d Mon Sep 17 00:00:00 2001 From: Matthias Kretschmann Date: Wed, 3 Jun 2020 12:50:33 +0200 Subject: [PATCH 28/28] markdown lint fix --- content/setup/compute-to-data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/setup/compute-to-data.md b/content/setup/compute-to-data.md index 4a4db489..96f697ea 100644 --- a/content/setup/compute-to-data.md +++ b/content/setup/compute-to-data.md @@ -9,7 +9,7 @@ description: Set Up a Compute-to-Data environment. First, create a folder with the following structure: -``` +```text ocean/ barge/ operator-service/