Fix glue namespace handling (#63)

* Changed: namespace handling for glue

* Changed: set glue as default agent system

* Changed: bump up version

.github/helm-docs.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 CHART_DIRS="$(git diff --find-renames --name-only "$(git rev-parse --abbrev-ref HEAD)" remotes/origin/main -- 'charts' | grep '[cC]hart.yaml' | sed -e 's#/[Cc]hart.yaml##g')"
-HELM_DOCS_VERSION="1.5.0"
+HELM_DOCS_VERSION="1.8.1"
 
 curl --silent --show-error --fail --location --output /tmp/helm-docs.tar.gz https://github.com/norwoodj/helm-docs/releases/download/v"${HELM_DOCS_VERSION}"/helm-docs_"${HELM_DOCS_VERSION}"_Linux_x86_64.tar.gz
 tar -xf /tmp/helm-docs.tar.gz helm-docs

charts/clearml/Chart.lock

@@ -7,6 +7,6 @@ dependencies:
   version: 10.3.4
 - name: elasticsearch
   repository: https://helm.elastic.co
-  version: 7.10.1
-digest: sha256:aefd3992b2ab085161e4cca35c6f73dd33f8d19272a9405b5ee4e8c2a0e79bba
-generated: "2021-01-05T14:26:33.629164+01:00"
+  version: 7.16.2
+digest: sha256:ac733cb02d50e8398c1d2832988333896f1c7b765c19a0f1eea5e9b24bdb8207
+generated: "2022-01-05T07:52:34.913745+01:00"

charts/clearml/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: clearml
 description: MLOps platform
 type: application
-version: "3.2.2"
-appVersion: "1.1.1"
+version: "3.10.0"
+appVersion: "1.4.0"
 home: https://clear.ml
 icon: https://raw.githubusercontent.com/allegroai/clearml/master/docs/clearml-logo.svg
 sources:
@@ -18,14 +18,14 @@ keywords:
   - mlops
 dependencies:
   - name: redis
-    version: "~10.9.0"
+    version: "10.9.0"
     repository: "https://charts.bitnami.com/bitnami"
     condition: redis.enabled
   - name: mongodb
-    version: "~10.3.2"
+    version: "10.3.4"
     repository: "https://charts.bitnami.com/bitnami"
     condition: mongodb.enabled
   - name: elasticsearch
-    version: "~7.10.1"
+    version: "7.16.2"
     repository: "https://helm.elastic.co"
     condition: elasticsearch.enabled

charts/clearml/README.md

@@ -1,6 +1,6 @@
 # ClearML Ecosystem for Kubernetes
 
-![Version: 3.2.2](https://img.shields.io/badge/Version-3.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.1](https://img.shields.io/badge/AppVersion-1.1.1-informational?style=flat-square)
+![Version: 3.10.0](https://img.shields.io/badge/Version-3.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.4.0](https://img.shields.io/badge/AppVersion-1.4.0-informational?style=flat-square)
 
 MLOps platform
 
@@ -10,7 +10,7 @@ MLOps platform
 
 | Name | Email | Url |
 | ---- | ------ | --- |
-| valeriano-manassero |  | https://github.com/valeriano-manassero |
+| valeriano-manassero |  | <https://github.com/valeriano-manassero> |
 
 ## Introduction
 
@@ -31,9 +31,7 @@ For development/evaluation it's possible to use [kind](https://kind.sigs.k8s.io)
 After installation, following commands will create a complete ClearML insatllation:
 
 ```
-mkdir -pm 777 /tmp/clearml-kind
-
-cat <<EOF > /tmp/clearml-kind.yaml
+cat <<EOF | kind create cluster --config=-                                                                  ─╯
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
@@ -62,8 +60,6 @@ nodes:
     containerPath: /var/local-path-provisioner
 EOF
 
-kind create cluster --config /tmp/clearml-kind.yaml
-
 helm install clearml allegroai/clearml
 ```
 
@@ -89,6 +85,24 @@ This will create 3 ingress rules:
 
 Just pointing the domain records to the IP where ingress controller is responding will complete the deployment process.
 
+## Upgrades/ Values upgrades
+
+Updating to latest version of this chart can be done in two steps:
+
+```
+helm repo update
+helm upgrade clearml allegroai/clearml
+```
+
+Changing values on existing installation can be done with:
+
+```
+helm upgrade clearml allegroai/clearml --version <CURRENT CHART VERSION> -f custom_values.yaml
+```
+
+Please note: updating values only should always be done setting explicit chart version to avoid a possible chart update.
+Keeping separate updates procedures between version and values can be a good practice to seprate potential concerns.
+
 ## Additional Configuration for ClearML Server
 
 You can also configure the **clearml-server** for:
@@ -107,9 +121,9 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://charts.bitnami.com/bitnami | mongodb | ~10.3.2 |
-| https://charts.bitnami.com/bitnami | redis | ~10.9.0 |
-| https://helm.elastic.co | elasticsearch | ~7.10.1 |
+| https://charts.bitnami.com/bitnami | mongodb | 10.3.4 |
+| https://charts.bitnami.com/bitnami | redis | 10.9.0 |
+| https://helm.elastic.co | elasticsearch | 7.16.2 |
 
 ## Values
 
@@ -127,7 +141,7 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | agentGroups.agent-group-cpu.clearmlGitPassword | string | `nil` |  |
 | agentGroups.agent-group-cpu.clearmlGitUser | string | `nil` |  |
 | agentGroups.agent-group-cpu.clearmlSecretKey | string | `nil` |  |
-| agentGroups.agent-group-cpu.enabled | bool | `true` |  |
+| agentGroups.agent-group-cpu.enabled | bool | `false` |  |
 | agentGroups.agent-group-cpu.image.pullPolicy | string | `"IfNotPresent"` |  |
 | agentGroups.agent-group-cpu.image.repository | string | `"ubuntu"` |  |
 | agentGroups.agent-group-cpu.image.tag | string | `"18.04"` |  |
@@ -151,7 +165,7 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | agentGroups.agent-group-gpu.clearmlGitPassword | string | `nil` |  |
 | agentGroups.agent-group-gpu.clearmlGitUser | string | `nil` |  |
 | agentGroups.agent-group-gpu.clearmlSecretKey | string | `nil` |  |
-| agentGroups.agent-group-gpu.enabled | bool | `true` |  |
+| agentGroups.agent-group-gpu.enabled | bool | `false` |  |
 | agentGroups.agent-group-gpu.image.pullPolicy | string | `"IfNotPresent"` |  |
 | agentGroups.agent-group-gpu.image.repository | string | `"nvidia/cuda"` |  |
 | agentGroups.agent-group-gpu.image.tag | string | `"11.0-base-ubuntu18.04"` |  |
@@ -163,6 +177,19 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | agentGroups.agent-group-gpu.replicaCount | int | `0` |  |
 | agentGroups.agent-group-gpu.tolerations | list | `[]` |  |
 | agentGroups.agent-group-gpu.updateStrategy | string | `"Recreate"` |  |
+| agentk8sglue.defaultDockerImage | string | `"nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04"` |  |
+| agentk8sglue.enabled | bool | `true` |  |
+| agentk8sglue.id | string | `"k8s-agent"` |  |
+| agentk8sglue.image.repository | string | `"allegroai/clearml-agent-k8s"` |  |
+| agentk8sglue.image.tag | string | `"latest"` |  |
+| agentk8sglue.maxPods | int | `10` |  |
+| agentk8sglue.podTemplate.env | list | `[]` |  |
+| agentk8sglue.podTemplate.nodeSelector | object | `{}` |  |
+| agentk8sglue.podTemplate.resources | object | `{}` |  |
+| agentk8sglue.podTemplate.tolerations | list | `[]` |  |
+| agentk8sglue.podTemplate.volumes | list | `[]` |  |
+| agentk8sglue.queue | string | `"default"` |  |
+| agentk8sglue.serviceAccountName | string | `"default"` |  |
 | agentservices.affinity | object | `{}` |  |
 | agentservices.agentVersion | string | `""` |  |
 | agentservices.awsAccessKeyId | string | `nil` |  |
@@ -189,14 +216,14 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | agentservices.storage.data.class | string | `"standard"` |  |
 | agentservices.storage.data.size | string | `"50Gi"` |  |
 | agentservices.tolerations | list | `[]` |  |
-| apiserver.additionalConfigs | object | `{}` |  |
+| apiserver.additionalConfigs | object | `{}` | additional configurations that can be used by api server; check examples in values.yaml file |
 | apiserver.affinity | object | `{}` |  |
 | apiserver.authCookiesMaxAge | int | `864000` | Amount of seconds the authorization cookie will last in user browser |
 | apiserver.configDir | string | `"/opt/clearml/config"` |  |
 | apiserver.extraEnvs | list | `[]` |  |
 | apiserver.image.pullPolicy | string | `"IfNotPresent"` |  |
 | apiserver.image.repository | string | `"allegroai/clearml"` |  |
-| apiserver.image.tag | string | `"1.1.1"` |  |
+| apiserver.image.tag | string | `"1.4.0"` |  |
 | apiserver.livenessDelay | int | `60` |  |
 | apiserver.nodeSelector | object | `{}` |  |
 | apiserver.podAnnotations | object | `{}` |  |
@@ -256,7 +283,7 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | fileserver.extraEnvs | list | `[]` |  |
 | fileserver.image.pullPolicy | string | `"IfNotPresent"` |  |
 | fileserver.image.repository | string | `"allegroai/clearml"` |  |
-| fileserver.image.tag | string | `"1.1.1"` |  |
+| fileserver.image.tag | string | `"1.4.0"` |  |
 | fileserver.nodeSelector | object | `{}` |  |
 | fileserver.podAnnotations | object | `{}` |  |
 | fileserver.replicaCount | int | `1` |  |
@@ -268,12 +295,20 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | fileserver.storage.data.size | string | `"50Gi"` |  |
 | fileserver.tolerations | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
+| ingress.api.annotations | object | `{}` |  |
+| ingress.api.enabled | bool | `false` |  |
 | ingress.api.hostName | string | `"api.clearml.127-0-0-1.nip.io"` |  |
+| ingress.api.path | string | `"/"` |  |
 | ingress.api.tlsSecretName | string | `""` |  |
+| ingress.app.annotations | object | `{}` |  |
+| ingress.app.enabled | bool | `false` |  |
 | ingress.app.hostName | string | `"app.clearml.127-0-0-1.nip.io"` |  |
+| ingress.app.path | string | `"/"` |  |
 | ingress.app.tlsSecretName | string | `""` |  |
-| ingress.enabled | bool | `false` |  |
+| ingress.files.annotations | object | `{}` |  |
+| ingress.files.enabled | bool | `false` |  |
 | ingress.files.hostName | string | `"files.clearml.127-0-0-1.nip.io"` |  |
+| ingress.files.path | string | `"/"` |  |
 | ingress.files.tlsSecretName | string | `""` |  |
 | ingress.name | string | `"clearml-server-ingress"` |  |
 | mongodb.architecture | string | `"standalone"` |  |
@@ -302,11 +337,12 @@ For detailed instructions, see the [Optional Configuration](https://github.com/a
 | secret.credentials.tests.accessKey | string | `"ENP39EQM4SLACGD5FXB7"` | Set for tests_user_key field |
 | secret.credentials.tests.secretKey | string | `"lPcm0imbcBZ8mwgO7tpadutiS3gnJD05x9j7afwXPS35IKbpiQ"` | Set for tests_user_secret field |
 | secret.httpSession | string | `"9Tw20RbhJ1bLBiHEOWXvhplKGUbTgLzAtwFN2oLQvWwS0uRpD5"` | Set for http_session field |
+| webserver.additionalConfigs | object | `{}` |  |
 | webserver.affinity | object | `{}` |  |
 | webserver.extraEnvs | list | `[]` |  |
 | webserver.image.pullPolicy | string | `"IfNotPresent"` |  |
 | webserver.image.repository | string | `"allegroai/clearml"` |  |
-| webserver.image.tag | string | `"1.1.1"` |  |
+| webserver.image.tag | string | `"1.4.0"` |  |
 | webserver.nodeSelector | object | `{}` |  |
 | webserver.podAnnotations | object | `{}` |  |
 | webserver.replicaCount | int | `1` |  |

charts/clearml/README.md.gotmpl

@@ -28,9 +28,7 @@ For development/evaluation it's possible to use [kind](https://kind.sigs.k8s.io)
 After installation, following commands will create a complete ClearML insatllation:
 
 ```
-mkdir -pm 777 /tmp/clearml-kind
-
-cat <<EOF > /tmp/clearml-kind.yaml
+cat <<EOF | kind create cluster --config=-                                                                  ─╯
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
@@ -59,8 +57,6 @@ nodes:
     containerPath: /var/local-path-provisioner
 EOF
 
-kind create cluster --config /tmp/clearml-kind.yaml
-
 helm install clearml allegroai/clearml
 ```
 
@@ -86,6 +82,24 @@ This will create 3 ingress rules:
 
 Just pointing the domain records to the IP where ingress controller is responding will complete the deployment process.
 
+## Upgrades/ Values upgrades
+
+Updating to latest version of this chart can be done in two steps:
+
+```
+helm repo update
+helm upgrade clearml allegroai/clearml
+```
+
+Changing values on existing installation can be done with:
+
+```
+helm upgrade clearml allegroai/clearml --version <CURRENT CHART VERSION> -f custom_values.yaml
+```
+
+Please note: updating values only should always be done setting explicit chart version to avoid a possible chart update. 
+Keeping separate updates procedures between version and values can be a good practice to seprate potential concerns.
+
 ## Additional Configuration for ClearML Server
 
 You can also configure the **clearml-server** for:

charts/clearml/ci/README.md

@@ -0,0 +1,7 @@
+Place values files with different values in this directory to ensure these cases are tested by the CI as well.
+
+https://github.com/helm/chart-testing/blob/main/doc/ct_install.md
+
+```
+"Charts may have multiple custom values files matching the glob pattern '*-values.yaml' in a directory named 'ci' in the root of the chart's directory. The chart is installed and tested for each of these files. If no custom values file is present, the chart is installed and tested with defaults."
+```

charts/clearml/ci/default-values.yaml

@@ -0,0 +1 @@
+# empty so default values.yaml gets tested

charts/clearml/ci/k8sagent-values.yaml

@@ -0,0 +1,2 @@
+agentk8sglue:
+  enabled: true

charts/clearml/templates/_helpers.tpl

@@ -140,3 +140,17 @@ Create the name of the Files service to use
 {{- printf "%s%s%s%s" "http://" (include "clearml.fullname" .) "-fileserver:" (.Values.fileserver.service.port | toString) }}
 {{- end }}
 {{- end }}
+
+{{/*
+Return the proper Docker Image Registry Secret Names
+*/}}
+{{- define "clearml.imagePullSecrets" -}}
+{{- if .Values.global }}
+{{- if .Values.global.imagePullSecrets }}
+imagePullSecrets:
+{{- range .Values.global.imagePullSecrets }}
+  - name: {{ . }}
+{{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}

charts/clearml/templates/configmap-agentk8s-template.yaml

@@ -0,0 +1,57 @@
+{{- if .Values.agentk8sglue.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k8sagent-pod-template
+data:
+  template.yaml: |
+    apiVersion: v1
+    metadata:
+      namespace: {{ .Release.Namespace }}
+    spec:
+      serviceAccountName: {{ .Values.agentk8sglue.serviceAccountName }}
+      volumes:
+      {{- range .Values.agentk8sglue.podTemplate.volumes }}
+      - name: {{ .name }}
+        persistentVolumeClaim:
+          claimName: {{ .name }}
+      {{- end }}
+      containers:
+      - resources:
+          {{- toYaml .Values.agentk8sglue.podTemplate.resources | nindent 10 }}
+        ports:
+        - containerPort: 10022
+        volumeMounts:
+        {{- range .Values.agentk8sglue.podTemplate.volumes }}
+        - mountPath: {{ .path }}
+          name: {{ .name }}
+        {{- end }}
+        env:
+        - name: CLEARML_API_HOST
+          value: "http://{{ include "clearml.fullname" . }}-apiserver:{{ .Values.apiserver.service.port }}"
+        - name: CLEARML_WEB_HOST
+          value: "http://{{ include "clearml.fullname" . }}-webserver"
+        - name: CLEARML_FILES_HOST
+          value: "http://{{ include "clearml.fullname" . }}-fileserver:{{ .Values.fileserver.service.port }}"
+        - name: CLEARML_API_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: clearml-conf
+              key: apiserver_key
+        - name: CLEARML_API_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: clearml-conf
+              key: apiserver_secret
+        {{- if .Values.agentk8sglue.podTemplate.env }}
+        {{ toYaml .Values.agentk8sglue.podTemplate.env | nindent 8 }}
+        {{- end }}
+        {{- with .Values.agentk8sglue.podTemplate.nodeSelector}}
+        nodeSelector:
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.agentk8sglue.podTemplate.tolerations }}
+        tolerations:
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+{{- end }}

charts/clearml/templates/configmap-webserver.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.webserver.additionalConfigs -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ include "clearml.fullname" . }}-webserver-configmap"
+  labels:
+    {{- include "clearml.labels" . | nindent 4 }}
+data:
+  {{- range $key, $val := .Values.webserver.additionalConfigs }}
+  {{ $key }}: |
+    {{- $val | nindent 4 }}
+  {{- end }}
+{{- end -}}

charts/clearml/templates/deployment-agentk8s.yaml

@@ -0,0 +1,64 @@
+{{- if .Values.agentk8sglue.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ include "clearml.fullname" . }}-k8sagent"
+  labels:
+    app: k8sagent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: k8sagent
+  template:
+    metadata:
+      labels:
+        app: k8sagent
+    spec:
+      containers:
+      - name: k8s-glue-container
+        image: "{{ .Values.agentk8sglue.image.repository }}:{{ .Values.agentk8sglue.image.tag }}"
+        imagePullPolicy: Always
+        command: ["/bin/bash", "-c", "export PATH=$PATH:$HOME/bin; source /root/.bashrc && /root/entrypoint.sh"]
+        volumeMounts:
+          - name: k8sagent-pod-template
+            mountPath: /root/template
+        env:
+          - name: CLEARML_API_HOST
+            value: "http://{{ include "clearml.fullname" . }}-apiserver:{{ .Values.apiserver.service.port }}"
+          - name: CLEARML_WEB_HOST
+            value: "http://{{ include "clearml.fullname" . }}-webserver"
+          - name: CLEARML_FILES_HOST
+            value: "http://{{ include "clearml.fullname" . }}-fileserver:{{ .Values.fileserver.service.port }}"
+          - name: K8S_GLUE_MAX_PODS
+            value: "{{.Values.agentk8sglue.maxPods}}"
+          - name: K8S_GLUE_QUEUE
+            value: "{{.Values.agentk8sglue.queue}}"
+          - name: K8S_GLUE_EXTRA_ARGS
+            value: "--namespace {{ .Release.Namespace }} --template-yaml /root/template/template.yaml"
+          - name: K8S_DEFAULT_NAMESPACE
+            value: "{{ .Release.Namespace }}"
+          - name: CLEARML_API_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: clearml-conf
+                key: apiserver_key
+          - name: CLEARML_API_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: clearml-conf
+                key: apiserver_secret
+          - name: CLEARML_WORKER_ID
+            value: "{{.Values.agentk8sglue.id}}"
+          - name: CLEARML_AGENT_UPDATE_REPO
+            value: ""
+          - name: FORCE_CLEARML_AGENT_REPO
+            value: ""
+          - name: CLEARML_DOCKER_IMAGE
+            value: "{{.Values.agentk8sglue.defaultDockerImage}}"
+      volumes:
+        - name: k8sagent-pod-template
+          configMap: 
+            name: k8sagent-pod-template
+{{- end }}
+

charts/clearml/templates/deployment-apiserver.yaml

@@ -19,6 +19,7 @@ spec:
       labels:
         {{- include "clearml.selectorLabelsApiServer" . | nindent 8 }}
     spec:
+{{- include "clearml.imagePullSecrets" . | indent 6 }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.apiserver.image.repository }}:{{ .Values.apiserver.image.tag | default .Chart.AppVersion }}"

charts/clearml/templates/deployment-fileserver.yaml

@@ -22,6 +22,7 @@ spec:
         - name: fileserver-data
           persistentVolumeClaim:
             claimName: {{ include "clearml.fullname" . }}-fileserver-data
+{{- include "clearml.imagePullSecrets" . | indent 6 }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.fileserver.image.repository }}:{{ .Values.fileserver.image.tag | default .Chart.AppVersion }}"

charts/clearml/templates/deployment-webserver.yaml

@@ -18,6 +18,7 @@ spec:
       labels:
         {{- include "clearml.selectorLabelsWebServer" . | nindent 8 }}
     spec:
+{{- include "clearml.imagePullSecrets" . | indent 6 }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.webserver.image.repository }}:{{ .Values.webserver.image.tag | default .Chart.AppVersion }}"
@@ -38,6 +39,11 @@ spec:
                 - curl
                 - -X OPTIONS
                 - http://0.0.0.0:80/
+          {{- if .Values.webserver.additionalConfigs }}
+          volumeMounts:
+          - name: webserver-config
+            mountPath: /opt/clearml/config
+          {{- end }}
           env:
           - name: NGINX_APISERVER_ADDRESS
             value: "http://{{ include "clearml.fullname" . }}-apiserver:{{ .Values.apiserver.service.port }}"
@@ -50,6 +56,12 @@ spec:
             - webserver
           resources:
             {{- toYaml .Values.webserver.resources | nindent 12 }}
+      {{- if .Values.webserver.additionalConfigs }}
+      volumes:
+        - name: webserver-config
+          configMap:
+            name: "{{ include "clearml.fullname" . }}-webserver-configmap"
+      {{- end }}
       {{- with .Values.webserver.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -61,4 +73,4 @@ spec:
       {{- with .Values.webserver.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}

charts/clearml/templates/ingress-api.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enabled -}}
+{{- if .Values.ingress.api.enabled -}}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@@ -11,10 +11,13 @@ metadata:
   name: {{ include "clearml.fullname" . }}-api
   labels:
     {{- include "clearml.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
+  {{- $annotations := .Values.ingress.annotations }}
+  {{- if .Values.ingress.api.annotations }}
+  {{- $annotations = mergeOverwrite $annotations .Values.ingress.api.annotations }}
   {{- end }}
+  annotations:
+    {{- toYaml $annotations | nindent 4 }}
+
 spec:
   {{- if .Values.ingress.api.tlsSecretName }}
   tls:
@@ -26,7 +29,7 @@ spec:
   - host: {{ .Values.ingress.api.hostName }}
     http:
       paths:
-      - path: "/"
+      - path: {{ .Values.ingress.api.path }}
   {{ if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
         pathType: Prefix
         backend:

charts/clearml/templates/ingress-app.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enabled -}}
+{{- if .Values.ingress.app.enabled -}}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@@ -11,10 +11,12 @@ metadata:
   name: {{ include "clearml.fullname" . }}-app
   labels:
     {{- include "clearml.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
+  {{- $annotations := .Values.ingress.annotations }}
+  {{- if .Values.ingress.app.annotations }}
+  {{- $annotations = mergeOverwrite $annotations .Values.ingress.app.annotations }}
   {{- end }}
+  annotations:
+    {{- toYaml $annotations | nindent 4 }}
 spec:
   {{- if .Values.ingress.app.tlsSecretName }}
   tls:
@@ -26,7 +28,7 @@ spec:
   - host: {{ .Values.ingress.app.hostName }}
     http:
       paths:
-      - path: "/"
+      - path: {{ .Values.ingress.app.path }}
   {{ if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
         pathType: Prefix
         backend:

charts/clearml/templates/ingress-files.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enabled -}}
+{{- if .Values.ingress.files.enabled -}}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@@ -11,10 +11,12 @@ metadata:
   name: {{ include "clearml.fullname" . }}-files
   labels:
     {{- include "clearml.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
+  {{- $annotations := .Values.ingress.annotations }}
+  {{- if .Values.ingress.files.annotations }}
+  {{- $annotations = mergeOverwrite $annotations .Values.ingress.files.annotations }}
   {{- end }}
+  annotations:
+    {{- toYaml $annotations | nindent 4 }}
 spec:
   {{- if .Values.ingress.files.tlsSecretName }}
   tls:
@@ -26,7 +28,7 @@ spec:
   - host: {{ .Values.ingress.files.hostName }}
     http:
       paths:
-      - path: "/"
+      - path: {{ .Values.ingress.files.path }}
   {{ if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
         pathType: Prefix
         backend:

charts/clearml/templates/rbac-agentk8s.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.agentk8sglue.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: k8sagent-pods-access
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs: ["get", "list", "watch", "create", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: k8sagent-pods-access
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: k8sagent-pods-access
+{{- end }}

charts/clearml/values.yaml

@@ -1,18 +1,29 @@
+# global:
+#   imagePullSecrets:
+#   - docker-cfg
 clearml:
   defaultCompany: "d1bd92a3b039400cbafc60a7a5b1e52b"
 ingress:
-  enabled: false
   name: clearml-server-ingress
   annotations: {}
   app:
+    enabled: false
     hostName: "app.clearml.127-0-0-1.nip.io"
     tlsSecretName: ""
+    annotations: {}
+    path: "/"
   api:
+    enabled: false
     hostName: "api.clearml.127-0-0-1.nip.io"
     tlsSecretName: ""
+    annotations: {}
+    path: "/"
   files:
+    enabled: false
     hostName: "files.clearml.127-0-0-1.nip.io"
     tlsSecretName: ""
+    annotations: {}
+    path: "/"
 
 secret:
   # -- Set for http_session field
@@ -56,7 +67,7 @@ apiserver:
   image:
     repository: "allegroai/clearml"
     pullPolicy: IfNotPresent
-    tag: "1.1.1"
+    tag: "1.4.0"
 
   extraEnvs: []
 
@@ -80,6 +91,7 @@ apiserver:
 
   affinity: {}
 
+  # -- additional configurations that can be used by api server; check examples in values.yaml file
   additionalConfigs: {}
     # services.conf: |
     #   tasks {
@@ -90,6 +102,25 @@ apiserver:
     #       watch_interval_sec: 900
     #     }
     #   }
+    # apiserver.conf: |
+    #   auth {
+    #     fixed_users {
+    #       enabled: true
+    #       pass_hashed: false
+    #       users: [
+    #         {
+    #           username: "jane"
+    #           password: "12345678"
+    #           name: "Jane Doe"
+    #         },
+    #         {
+    #           username: "john"
+    #           password: "12345678"
+    #           name: "John Doe"
+    #         },
+    #       ]
+    #     }
+    #   }
 
 fileserver:
   service:
@@ -105,7 +136,7 @@ fileserver:
   image:
     repository: "allegroai/clearml"
     pullPolicy: IfNotPresent
-    tag: "1.1.1"
+    tag: "1.4.0"
 
   extraEnvs: []
 
@@ -150,7 +181,7 @@ webserver:
   image:
     repository: "allegroai/clearml"
     pullPolicy: IfNotPresent
-    tag: "1.1.1"
+    tag: "1.4.0"
 
   podAnnotations: {}
 
@@ -172,6 +203,8 @@ webserver:
 
   affinity: {}
 
+  additionalConfigs: {}
+
 agentservices:
   enabled: false
   clearmlHostIp: null
@@ -224,7 +257,7 @@ agentservices:
 
 agentGroups:
   agent-group-cpu:
-    enabled: true
+    enabled: false
     name: agent-group-cpu
     replicaCount: 1
     updateStrategy: Recreate
@@ -258,7 +291,7 @@ agentGroups:
     affinity: {}
 
   agent-group-gpu:
-    enabled: true
+    enabled: false
     name: agent-group-gpu
     replicaCount: 0
     updateStrategy: Recreate
@@ -291,6 +324,43 @@ agentGroups:
 
     affinity: {}
 
+# This agent will spawn queued experiments in new pods, a good use case is to combine this with
+# GPU autoscaling nodes.
+# https://github.com/allegroai/clearml-agent/tree/master/docker/k8s-glue
+agentk8sglue:
+  enabled: true
+  image:
+    repository: "allegroai/clearml-agent-k8s"
+    tag: "latest"
+  serviceAccountName: default
+  maxPods: 10
+  defaultDockerImage: nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04   # default docker image that is spawned as new pod
+  queue: default
+  id: k8s-agent
+  podTemplate:
+    volumes: []
+      # - name: "yourvolume"
+      #   path: "/yourpath"
+    env: []
+      # # to setup access to private repo, setup secret with git credentials:
+      # - name: CLEARML_AGENT_GIT_USER
+      #   value: mygitusername
+      # - name: CLEARML_AGENT_GIT_PASS
+      #   valueFrom:
+      #     secretKeyRef:
+      #       name: git-password
+      #       key: git-password
+    resources: {}
+      # limits:
+      #   nvidia.com/gpu: 1
+    tolerations: []
+      # - key: "nvidia.com/gpu"
+      #   operator: Exists
+      #   effect: "NoSchedule"
+    nodeSelector: {}
+      # fleet: gpu-nodes
+
+
 externalServices:
   # -- Existing ElasticSearch Hostname to use if elasticsearch.enabled is false
   elasticsearchHost: ""
@@ -337,7 +407,7 @@ mongodb:  # configuration from https://github.com/bitnami/charts/blob/master/bit
     port: 27017
     portName: mongo-service
 
-elasticsearch:  # configuration from https://github.com/elastic/helm-charts/blob/7.10/elasticsearch/values.yaml
+elasticsearch:  # configuration from https://github.com/elastic/helm-charts/blob/7.16/elasticsearch/values.yaml
   enabled: true
   httpPort: 9200
   roles: