Release 1.15.0 (#272)

* Changed: upgrade app to 1.15 * Fixed: missing scheme * Added: async delete
2025-01-31 00:56:52 +00:00 · 2024-03-21 14:58:07 +01:00 · 2024-03-21 14:58:07 +01:00 · 4ca4bc82c4
commit 4ca4bc82c4
parent 75d9086bb9
5 changed files with 179 additions and 14 deletions
--- a/charts/clearml/Chart.yaml
+++ b/charts/clearml/Chart.yaml
@ -2,8 +2,8 @@ apiVersion: v2
 name: clearml
 description: MLOps platform
 type: application
-version: "7.7.1"
-appVersion: "1.14.1"
+version: "7.8.0"
+appVersion: "1.15"
 kubeVersion: ">= 1.21.0-0 < 1.30.0-0"
 home: https://clear.ml
 icon: https://raw.githubusercontent.com/allegroai/clearml/master/docs/clearml-logo.svg
@ -33,4 +33,6 @@ dependencies:
 annotations:
  artifacthub.io/changes: |
    - kind: changed
-      description: kubernetes 1.29 support
+      description: application version 1.15
+    - kind: added
+      description: async delete deployment
--- a/charts/clearml/README.md
+++ b/charts/clearml/README.md
@ -1,6 +1,6 @@
 # ClearML Ecosystem for Kubernetes

-![Version: 7.7.1](https://img.shields.io/badge/Version-7.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.14.1](https://img.shields.io/badge/AppVersion-1.14.1-informational?style=flat-square)
+![Version: 7.8.0](https://img.shields.io/badge/Version-7.8.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.15](https://img.shields.io/badge/AppVersion-1.15-informational?style=flat-square)

 MLOps platform

@ -145,7 +145,7 @@ Kubernetes: `>= 1.21.0-0 < 1.30.0-0`

 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| apiserver | object | `{"additionalConfigs":{},"affinity":{},"containerSecurityContext":{},"enabled":true,"existingAdditionalConfigsConfigMap":"","existingAdditionalConfigsSecret":"","extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.14.1-448"},"ingress":{"annotations":{},"enabled":false,"hostName":"api.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"prepopulateEnabled":true,"processes":{"count":8,"maxRequests":1000,"maxRequestsJitter":300,"timeout":24000},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30008,"port":8008,"type":"NodePort"},"serviceAccountName":"clearml","tolerations":[]}` | Api Server configurations |
+| apiserver | object | `{"additionalConfigs":{},"affinity":{},"containerSecurityContext":{},"enabled":true,"existingAdditionalConfigsConfigMap":"","existingAdditionalConfigsSecret":"","extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.15.0-472"},"ingress":{"annotations":{},"enabled":false,"hostName":"api.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"prepopulateEnabled":true,"processes":{"count":8,"maxRequests":1000,"maxRequestsJitter":300,"timeout":24000},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30008,"port":8008,"type":"NodePort"},"serviceAccountName":"clearml","tolerations":[]}` | Api Server configurations |
 | apiserver.additionalConfigs | object | `{}` | files declared in this parameter will be mounted and read by apiserver (examples in values.yaml) if not overridden by existingAdditionalConfigsSecret |
 | apiserver.affinity | object | `{}` | Api Server affinity setup |
 | apiserver.containerSecurityContext | object | `{}` | Api Server containers security context |
@ -153,7 +153,7 @@ Kubernetes: `>= 1.21.0-0 < 1.30.0-0`
 | apiserver.existingAdditionalConfigsConfigMap | string | `""` | reference for files declared in existing ConfigMap will be mounted and read by apiserver (examples in values.yaml) |
 | apiserver.existingAdditionalConfigsSecret | string | `""` | reference for files declared in existing Secret will be mounted and read by apiserver (examples in values.yaml) if not overridden by existingAdditionalConfigsConfigMap |
 | apiserver.extraEnvs | list | `[]` | Api Server extra envrinoment variables |
-| apiserver.image | object | `{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.14.1-448"}` | Api Server image configuration |
+| apiserver.image | object | `{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.15.0-472"}` | Api Server image configuration |
 | apiserver.ingress | object | `{"annotations":{},"enabled":false,"hostName":"api.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""}` | Ingress configuration for Api Server component |
 | apiserver.ingress.annotations | object | `{}` | Ingress annotations |
 | apiserver.ingress.enabled | bool | `false` | Enable/Disable ingress |
@ -201,12 +201,12 @@ Kubernetes: `>= 1.21.0-0 < 1.30.0-0`
 | externalServices.mongodbConnectionStringBackend | string | `"mongodb://mongodb_hostnamehostname:27017/backend"` | Existing MongoDB connection string for AUTH to use if mongodb.enabled is false (example in values.yaml) |
 | externalServices.redisHost | string | `"redis_hostname"` | Existing Redis Hostname to use if redis.enabled is false (example in values.yaml) |
 | externalServices.redisPort | int | `6379` | Existing Redis Port to use if redis.enabled is false |
-| fileserver | object | `{"affinity":{},"containerSecurityContext":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.14.1-448"},"ingress":{"annotations":{},"enabled":false,"hostName":"files.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30081,"port":8081,"type":"NodePort"},"serviceAccountName":"clearml","storage":{"data":{"accessMode":"ReadWriteOnce","class":"","existingPVC":"","size":"50Gi"},"enabled":true},"tolerations":[]}` | File Server configurations |
+| fileserver | object | `{"affinity":{},"containerSecurityContext":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.15.0-472"},"ingress":{"annotations":{},"enabled":false,"hostName":"files.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30081,"port":8081,"type":"NodePort"},"serviceAccountName":"clearml","storage":{"data":{"accessMode":"ReadWriteOnce","class":"","existingPVC":"","size":"50Gi"},"enabled":true},"tolerations":[]}` | File Server configurations |
 | fileserver.affinity | object | `{}` | File Server affinity setup |
 | fileserver.containerSecurityContext | object | `{}` | File Server containers security context |
 | fileserver.enabled | bool | `true` | Enable/Disable component deployment |
 | fileserver.extraEnvs | list | `[]` | File Server extra envrinoment variables |
-| fileserver.image | object | `{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.14.1-448"}` | File Server image configuration |
+| fileserver.image | object | `{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.15.0-472"}` | File Server image configuration |
 | fileserver.ingress | object | `{"annotations":{},"enabled":false,"hostName":"files.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""}` | Ingress configuration for File Server component |
 | fileserver.ingress.annotations | object | `{}` | Ingress annotations |
 | fileserver.ingress.enabled | bool | `false` | Enable/Disable ingress |
@ -241,13 +241,13 @@ Kubernetes: `>= 1.21.0-0 < 1.30.0-0`
 | imageCredentials.username | string | `"someone"` | Registry username |
 | mongodb | object | `{"architecture":"standalone","auth":{"enabled":false},"enabled":true,"persistence":{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"50Gi","storageClass":null},"replicaCount":1}` | Configuration from https://github.com/bitnami/charts/blob/master/bitnami/mongodb/values.yaml |
 | redis | object | `{"architecture":"standalone","auth":{"enabled":false},"databaseNumber":0,"enabled":true,"master":{"name":"{{ .Release.Name }}-redis-master","persistence":{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"5Gi","storageClass":null},"port":6379}}` | Configuration from https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml |
-| webserver | object | `{"additionalConfigs":{},"affinity":{},"containerSecurityContext":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.14.1-448"},"ingress":{"annotations":{},"enabled":false,"hostName":"app.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30080,"port":8080,"type":"NodePort"},"serviceAccountName":"clearml","tolerations":[]}` | Web Server configurations |
+| webserver | object | `{"additionalConfigs":{},"affinity":{},"containerSecurityContext":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.15.0-472"},"ingress":{"annotations":{},"enabled":false,"hostName":"app.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30080,"port":8080,"type":"NodePort"},"serviceAccountName":"clearml","tolerations":[]}` | Web Server configurations |
 | webserver.additionalConfigs | object | `{}` | Additional specific webserver configurations |
 | webserver.affinity | object | `{}` | Web Server affinity setup |
 | webserver.containerSecurityContext | object | `{}` | Web Server containers security context |
 | webserver.enabled | bool | `true` | Enable/Disable component deployment |
 | webserver.extraEnvs | list | `[]` | Web Server extra envrinoment variables |
-| webserver.image | object | `{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.14.1-448"}` | Web Server image configuration |
+| webserver.image | object | `{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"1.15.0-472"}` | Web Server image configuration |
 | webserver.ingress | object | `{"annotations":{},"enabled":false,"hostName":"app.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""}` | Ingress configuration for Web Server component |
 | webserver.ingress.annotations | object | `{}` | Ingress annotations |
 | webserver.ingress.enabled | bool | `false` | Enable/Disable ingress |
--- a/charts/clearml/templates/_helpers.tpl
+++ b/charts/clearml/templates/_helpers.tpl
@ -160,6 +160,23 @@ Create configuration secret name
 {{- if .Values.clearml.existingSecret -}} {{ default "clearml-conf" .Values.clearml.existingSecret | quote }} {{- else -}} "clearml-conf" {{- end }}
 {{- end }}

+{{/*
+compose file url
+*/}}
+{{- define "clearml.fileUrl" -}}
+{{- if .Values.clearml.clientConfigurationFilesUrl }}
+{{- .Values.clearml.clientConfigurationFilesUrl }}
+{{- else if .Values.fileserver.ingress.enabled }}
+{{- $protocol := "http" }}
+{{- if .Values.fileserver.ingress.tlsSecretName }}
+{{- $protocol = "https" }}
+{{- end }}
+{{- printf "%s%s%s" $protocol "://" .Values.fileserver.ingress.hostName }}
+{{- else }}
+{{- printf "%s%s%s%s" "http://" (include "fileserver.referenceName" .) ":" .Values.fileserver.service.port }}
+{{- end }}
+{{- end }}
+
 {{/*
 Elasticsearch Service name
 */}}
@ -174,12 +191,19 @@ Elasticsearch Service port
 {{- .Values.elasticsearch.httpPort }}
 {{- end }}

+{{/*
+Elasticsearch Service schema
+*/}}
+{{- define "elasticsearch.servicescheme" -}}
+{{- .Values.elasticsearch.httpScheme }}
+{{- end }}
+
 {{/*
 Elasticsearch Comnnection string
 */}}
 {{- define "elasticsearch.connectionstring" -}}
 {{- if .Values.elasticsearch.enabled }}
-{{- printf "[{\"host\":\"%s\",\"port\":%s}]" (include "elasticsearch.servicename" .) (include "elasticsearch.serviceport" .) | quote }}
+{{- printf "[{\"host\":\"%s\",\"port\":%s,\"scheme\":\"%s\"}]" (include "elasticsearch.servicename" .) (include "elasticsearch.serviceport" .) (include "elasticsearch.servicescheme" .) | quote }}
 {{- else }}
 {{- .Values.externalServices.elasticsearchConnectionString | quote }}
 {{- end }}
--- a/charts/clearml/templates/apiserver-asyncdelete-deployment.yaml
+++ b/charts/clearml/templates/apiserver-asyncdelete-deployment.yaml
@ -0,0 +1,139 @@
+{{- if .Values.apiserver.enabled }}
+{{- if (include "clearml.fileUrl" .) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "apiserver.referenceName" . }}-asyncdelete
+  labels:
+    {{- include "clearml.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "apiserver.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.apiserver.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "apiserver.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ .Values.apiserver.serviceAccountName }}-apiserver
+      {{- if .Values.imageCredentials.enabled }}
+      imagePullSecrets:
+      {{- if .Values.imageCredentials.existingSecret }}
+      - name: {{ .Values.imageCredentials.existingSecret }}
+      {{- else }}
+      - name: clearml-registry-key
+      {{- end }}
+      {{- end }}
+      {{- if or .Values.apiserver.additionalConfigs .Values.apiserver.existingAdditionalConfigsConfigMap  .Values.apiserver.existingAdditionalConfigsSecret }}
+      volumes:
+        - name: apiserver-config
+          {{- if or .Values.apiserver.existingAdditionalConfigsConfigMap }}
+          configMap:
+            name: {{ .Values.apiserver.existingAdditionalConfigsConfigMap }}
+          {{- else if or .Values.apiserver.existingAdditionalConfigsSecret }}
+          secret:
+            secretName: {{ .Values.apiserver.existingAdditionalConfigsSecret }}
+          {{- else if or .Values.apiserver.additionalConfigs }}
+          configMap:
+            name: "{{ include "apiserver.referenceName" . }}-configmap"
+          {{- end }}
+      {{- end }}
+      securityContext:
+        {{ toYaml .Values.apiserver.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: init-apiserver
+          image: "{{ include "registryNamePrefix" (dict "globalValues" .Values.global "imageRegistryValue" .Values.apiserver.image.registry) }}{{ .Values.apiserver.image.repository }}:{{ .Values.apiserver.image.tag }}"
+          command:
+            - /bin/sh
+            - -c
+            - >
+              set -x;
+              {{- if .Values.elasticsearch.enabled }}
+              while [ $(curl -sw '%{http_code}' "http://{{ include "elasticsearch.servicename" . }}:{{ include "elasticsearch.serviceport" . }}/_cluster/health" -o /dev/null) -ne 200 ] ; do
+                echo "waiting for elasticsearch" ;
+                sleep 5 ;
+              done ;
+              {{- end }}
+              {{- if .Values.mongodb.enabled }}
+              while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ .Release.Name }}-{{ include "mongodb.hostname" . }}:27017" -o /dev/null; echo $?) -ne 49 ] ; do
+                echo "waiting for mongodb" ;
+                sleep 5 ;
+              done ;
+              {{- end }}
+              {{- if .Values.redis.enabled }}
+              while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ include "redis.servicename" . }}:{{ include "redis.serviceport" . }}" -o /dev/null; echo $?) -ne 49 ] ; do
+                echo "waiting for redis" ;
+                sleep 5 ;
+              done ;
+              {{- end }}
+          securityContext:
+            {{ toYaml .Values.apiserver.containerSecurityContext | nindent 12 }}
+          resources:
+            {{- toYaml .Values.apiserver.initContainers.resources | nindent 12 }}
+      containers:
+        - name: clearml-apiserver
+          image: "{{ include "registryNamePrefix" (dict "globalValues" .Values.global "imageRegistryValue" .Values.apiserver.image.registry) }}{{ .Values.apiserver.image.repository }}:{{ .Values.apiserver.image.tag }}"
+          imagePullPolicy: {{ .Values.apiserver.image.pullPolicy }}
+          command:
+            - /bin/sh
+            - -c
+            - >
+              python3 -m jobs.async_urls_delete --fileserver-host http://{{ include "fileserver.referenceName" . }}:{{ .Values.fileserver.service.port }}
+          env:
+          - name: CLEARML_REDIS_SERVICE_HOST
+            value: {{ include "redis.servicename" . }}
+          - name: CLEARML_REDIS_SERVICE_PORT
+            value: "{{ include "redis.serviceport" . }}"
+          {{- if .Values.mongodb.enabled }}
+          - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING
+            value: {{ include "mongodb.connectionstring" . | quote }}
+          {{- else }}
+          - name: CLEARML__HOSTS__MONGO__BACKEND__HOST
+            value: {{ .Values.externalServices.mongodbConnectionStringBackend | quote }}
+          - name: CLEARML__HOSTS__MONGO__AUTH__HOST
+            value: {{ .Values.externalServices.mongodbConnectionStringAuth | quote }}
+          {{- end }}
+          - name: CLEARML__HOSTS__ELASTIC__WORKERS__HOSTS
+            value: {{ include "elasticsearch.connectionstring" . }}
+          - name: CLEARML__HOSTS__ELASTIC__EVENTS__HOSTS
+            value: {{ include "elasticsearch.connectionstring" . }}
+          - name: CLEARML__HOSTS__ELASTIC__DATASETS__HOSTS
+            value: {{ include "elasticsearch.connectionstring" . }}
+          - name: CLEARML__HOSTS__ELASTIC__LOGS__HOSTS
+            value: {{ include "elasticsearch.connectionstring" . }}
+          - name: CLEARML__logging__handlers__text_file__filename
+            value: "/dev/null"
+          - name: PYTHONPATH
+            value: /opt/clearml/apiserver
+          - name: CLEARML__apiserver__default_company
+            value: "{{ .Values.clearml.defaultCompanyGuid }}"
+          - name: CLEARML__services__async_urls_delete__fileserver__url_prefixes
+            value: "[\"{{ include "clearml.fileUrl" . }}\"]"
+          - name: CLEARML__secure__auth__token_secret
+            valueFrom:
+              secretKeyRef:
+                name: {{ include "clearml.confSecretName" .}}
+                key: secure_auth_token_secret
+          resources:
+            {{- toYaml .Values.apiserver.resources | nindent 12 }}
+          securityContext:
+            {{ toYaml .Values.apiserver.containerSecurityContext | nindent 12 }}
+      {{- with .Values.apiserver.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.apiserver.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.apiserver.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}
+{{- end }}
--- a/charts/clearml/values.yaml
+++ b/charts/clearml/values.yaml
@ -65,7 +65,7 @@ apiserver:
    registry: ""
    repository: "allegroai/clearml"
    pullPolicy: IfNotPresent
-    tag: "1.14.1-448"
+    tag: "1.15.0-472"
  # -- Api Server internal service configuration
  service:
    # -- specific annotation for Api Server service
@ -181,7 +181,7 @@ fileserver:
    registry: ""
    repository: "allegroai/clearml"
    pullPolicy: IfNotPresent
-    tag: "1.14.1-448"
+    tag: "1.15.0-472"
  # -- File Server internal service configuration
  service:
    # -- specific annotation for File Server service
@ -265,7 +265,7 @@ webserver:
    registry: ""
    repository: "allegroai/clearml"
    pullPolicy: IfNotPresent
-    tag: "1.14.1-448"
+    tag: "1.15.0-472"
  # -- Web Server internal service configuration
  service:
    # -- specific annotation for Web Server service