Migrate to python 3.9 bullseye docker images

Update Mongo driver version

apiserver/database/errors.py

@@ -16,7 +16,7 @@ from mongoengine.errors import (
     LookUpError,
     InvalidQueryError,
 )
-from pymongo.errors import PyMongoError, NotMasterError
+from pymongo.errors import PyMongoError, NotPrimaryError
 
 from apiserver.apierrors import errors
 
@@ -198,7 +198,7 @@ def translate_errors_context(message=None, **kwargs):
         MongoEngineErrorsHandler.invalid_query_error(e, message, **kwargs)
     except PyMongoError as e:
         raise errors.server_error.InternalError(message, err=str(e))
-    except NotMasterError as e:
+    except NotPrimaryError as e:
         raise errors.server_error.InternalError(message, err=str(e))
     except MakeGetAllQueryError as e:
         raise errors.bad_request.ValidationError(e.error, field=e.field)

apiserver/mongo/initialize/migration.py

@@ -19,7 +19,7 @@ _migration_dir = _parent_dir / _migrations
 
 def check_mongo_empty() -> bool:
     return not all(
-        get_db(alias).collection_names() for alias in utils.get_options(Database)
+        get_db(alias).list_collection_names() for alias in utils.get_options(Database)
     )
 
 

apiserver/requirements.txt

@@ -1,4 +1,4 @@
-attrs>=22.1.0
+attrs>=22.1.0,<23
 azure-storage-blob>=12.13.1
 bcrypt>=3.1.4
 boltons>=19.1.0
@@ -11,28 +11,25 @@ fastjsonschema>=2.8
 flask-compress>=1.4.0
 flask-cors>=3.0.5
 flask>=0.12.2
-funcsigs==1.0.2
 furl>=2.0.0
-google-cloud-storage==2.0.0
-protobuf==3.19.5
+google-cloud-storage>=2.8.0
 gunicorn>=19.7.1
-humanfriendly==4.18
-jinja2==2.11.3
+humanfriendly>=4.17
+jinja2<3.0
 jsonmodels>=2.3
 jsonschema>=2.6.0
 luqum>=0.10.0
+markupsafe==2.0.1
 mongoengine==0.24.2
 nested_dict>=1.61
 packaging==20.3
 psutil>=5.6.5
-pyhocon>=0.3.35
+pyhocon>=0.3.35r
 pyjwt>=2.4.0
-pymongo[srv]==3.12.0
+pymongo[srv]==4.1.1
 python-rapidjson>=0.6.3
-redis==4.4.4
-redis-py-cluster>=2.1.3
+redis>=4.5.4,<5
 requests>=2.13.0
 semantic_version>=2.8.3,<3
 six
-tqdm
 validators>=0.12.4

docker/build/Dockerfile

@@ -10,14 +10,11 @@ RUN mv clearml-web /opt/open-webapp
 COPY --chmod=744 docker/build/internal_files/build_webapp.sh /tmp/internal_files/
 RUN /bin/bash -c '/tmp/internal_files/build_webapp.sh'
 
-FROM centos:7 AS staging_image
+FROM python:3.9-slim-bullseye
 COPY --chmod=744 docker/build/internal_files/entrypoint.sh /opt/clearml/
 COPY fileserver /opt/clearml/fileserver/
 COPY apiserver /opt/clearml/apiserver/
 
-FROM centos:7
-COPY --from=staging_image /opt/clearml/ /opt/clearml/
-
 COPY --chmod=744 docker/build/internal_files/final_image_preparation.sh /tmp/internal_files/
 COPY docker/build/internal_files/clearml.conf.template /tmp/internal_files/
 COPY docker/build/internal_files/clearml_subpath.conf.template /tmp/internal_files/

docker/build/internal_files/clearml.conf.template

@@ -1,98 +1,60 @@
-# For more information on configuration, see:
-#   * Official English Documentation: http://nginx.org/en/docs/
-#   * Official Russian Documentation: http://nginx.org/ru/docs/
+server {
+    listen       80 default_server;
+    ${COMMENT_IPV6_LISTEN}listen       [::]:80 default_server;
+    server_name  _;
+    root         /usr/share/nginx/html;
+    proxy_http_version 1.1;
+    client_max_body_size 0;
 
-user nginx;
-worker_processes auto;
-error_log /var/log/nginx/error.log;
-pid /run/nginx.pid;
+    # compression
+    gzip            on;
+    gzip_comp_level 9;
+    gzip_http_version 1.0;
+    gzip_min_length 512;
+    gzip_proxied    expired no-cache no-store private auth;
+    gzip_types  text/plain
+                text/css
+                application/json
+                application/javascript
+                application/x-javascript
+                text/xml application/xml
+                application/xml+rss
+                text/javascript
+                application/x-font-ttf
+                font/woff2
+                image/svg+xml
+                image/x-icon;
 
-# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
-include /usr/share/nginx/modules/*.conf;
+    # Load configuration files for the default server block.
+    include /etc/nginx/default.d/*.conf;
 
-events {
-    worker_connections 1024;
-}
+    location / {
+        try_files $uri$args $uri$args/ $uri index.html /index.html;
+    }
 
-http {
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
+    location /version.json {
+        add_header Cache-Control 'no-cache';
+    }
 
-    access_log  /var/log/nginx/access.log  main;
+    location /api {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_pass ${NGINX_APISERVER_ADDR};
+        rewrite /api/(.*) /$1  break;
+    }
 
-    sendfile            on;
-    tcp_nopush          on;
-    tcp_nodelay         on;
-    keepalive_timeout   65;
-    types_hash_max_size 2048;
+    location /files {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_pass ${NGINX_FILESERVER_ADDR};
+        rewrite /files/(.*) /$1  break;
+    }
 
-    include             /etc/nginx/mime.types;
-    default_type        application/octet-stream;
+    error_page 404 /404.html;
+        location = /40x.html {
+    }
 
-    # Load modular configuration files from the /etc/nginx/conf.d directory.
-    # See http://nginx.org/en/docs/ngx_core_module.html#include
-    # for more information.
-    include /etc/nginx/conf.d/*.conf;
-
-    server {
-        listen       80 default_server;
-        ${COMMENT_IPV6_LISTEN}listen       [::]:80 default_server;
-        server_name  _;
-        root         /usr/share/nginx/html;
-        proxy_http_version 1.1;
-        client_max_body_size 0;
-
-        # compression
-        gzip            on;
-        gzip_comp_level 9;
-        gzip_http_version 1.0;
-        gzip_min_length 512;
-        gzip_proxied    expired no-cache no-store private auth;
-        gzip_types  text/plain
-                    text/css
-                    application/json
-                    application/javascript
-                    application/x-javascript
-                    text/xml application/xml
-                    application/xml+rss
-                    text/javascript
-                    application/x-font-ttf
-                    font/woff2
-                    image/svg+xml
-                    image/x-icon;
-
-        # Load configuration files for the default server block.
-        include /etc/nginx/default.d/*.conf;
-
-        location / {
-            try_files $uri$args $uri$args/ $uri index.html /index.html;
-        }
-
-        location /version.json {
-            add_header Cache-Control 'no-cache';
-        }
-
-        location /api {
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_pass ${NGINX_APISERVER_ADDR};
-            rewrite /api/(.*) /$1  break;
-        }
-
-        location /files {
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_pass ${NGINX_FILESERVER_ADDR};
-            rewrite /files/(.*) /$1  break;
-        }
-
-        error_page 404 /404.html;
-            location = /40x.html {
-        }
-
-        error_page 500 502 503 504 /50x.html;
-            location = /50x.html {
-        }
+    error_page 500 502 503 504 /50x.html;
+        location = /50x.html {
     }
 }

docker/build/internal_files/entrypoint.sh

@@ -49,7 +49,7 @@ EOF
     export NGINX_APISERVER_ADDR=${NGINX_APISERVER_ADDRESS:-http://apiserver:8008}
     export NGINX_FILESERVER_ADDR=${NGINX_FILESERVER_ADDRESS:-http://fileserver:8081}
     COMMENT_IPV6_LISTEN=$([ "$DISABLE_NGINX_IPV6" = "true" ] && echo "#" || echo "") \
-     envsubst '${COMMENT_IPV6_LISTEN} ${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/nginx.conf
+     envsubst '${COMMENT_IPV6_LISTEN} ${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/sites-enabled/default
 
     if [[ -n "${CLEARML_SERVER_SUB_PATH}" ]]; then
       envsubst '${CLEARML_SERVER_SUB_PATH}' < /etc/nginx/clearml_subpath.conf.template > /etc/nginx/default.d/clearml_subpath.conf

docker/build/internal_files/final_image_preparation.sh

@@ -3,19 +3,20 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-yum update -y
-yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-yum install -y python36 python36-pip nginx gcc gcc-c++ python3-devel gettext
-yum -y upgrade
-python3 -m pip install -U pip
+apt-get update -y
+apt-get install -y python3-setuptools python3-dev build-essential nginx gettext
+apt-get install -y vim curl
+
+python3 -m ensurepip
+python3 -m pip install --upgrade pip
 python3 -m pip install -r /opt/clearml/fileserver/requirements.txt
 python3 -m pip install -r /opt/clearml/apiserver/requirements.txt
 mkdir -p /opt/clearml/log
 mkdir -p /opt/clearml/config
-ln -s /dev/stdout /var/log/nginx/access.log
-ln -s /dev/stderr /var/log/nginx/error.log
-mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.orig
+ln -svf /dev/stdout /var/log/nginx/access.log
+ln -svf /dev/stderr /var/log/nginx/error.log
 mv /tmp/internal_files/clearml.conf.template /etc/nginx/clearml.conf.template
 mv /tmp/internal_files/clearml_subpath.conf.template /etc/nginx/clearml_subpath.conf.template
-pip cache purge
-yum clean all
+
+rm -d -r "$(pip cache dir)"
+apt-get clean

fileserver/requirements.txt

@@ -1,5 +1,7 @@
 boltons>=19.1.0
-Flask
-Flask-Cors>=3.0.5
-Flask-Compress>=1.4.0
+flask-compress>=1.4.0
+flask-cors>=3.0.5
+flask>=0.12.2
+gunicorn>=19.7.1
+markupsafe==2.0.1
 pyhocon>=0.3.35