Migrate to python 3.9 bullseye docker images

Update Mongo driver version

apiserver/database/errors.py

@@ -16,7 +16,7 @@ from mongoengine.errors import (
     LookUpError,
     InvalidQueryError,
 )
-from pymongo.errors import PyMongoError, NotMasterError
+from pymongo.errors import PyMongoError, NotPrimaryError
 
 from apiserver.apierrors import errors
 
@@ -198,7 +198,7 @@ def translate_errors_context(message=None, **kwargs):
         MongoEngineErrorsHandler.invalid_query_error(e, message, **kwargs)
     except PyMongoError as e:
         raise errors.server_error.InternalError(message, err=str(e))
-    except NotMasterError as e:
+    except NotPrimaryError as e:
         raise errors.server_error.InternalError(message, err=str(e))
     except MakeGetAllQueryError as e:
         raise errors.bad_request.ValidationError(e.error, field=e.field)

apiserver/mongo/initialize/migration.py

@@ -19,7 +19,7 @@ _migration_dir = _parent_dir / _migrations
 
 def check_mongo_empty() -> bool:
     return not all(
-        get_db(alias).collection_names() for alias in utils.get_options(Database)
+        get_db(alias).list_collection_names() for alias in utils.get_options(Database)
     )
 
 

apiserver/requirements.txt

@@ -1,4 +1,4 @@
-attrs>=22.1.0
+attrs>=22.1.0,<23
 azure-storage-blob>=12.13.1
 bcrypt>=3.1.4
 boltons>=19.1.0
@@ -11,28 +11,25 @@ fastjsonschema>=2.8
 flask-compress>=1.4.0
 flask-cors>=3.0.5
 flask>=0.12.2
-funcsigs==1.0.2
 furl>=2.0.0
-google-cloud-storage==2.0.0
-protobuf==3.19.5
+google-cloud-storage>=2.8.0
 gunicorn>=19.7.1
-humanfriendly==4.18
-jinja2==2.11.3
+humanfriendly>=4.17
+jinja2<3.0
 jsonmodels>=2.3
 jsonschema>=2.6.0
 luqum>=0.10.0
+markupsafe==2.0.1
 mongoengine==0.24.2
 nested_dict>=1.61
 packaging==20.3
 psutil>=5.6.5
-pyhocon>=0.3.35
+pyhocon>=0.3.35r
 pyjwt>=2.4.0
-pymongo[srv]==3.12.0
+pymongo[srv]==4.1.1
 python-rapidjson>=0.6.3
-redis==4.4.4
-redis-py-cluster>=2.1.3
+redis>=4.5.4,<5
 requests>=2.13.0
 semantic_version>=2.8.3,<3
 six
-tqdm
 validators>=0.12.4

docker/build/Dockerfile

@@ -10,14 +10,11 @@ RUN mv clearml-web /opt/open-webapp
 COPY --chmod=744 docker/build/internal_files/build_webapp.sh /tmp/internal_files/
 RUN /bin/bash -c '/tmp/internal_files/build_webapp.sh'
 
-FROM centos:7 AS staging_image
+FROM python:3.9-slim-bullseye
 COPY --chmod=744 docker/build/internal_files/entrypoint.sh /opt/clearml/
 COPY fileserver /opt/clearml/fileserver/
 COPY apiserver /opt/clearml/apiserver/
 
-FROM centos:7
-COPY --from=staging_image /opt/clearml/ /opt/clearml/
-
 COPY --chmod=744 docker/build/internal_files/final_image_preparation.sh /tmp/internal_files/
 COPY docker/build/internal_files/clearml.conf.template /tmp/internal_files/
 COPY docker/build/internal_files/clearml_subpath.conf.template /tmp/internal_files/

docker/build/internal_files/clearml.conf.template

@@ -1,40 +1,3 @@
-# For more information on configuration, see:
-#   * Official English Documentation: http://nginx.org/en/docs/
-#   * Official Russian Documentation: http://nginx.org/ru/docs/
-
-user nginx;
-worker_processes auto;
-error_log /var/log/nginx/error.log;
-pid /run/nginx.pid;
-
-# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
-include /usr/share/nginx/modules/*.conf;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
-
-    access_log  /var/log/nginx/access.log  main;
-
-    sendfile            on;
-    tcp_nopush          on;
-    tcp_nodelay         on;
-    keepalive_timeout   65;
-    types_hash_max_size 2048;
-
-    include             /etc/nginx/mime.types;
-    default_type        application/octet-stream;
-
-    # Load modular configuration files from the /etc/nginx/conf.d directory.
-    # See http://nginx.org/en/docs/ngx_core_module.html#include
-    # for more information.
-    include /etc/nginx/conf.d/*.conf;
-
 server {
     listen       80 default_server;
     ${COMMENT_IPV6_LISTEN}listen       [::]:80 default_server;
@@ -95,4 +58,3 @@ http {
         location = /50x.html {
     }
 }
-}

docker/build/internal_files/entrypoint.sh

@@ -49,7 +49,7 @@ EOF
     export NGINX_APISERVER_ADDR=${NGINX_APISERVER_ADDRESS:-http://apiserver:8008}
     export NGINX_FILESERVER_ADDR=${NGINX_FILESERVER_ADDRESS:-http://fileserver:8081}
     COMMENT_IPV6_LISTEN=$([ "$DISABLE_NGINX_IPV6" = "true" ] && echo "#" || echo "") \
-     envsubst '${COMMENT_IPV6_LISTEN} ${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/nginx.conf
+     envsubst '${COMMENT_IPV6_LISTEN} ${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/sites-enabled/default
 
     if [[ -n "${CLEARML_SERVER_SUB_PATH}" ]]; then
       envsubst '${CLEARML_SERVER_SUB_PATH}' < /etc/nginx/clearml_subpath.conf.template > /etc/nginx/default.d/clearml_subpath.conf

docker/build/internal_files/final_image_preparation.sh

@@ -3,19 +3,20 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-yum update -y
-yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-yum install -y python36 python36-pip nginx gcc gcc-c++ python3-devel gettext
-yum -y upgrade
-python3 -m pip install -U pip
+apt-get update -y
+apt-get install -y python3-setuptools python3-dev build-essential nginx gettext
+apt-get install -y vim curl
+
+python3 -m ensurepip
+python3 -m pip install --upgrade pip
 python3 -m pip install -r /opt/clearml/fileserver/requirements.txt
 python3 -m pip install -r /opt/clearml/apiserver/requirements.txt
 mkdir -p /opt/clearml/log
 mkdir -p /opt/clearml/config
-ln -s /dev/stdout /var/log/nginx/access.log
-ln -s /dev/stderr /var/log/nginx/error.log
-mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.orig
+ln -svf /dev/stdout /var/log/nginx/access.log
+ln -svf /dev/stderr /var/log/nginx/error.log
 mv /tmp/internal_files/clearml.conf.template /etc/nginx/clearml.conf.template
 mv /tmp/internal_files/clearml_subpath.conf.template /etc/nginx/clearml_subpath.conf.template
-pip cache purge
-yum clean all
+
+rm -d -r "$(pip cache dir)"
+apt-get clean

fileserver/requirements.txt

@@ -1,5 +1,7 @@
 boltons>=19.1.0
-Flask
-Flask-Cors>=3.0.5
-Flask-Compress>=1.4.0
+flask-compress>=1.4.0
+flask-cors>=3.0.5
+flask>=0.12.2
+gunicorn>=19.7.1
+markupsafe==2.0.1
 pyhocon>=0.3.35