diff --git a/apiserver/database/errors.py b/apiserver/database/errors.py index daa96de..1d62f47 100644 --- a/apiserver/database/errors.py +++ b/apiserver/database/errors.py @@ -16,7 +16,7 @@ from mongoengine.errors import ( LookUpError, InvalidQueryError, ) -from pymongo.errors import PyMongoError, NotMasterError +from pymongo.errors import PyMongoError, NotPrimaryError from apiserver.apierrors import errors @@ -198,7 +198,7 @@ def translate_errors_context(message=None, **kwargs): MongoEngineErrorsHandler.invalid_query_error(e, message, **kwargs) except PyMongoError as e: raise errors.server_error.InternalError(message, err=str(e)) - except NotMasterError as e: + except NotPrimaryError as e: raise errors.server_error.InternalError(message, err=str(e)) except MakeGetAllQueryError as e: raise errors.bad_request.ValidationError(e.error, field=e.field) diff --git a/apiserver/mongo/initialize/migration.py b/apiserver/mongo/initialize/migration.py index daa58b0..04f6da3 100644 --- a/apiserver/mongo/initialize/migration.py +++ b/apiserver/mongo/initialize/migration.py @@ -19,7 +19,7 @@ _migration_dir = _parent_dir / _migrations def check_mongo_empty() -> bool: return not all( - get_db(alias).collection_names() for alias in utils.get_options(Database) + get_db(alias).list_collection_names() for alias in utils.get_options(Database) ) diff --git a/apiserver/requirements.txt b/apiserver/requirements.txt index 76d3f88..5815fd5 100644 --- a/apiserver/requirements.txt +++ b/apiserver/requirements.txt @@ -1,4 +1,4 @@ -attrs>=22.1.0 +attrs>=22.1.0,<23 azure-storage-blob>=12.13.1 bcrypt>=3.1.4 boltons>=19.1.0 @@ -11,28 +11,25 @@ fastjsonschema>=2.8 flask-compress>=1.4.0 flask-cors>=3.0.5 flask>=0.12.2 -funcsigs==1.0.2 furl>=2.0.0 -google-cloud-storage==2.0.0 -protobuf==3.19.5 +google-cloud-storage>=2.8.0 gunicorn>=19.7.1 -humanfriendly==4.18 -jinja2==2.11.3 +humanfriendly>=4.17 +jinja2<3.0 jsonmodels>=2.3 jsonschema>=2.6.0 luqum>=0.10.0 +markupsafe==2.0.1 mongoengine==0.24.2 nested_dict>=1.61 packaging==20.3 psutil>=5.6.5 -pyhocon>=0.3.35 +pyhocon>=0.3.35r pyjwt>=2.4.0 -pymongo[srv]==3.12.0 +pymongo[srv]==4.1.1 python-rapidjson>=0.6.3 -redis==4.4.4 -redis-py-cluster>=2.1.3 +redis>=4.5.4,<5 requests>=2.13.0 semantic_version>=2.8.3,<3 six -tqdm validators>=0.12.4 \ No newline at end of file diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 3c5e36f..3810d3d 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -10,14 +10,11 @@ RUN mv clearml-web /opt/open-webapp COPY --chmod=744 docker/build/internal_files/build_webapp.sh /tmp/internal_files/ RUN /bin/bash -c '/tmp/internal_files/build_webapp.sh' -FROM centos:7 AS staging_image +FROM python:3.9-slim-bullseye COPY --chmod=744 docker/build/internal_files/entrypoint.sh /opt/clearml/ COPY fileserver /opt/clearml/fileserver/ COPY apiserver /opt/clearml/apiserver/ -FROM centos:7 -COPY --from=staging_image /opt/clearml/ /opt/clearml/ - COPY --chmod=744 docker/build/internal_files/final_image_preparation.sh /tmp/internal_files/ COPY docker/build/internal_files/clearml.conf.template /tmp/internal_files/ COPY docker/build/internal_files/clearml_subpath.conf.template /tmp/internal_files/ diff --git a/docker/build/internal_files/clearml.conf.template b/docker/build/internal_files/clearml.conf.template index 9266805..202627e 100644 --- a/docker/build/internal_files/clearml.conf.template +++ b/docker/build/internal_files/clearml.conf.template @@ -1,98 +1,60 @@ -# For more information on configuration, see: -# * Official English Documentation: http://nginx.org/en/docs/ -# * Official Russian Documentation: http://nginx.org/ru/docs/ +server { + listen 80 default_server; + ${COMMENT_IPV6_LISTEN}listen [::]:80 default_server; + server_name _; + root /usr/share/nginx/html; + proxy_http_version 1.1; + client_max_body_size 0; -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log; -pid /run/nginx.pid; + # compression + gzip on; + gzip_comp_level 9; + gzip_http_version 1.0; + gzip_min_length 512; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain + text/css + application/json + application/javascript + application/x-javascript + text/xml application/xml + application/xml+rss + text/javascript + application/x-font-ttf + font/woff2 + image/svg+xml + image/x-icon; -# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. -include /usr/share/nginx/modules/*.conf; + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; -events { - worker_connections 1024; -} + location / { + try_files $uri$args $uri$args/ $uri index.html /index.html; + } -http { - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; + location /version.json { + add_header Cache-Control 'no-cache'; + } - access_log /var/log/nginx/access.log main; + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_pass ${NGINX_APISERVER_ADDR}; + rewrite /api/(.*) /$1 break; + } - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; + location /files { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_pass ${NGINX_FILESERVER_ADDR}; + rewrite /files/(.*) /$1 break; + } - include /etc/nginx/mime.types; - default_type application/octet-stream; + error_page 404 /404.html; + location = /40x.html { + } - # Load modular configuration files from the /etc/nginx/conf.d directory. - # See http://nginx.org/en/docs/ngx_core_module.html#include - # for more information. - include /etc/nginx/conf.d/*.conf; - - server { - listen 80 default_server; - ${COMMENT_IPV6_LISTEN}listen [::]:80 default_server; - server_name _; - root /usr/share/nginx/html; - proxy_http_version 1.1; - client_max_body_size 0; - - # compression - gzip on; - gzip_comp_level 9; - gzip_http_version 1.0; - gzip_min_length 512; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain - text/css - application/json - application/javascript - application/x-javascript - text/xml application/xml - application/xml+rss - text/javascript - application/x-font-ttf - font/woff2 - image/svg+xml - image/x-icon; - - # Load configuration files for the default server block. - include /etc/nginx/default.d/*.conf; - - location / { - try_files $uri$args $uri$args/ $uri index.html /index.html; - } - - location /version.json { - add_header Cache-Control 'no-cache'; - } - - location /api { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_pass ${NGINX_APISERVER_ADDR}; - rewrite /api/(.*) /$1 break; - } - - location /files { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_pass ${NGINX_FILESERVER_ADDR}; - rewrite /files/(.*) /$1 break; - } - - error_page 404 /404.html; - location = /40x.html { - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - } + error_page 500 502 503 504 /50x.html; + location = /50x.html { } } \ No newline at end of file diff --git a/docker/build/internal_files/entrypoint.sh b/docker/build/internal_files/entrypoint.sh index 3c3507d..8330935 100644 --- a/docker/build/internal_files/entrypoint.sh +++ b/docker/build/internal_files/entrypoint.sh @@ -49,7 +49,7 @@ EOF export NGINX_APISERVER_ADDR=${NGINX_APISERVER_ADDRESS:-http://apiserver:8008} export NGINX_FILESERVER_ADDR=${NGINX_FILESERVER_ADDRESS:-http://fileserver:8081} COMMENT_IPV6_LISTEN=$([ "$DISABLE_NGINX_IPV6" = "true" ] && echo "#" || echo "") \ - envsubst '${COMMENT_IPV6_LISTEN} ${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/nginx.conf + envsubst '${COMMENT_IPV6_LISTEN} ${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/sites-enabled/default if [[ -n "${CLEARML_SERVER_SUB_PATH}" ]]; then envsubst '${CLEARML_SERVER_SUB_PATH}' < /etc/nginx/clearml_subpath.conf.template > /etc/nginx/default.d/clearml_subpath.conf diff --git a/docker/build/internal_files/final_image_preparation.sh b/docker/build/internal_files/final_image_preparation.sh index 195e142..b7b6026 100644 --- a/docker/build/internal_files/final_image_preparation.sh +++ b/docker/build/internal_files/final_image_preparation.sh @@ -3,19 +3,20 @@ set -o errexit set -o nounset set -o pipefail -yum update -y -yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -yum install -y python36 python36-pip nginx gcc gcc-c++ python3-devel gettext -yum -y upgrade -python3 -m pip install -U pip +apt-get update -y +apt-get install -y python3-setuptools python3-dev build-essential nginx gettext +apt-get install -y vim curl + +python3 -m ensurepip +python3 -m pip install --upgrade pip python3 -m pip install -r /opt/clearml/fileserver/requirements.txt python3 -m pip install -r /opt/clearml/apiserver/requirements.txt mkdir -p /opt/clearml/log mkdir -p /opt/clearml/config -ln -s /dev/stdout /var/log/nginx/access.log -ln -s /dev/stderr /var/log/nginx/error.log -mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.orig +ln -svf /dev/stdout /var/log/nginx/access.log +ln -svf /dev/stderr /var/log/nginx/error.log mv /tmp/internal_files/clearml.conf.template /etc/nginx/clearml.conf.template mv /tmp/internal_files/clearml_subpath.conf.template /etc/nginx/clearml_subpath.conf.template -pip cache purge -yum clean all + +rm -d -r "$(pip cache dir)" +apt-get clean diff --git a/fileserver/requirements.txt b/fileserver/requirements.txt index 0f282a7..d1209c6 100644 --- a/fileserver/requirements.txt +++ b/fileserver/requirements.txt @@ -1,5 +1,7 @@ boltons>=19.1.0 -Flask -Flask-Cors>=3.0.5 -Flask-Compress>=1.4.0 +flask-compress>=1.4.0 +flask-cors>=3.0.5 +flask>=0.12.2 +gunicorn>=19.7.1 +markupsafe==2.0.1 pyhocon>=0.3.35 \ No newline at end of file