Add Dockerfile

.gitignore

@@ -12,7 +12,6 @@ test-reports
 .pytest_cache
 venv
 *.noseids
-build
 *.egg-info
 .cache
 .mypy_cache

docker/build/Dockerfile

@@ -0,0 +1,35 @@
+FROM centos/nodejs-12-centos7 AS webapp
+
+USER root
+WORKDIR /opt
+
+RUN git clone https://github.com/allegroai/clearml-web.git
+RUN mv clearml-web /opt/open-webapp
+COPY docker/build/internal_files/build_webapp.sh /tmp/internal_files/
+RUN /bin/bash -c '/tmp/internal_files/build_webapp.sh'
+
+FROM centos:7 AS staging_image
+COPY docker/build/internal_files/entrypoint.sh /opt/clearml/
+COPY fileserver /opt/clearml/fileserver/
+COPY apiserver /opt/clearml/apiserver/
+
+FROM centos:7
+COPY --from=staging_image /opt/clearml/ /opt/clearml/
+
+COPY docker/build/internal_files/final_image_preparation.sh /tmp/internal_files/
+COPY docker/build/internal_files/clearml.conf.template /tmp/internal_files/
+RUN /bin/bash -c '/tmp/internal_files/final_image_preparation.sh'
+
+COPY --from=webapp /opt/open-webapp/build /usr/share/nginx/html
+
+EXPOSE 8080
+EXPOSE 8008
+EXPOSE 8081
+
+ARG VERSION
+ARG BUILD
+ENV CLEARML_SERVER_VERSION=${VERSION}
+ENV CLEARML_SERVER_BUILD=${BUILD}
+
+WORKDIR /opt/clearml/
+ENTRYPOINT ["/opt/clearml/entrypoint.sh"]

docker/build/internal_files/build_webapp.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+cd /opt/open-webapp/
+npm ci --unsafe-perm node-sass
+
+cd /opt/open-webapp/
+npm run build

docker/build/internal_files/clearml.conf.template

@@ -0,0 +1,96 @@
+# For more information on configuration, see:
+#   * Official English Documentation: http://nginx.org/en/docs/
+#   * Official Russian Documentation: http://nginx.org/ru/docs/
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+
+# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
+include /usr/share/nginx/modules/*.conf;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    # Load modular configuration files from the /etc/nginx/conf.d directory.
+    # See http://nginx.org/en/docs/ngx_core_module.html#include
+    # for more information.
+    include /etc/nginx/conf.d/*.conf;
+
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  _;
+        root         /usr/share/nginx/html;
+        proxy_http_version 1.1;
+
+        # comppression
+        gzip            on;
+        gzip_comp_level 9;
+        gzip_http_version 1.0;
+        gzip_min_length 512;
+        gzip_proxied    expired no-cache no-store private auth;
+        gzip_types  text/plain
+                    text/css
+                    application/json
+                    application/javascript
+                    application/x-javascript
+                    text/xml application/xml
+                    application/xml+rss
+                    text/javascript
+                    application/x-font-ttf
+                    font/woff2
+                    image/svg+xml
+                    image/x-icon;
+
+        # Load configuration files for the default server block.
+        include /etc/nginx/default.d/*.conf;
+
+        location / {
+            try_files $uri$args $uri$args/ $uri index.html /index.html;
+        }
+
+        location /version.json {
+            add_header Cache-Control 'no-cache';
+        }
+
+        location /api {
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header Host $host;
+            proxy_pass ${NGINX_APISERVER_ADDR};
+            rewrite /api/(.*) /$1  break;
+        }
+
+        location /files {
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header Host $host;
+            proxy_pass ${NGINX_FILESERVER_ADDR};
+            rewrite /files/(.*) /$1  break;
+        }
+
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }

docker/build/internal_files/entrypoint.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -e
+
+mkdir -p /var/log/clearml
+
+SERVER_TYPE=$1
+
+if (( $# < 1 )) ; then
+    echo "The server type was not stated. It should be either apiserver, webserver or fileserver."
+    sleep 60
+    exit 1
+
+elif [[ ${SERVER_TYPE} == "apiserver" ]]; then
+    cd /opt/clearml/
+    python3 -m apiserver.apierrors_generator
+
+    if [[ -n $CLEARML_USE_GUNICORN ]]; then
+      MAX_REQUESTS=
+      if [[ -n $CLEARML_GUNICORN_MAX_REQUESTS ]]; then
+        MAX_REQUESTS="--max-requests $CLEARML_GUNICORN_MAX_REQUESTS"
+        if [[ -n $CLEARML_GUNICORN_MAX_REQUESTS_JITTER ]]; then
+          MAX_REQUESTS="$MAX_REQUESTS --max-requests-jitter $CLEARML_GUNICORN_MAX_REQUESTS_JITTER"
+        fi
+      fi
+
+      export GUNICORN_CMD_ARGS=${CLEARML_GUNICORN_CMD_ARGS}
+
+      # Note: don't be tempted to "fix" $MAX_REQUESTS with "$MAX_REQUESTS" as this produces an empty arg which fucks up gunicorn
+      gunicorn \
+        -w "${CLEARML_GUNICORN_WORKERS:-8}" \
+        -t "${CLEARML_GUNICORN_TIMEOUT:-600}" --bind="${CLEARML_GUNICORN_BIND:-0.0.0.0:8008}" \
+        $MAX_REQUESTS apiserver.server:app
+    else
+        python3 -m apiserver.server
+    fi
+
+elif [[ ${SERVER_TYPE} == "webserver" ]]; then
+
+    if [[ "${USER_KEY}" != "" ]] || [[ "${USER_SECRET}" != "" ]] || [[ "${COMPANY_ID}" != "" ]]; then
+      cat << EOF > /usr/share/nginx/html/credentials.json
+{
+  "userKey": "${USER_KEY}",
+  "userSecret": "${USER_SECRET}",
+  "companyID": "${COMPANY_ID}"
+}
+EOF
+    fi
+
+    export NGINX_APISERVER_ADDR=${NGINX_APISERVER_ADDRESS:-http://apiserver:8008}
+    export NGINX_FILESERVER_ADDR=${NGINX_FILESERVER_ADDRESS:-http://fileserver:8081}
+
+    envsubst '${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/nginx.conf
+
+    #start the server
+    /usr/sbin/nginx -g "daemon off;"
+
+elif [[ ${SERVER_TYPE} == "fileserver" ]]; then
+    cd /opt/clearml/fileserver/
+    if [ "$FILESERVER_USE_GUNICORN" = true ] ; then
+      gunicorn -t 600 --bind=0.0.0.0:8081 fileserver:app
+    else
+      python3 fileserver.py
+    fi
+
+else
+    echo "Server type ${SERVER_TYPE} is invalid. Please choose either apiserver, webserver or fileserver."
+fi

docker/build/internal_files/final_image_preparation.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+yum update -y
+yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+yum install -y python36 python36-pip nginx gcc python3-devel gettext
+yum -y upgrade
+python3 -m pip install -r /opt/clearml/fileserver/requirements.txt
+python3 -m pip install -r /opt/clearml/apiserver/requirements.txt
+mkdir -p /opt/clearml/log
+mkdir -p /opt/clearml/config
+ln -s /dev/stdout /var/log/nginx/access.log
+ln -s /dev/stderr /var/log/nginx/error.log
+mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.orig
+mv /tmp/internal_files/clearml.conf.template /etc/nginx/clearml.conf.template
+yum clean all