From c55b6f30dfcf845c7a39820d22347dec390e359a Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Mon, 18 Oct 2021 16:52:17 +0300 Subject: [PATCH] Add Dockerfile --- .gitignore | 1 - docker/build/Dockerfile | 35 +++++++ docker/build/internal_files/build_webapp.sh | 9 ++ .../internal_files/clearml.conf.template | 96 +++++++++++++++++++ docker/build/internal_files/entrypoint.sh | 67 +++++++++++++ .../internal_files/final_image_preparation.sh | 14 +++ 6 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 docker/build/Dockerfile create mode 100644 docker/build/internal_files/build_webapp.sh create mode 100644 docker/build/internal_files/clearml.conf.template create mode 100644 docker/build/internal_files/entrypoint.sh create mode 100644 docker/build/internal_files/final_image_preparation.sh diff --git a/.gitignore b/.gitignore index 2c0891f..0041156 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,6 @@ test-reports .pytest_cache venv *.noseids -build *.egg-info .cache .mypy_cache diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile new file mode 100644 index 0000000..2bb341c --- /dev/null +++ b/docker/build/Dockerfile @@ -0,0 +1,35 @@ +FROM centos/nodejs-12-centos7 AS webapp + +USER root +WORKDIR /opt + +RUN git clone https://github.com/allegroai/clearml-web.git +RUN mv clearml-web /opt/open-webapp +COPY docker/build/internal_files/build_webapp.sh /tmp/internal_files/ +RUN /bin/bash -c '/tmp/internal_files/build_webapp.sh' + +FROM centos:7 AS staging_image +COPY docker/build/internal_files/entrypoint.sh /opt/clearml/ +COPY fileserver /opt/clearml/fileserver/ +COPY apiserver /opt/clearml/apiserver/ + +FROM centos:7 +COPY --from=staging_image /opt/clearml/ /opt/clearml/ + +COPY docker/build/internal_files/final_image_preparation.sh /tmp/internal_files/ +COPY docker/build/internal_files/clearml.conf.template /tmp/internal_files/ +RUN /bin/bash -c '/tmp/internal_files/final_image_preparation.sh' + +COPY --from=webapp /opt/open-webapp/build /usr/share/nginx/html + +EXPOSE 8080 +EXPOSE 8008 +EXPOSE 8081 + +ARG VERSION +ARG BUILD +ENV CLEARML_SERVER_VERSION=${VERSION} +ENV CLEARML_SERVER_BUILD=${BUILD} + +WORKDIR /opt/clearml/ +ENTRYPOINT ["/opt/clearml/entrypoint.sh"] diff --git a/docker/build/internal_files/build_webapp.sh b/docker/build/internal_files/build_webapp.sh new file mode 100644 index 0000000..52a54c9 --- /dev/null +++ b/docker/build/internal_files/build_webapp.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -x +set -e + +cd /opt/open-webapp/ +npm ci --unsafe-perm node-sass + +cd /opt/open-webapp/ +npm run build diff --git a/docker/build/internal_files/clearml.conf.template b/docker/build/internal_files/clearml.conf.template new file mode 100644 index 0000000..4d7130a --- /dev/null +++ b/docker/build/internal_files/clearml.conf.template @@ -0,0 +1,96 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + root /usr/share/nginx/html; + proxy_http_version 1.1; + + # comppression + gzip on; + gzip_comp_level 9; + gzip_http_version 1.0; + gzip_min_length 512; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain + text/css + application/json + application/javascript + application/x-javascript + text/xml application/xml + application/xml+rss + text/javascript + application/x-font-ttf + font/woff2 + image/svg+xml + image/x-icon; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + try_files $uri$args $uri$args/ $uri index.html /index.html; + } + + location /version.json { + add_header Cache-Control 'no-cache'; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_pass ${NGINX_APISERVER_ADDR}; + rewrite /api/(.*) /$1 break; + } + + location /files { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_pass ${NGINX_FILESERVER_ADDR}; + rewrite /files/(.*) /$1 break; + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } diff --git a/docker/build/internal_files/entrypoint.sh b/docker/build/internal_files/entrypoint.sh new file mode 100644 index 0000000..060567f --- /dev/null +++ b/docker/build/internal_files/entrypoint.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -e + +mkdir -p /var/log/clearml + +SERVER_TYPE=$1 + +if (( $# < 1 )) ; then + echo "The server type was not stated. It should be either apiserver, webserver or fileserver." + sleep 60 + exit 1 + +elif [[ ${SERVER_TYPE} == "apiserver" ]]; then + cd /opt/clearml/ + python3 -m apiserver.apierrors_generator + + if [[ -n $CLEARML_USE_GUNICORN ]]; then + MAX_REQUESTS= + if [[ -n $CLEARML_GUNICORN_MAX_REQUESTS ]]; then + MAX_REQUESTS="--max-requests $CLEARML_GUNICORN_MAX_REQUESTS" + if [[ -n $CLEARML_GUNICORN_MAX_REQUESTS_JITTER ]]; then + MAX_REQUESTS="$MAX_REQUESTS --max-requests-jitter $CLEARML_GUNICORN_MAX_REQUESTS_JITTER" + fi + fi + + export GUNICORN_CMD_ARGS=${CLEARML_GUNICORN_CMD_ARGS} + + # Note: don't be tempted to "fix" $MAX_REQUESTS with "$MAX_REQUESTS" as this produces an empty arg which fucks up gunicorn + gunicorn \ + -w "${CLEARML_GUNICORN_WORKERS:-8}" \ + -t "${CLEARML_GUNICORN_TIMEOUT:-600}" --bind="${CLEARML_GUNICORN_BIND:-0.0.0.0:8008}" \ + $MAX_REQUESTS apiserver.server:app + else + python3 -m apiserver.server + fi + +elif [[ ${SERVER_TYPE} == "webserver" ]]; then + + if [[ "${USER_KEY}" != "" ]] || [[ "${USER_SECRET}" != "" ]] || [[ "${COMPANY_ID}" != "" ]]; then + cat << EOF > /usr/share/nginx/html/credentials.json +{ + "userKey": "${USER_KEY}", + "userSecret": "${USER_SECRET}", + "companyID": "${COMPANY_ID}" +} +EOF + fi + + export NGINX_APISERVER_ADDR=${NGINX_APISERVER_ADDRESS:-http://apiserver:8008} + export NGINX_FILESERVER_ADDR=${NGINX_FILESERVER_ADDRESS:-http://fileserver:8081} + + envsubst '${NGINX_APISERVER_ADDR} ${NGINX_FILESERVER_ADDR}' < /etc/nginx/clearml.conf.template > /etc/nginx/nginx.conf + + #start the server + /usr/sbin/nginx -g "daemon off;" + +elif [[ ${SERVER_TYPE} == "fileserver" ]]; then + cd /opt/clearml/fileserver/ + if [ "$FILESERVER_USE_GUNICORN" = true ] ; then + gunicorn -t 600 --bind=0.0.0.0:8081 fileserver:app + else + python3 fileserver.py + fi + +else + echo "Server type ${SERVER_TYPE} is invalid. Please choose either apiserver, webserver or fileserver." +fi diff --git a/docker/build/internal_files/final_image_preparation.sh b/docker/build/internal_files/final_image_preparation.sh new file mode 100644 index 0000000..207ecb2 --- /dev/null +++ b/docker/build/internal_files/final_image_preparation.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +yum update -y +yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +yum install -y python36 python36-pip nginx gcc python3-devel gettext +yum -y upgrade +python3 -m pip install -r /opt/clearml/fileserver/requirements.txt +python3 -m pip install -r /opt/clearml/apiserver/requirements.txt +mkdir -p /opt/clearml/log +mkdir -p /opt/clearml/config +ln -s /dev/stdout /var/log/nginx/access.log +ln -s /dev/stderr /var/log/nginx/error.log +mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.orig +mv /tmp/internal_files/clearml.conf.template /etc/nginx/clearml.conf.template +yum clean all \ No newline at end of file