diff --git a/upgrade/1_17_to_2_0/docker-compose-win10.yml b/upgrade/1_17_to_2_0/docker-compose-win10.yml new file mode 100644 index 0000000..f578182 --- /dev/null +++ b/upgrade/1_17_to_2_0/docker-compose-win10.yml @@ -0,0 +1,158 @@ +version: "3.6" +services: + + apiserver: + command: + - apiserver + container_name: clearml-apiserver + image: allegroai/clearml:1.17.1-554 + restart: unless-stopped + volumes: + - c:/opt/clearml/logs:/var/log/clearml + - c:/opt/clearml/config:/opt/clearml/config + - c:/opt/clearml/data/fileserver:/mnt/fileserver + depends_on: + - redis + - mongo + - elasticsearch + - fileserver + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + CLEARML_SERVER_DEPLOYMENT_TYPE: win10 + CLEARML__apiserver__pre_populate__enabled: "true" + CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" + CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" + CLEARML__services__async_urls_delete__enabled: "true" + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + ports: + - "8008:8008" + networks: + - backend + - frontend + + elasticsearch: + networks: + - backend + container_name: clearml-elastic + environment: + bootstrap.memory_lock: "true" + cluster.name: clearml + cluster.routing.allocation.node_initial_primaries_recoveries: "500" + cluster.routing.allocation.disk.watermark.low: 500mb + cluster.routing.allocation.disk.watermark.high: 500mb + cluster.routing.allocation.disk.watermark.flood_stage: 500mb + discovery.type: "single-node" + http.compression_level: "7" + node.name: clearml + reindex.remote.whitelist: "'*.*'" + xpack.security.enabled: "false" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3 + restart: unless-stopped + volumes: + - c:/opt/clearml/data/elastic_7:/usr/share/elasticsearch/data + - /usr/share/elasticsearch/logs + + fileserver: + networks: + - backend + - frontend + command: + - fileserver + container_name: clearml-fileserver + image: allegroai/clearml:1.17.1-554 + environment: + CLEARML__fileserver__delete__allow_batch: "true" + restart: unless-stopped + volumes: + - c:/opt/clearml/logs:/var/log/clearml + - c:/opt/clearml/data/fileserver:/mnt/fileserver + - c:/opt/clearml/config:/opt/clearml/config + + ports: + - "8081:8081" + + mongo: + networks: + - backend + container_name: clearml-mongo + image: mongo:5.0.26 + restart: unless-stopped + command: --setParameter internalQueryMaxBlockingSortMemoryUsageBytes=196100200 + volumes: + - c:/opt/clearml/data/mongo_4/db:/data/db + - c:/opt/clearml/data/mongo_4/configdb:/data/configdb + + redis: + networks: + - backend + container_name: clearml-redis + image: redis:6.2 + restart: unless-stopped + volumes: + - c:/opt/clearml/data/redis:/data + + webserver: + command: + - webserver + container_name: clearml-webserver + image: allegroai/clearml:1.17.1-554 + restart: unless-stopped + volumes: + - c:/clearml/logs:/var/log/clearml + depends_on: + - apiserver + ports: + - "8080:80" + networks: + - backend + - frontend + + async_delete: + depends_on: + - apiserver + - redis + - mongo + - elasticsearch + - fileserver + container_name: async_delete + image: allegroai/clearml:1.17.1-554 + networks: + - backend + restart: unless-stopped + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + PYTHONPATH: /opt/clearml/apiserver + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + entrypoint: + - python3 + - -m + - jobs.async_urls_delete + - --fileserver-host + - http://fileserver:8081 + volumes: + - c:/opt/clearml/logs:/var/log/clearml + - c:/opt/clearml/config:/opt/clearml/config + +networks: + backend: + driver: bridge + frontend: + name: frontend + driver: bridge \ No newline at end of file diff --git a/upgrade/1_17_to_2_0/docker-compose.yml b/upgrade/1_17_to_2_0/docker-compose.yml new file mode 100644 index 0000000..4a7684a --- /dev/null +++ b/upgrade/1_17_to_2_0/docker-compose.yml @@ -0,0 +1,195 @@ +version: "3.6" +services: + + apiserver: + command: + - apiserver + container_name: clearml-apiserver + image: allegroai/clearml:1.17.1-554 + restart: unless-stopped + volumes: + - /opt/clearml/logs:/var/log/clearml + - /opt/clearml/config:/opt/clearml/config + - /opt/clearml/data/fileserver:/mnt/fileserver + depends_on: + - redis + - mongo + - elasticsearch + - fileserver + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + CLEARML_SERVER_DEPLOYMENT_TYPE: linux + CLEARML__apiserver__pre_populate__enabled: "true" + CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" + CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" + CLEARML__services__async_urls_delete__enabled: "true" + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-} + CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-} + ports: + - "8008:8008" + networks: + - backend + - frontend + + elasticsearch: + networks: + - backend + container_name: clearml-elastic + environment: + bootstrap.memory_lock: "true" + cluster.name: clearml + cluster.routing.allocation.node_initial_primaries_recoveries: "500" + cluster.routing.allocation.disk.watermark.low: 500mb + cluster.routing.allocation.disk.watermark.high: 500mb + cluster.routing.allocation.disk.watermark.flood_stage: 500mb + discovery.type: "single-node" + http.compression_level: "7" + node.name: clearml + reindex.remote.whitelist: "'*.*'" + xpack.security.enabled: "false" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3 + restart: unless-stopped + volumes: + - /opt/clearml/data/elastic_7:/usr/share/elasticsearch/data + - /usr/share/elasticsearch/logs + + fileserver: + networks: + - backend + - frontend + command: + - fileserver + container_name: clearml-fileserver + image: allegroai/clearml:1.17.1-554 + environment: + CLEARML__fileserver__delete__allow_batch: "true" + restart: unless-stopped + volumes: + - /opt/clearml/logs:/var/log/clearml + - /opt/clearml/data/fileserver:/mnt/fileserver + - /opt/clearml/config:/opt/clearml/config + ports: + - "8081:8081" + + mongo: + networks: + - backend + container_name: clearml-mongo + image: mongo:5.0.26 + restart: unless-stopped + command: --setParameter internalQueryMaxBlockingSortMemoryUsageBytes=196100200 + volumes: + - /opt/clearml/data/mongo_4/db:/data/db + - /opt/clearml/data/mongo_4/configdb:/data/configdb + + redis: + networks: + - backend + container_name: clearml-redis + image: redis:6.2 + restart: unless-stopped + volumes: + - /opt/clearml/data/redis:/data + + webserver: + command: + - webserver + container_name: clearml-webserver + # environment: + # CLEARML_SERVER_SUB_PATH : clearml-web # Allow Clearml to be served with a URL path prefix. + image: allegroai/clearml:1.17.1-554 + restart: unless-stopped + depends_on: + - apiserver + ports: + - "8080:80" + networks: + - backend + - frontend + + async_delete: + depends_on: + - apiserver + - redis + - mongo + - elasticsearch + - fileserver + container_name: async_delete + image: allegroai/clearml:1.17.1-554 + networks: + - backend + restart: unless-stopped + environment: + CLEARML_ELASTIC_SERVICE_HOST: elasticsearch + CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_MONGODB_SERVICE_HOST: mongo + CLEARML_MONGODB_SERVICE_PORT: 27017 + CLEARML_REDIS_SERVICE_HOST: redis + CLEARML_REDIS_SERVICE_PORT: 6379 + PYTHONPATH: /opt/clearml/apiserver + CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + entrypoint: + - python3 + - -m + - jobs.async_urls_delete + - --fileserver-host + - http://fileserver:8081 + volumes: + - /opt/clearml/logs:/var/log/clearml + - /opt/clearml/config:/opt/clearml/config + + agent-services: + networks: + - backend + container_name: clearml-agent-services + image: allegroai/clearml-agent-services:latest + deploy: + restart_policy: + condition: on-failure + privileged: true + environment: + CLEARML_HOST_IP: ${CLEARML_HOST_IP} + CLEARML_WEB_HOST: ${CLEARML_WEB_HOST:-} + CLEARML_API_HOST: http://apiserver:8008 + CLEARML_FILES_HOST: ${CLEARML_FILES_HOST:-} + CLEARML_API_ACCESS_KEY: ${CLEARML_AGENT_ACCESS_KEY:-$CLEARML_API_ACCESS_KEY} + CLEARML_API_SECRET_KEY: ${CLEARML_AGENT_SECRET_KEY:-$CLEARML_API_SECRET_KEY} + CLEARML_AGENT_GIT_USER: ${CLEARML_AGENT_GIT_USER} + CLEARML_AGENT_GIT_PASS: ${CLEARML_AGENT_GIT_PASS} + CLEARML_AGENT_UPDATE_VERSION: ${CLEARML_AGENT_UPDATE_VERSION:->=0.17.0} + CLEARML_AGENT_DEFAULT_BASE_DOCKER: "ubuntu:18.04" + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-} + AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT:-} + AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY:-} + GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-} + CLEARML_WORKER_ID: "clearml-services" + CLEARML_AGENT_DOCKER_HOST_MOUNT: "/opt/clearml/agent:/root/.clearml" + SHUTDOWN_IF_NO_ACCESS_KEY: 1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /opt/clearml/agent:/root/.clearml + depends_on: + - apiserver + entrypoint: > + bash -c "curl --retry 10 --retry-delay 10 --retry-connrefused 'http://apiserver:8008/debug.ping' && /usr/agent/entrypoint.sh" + +networks: + backend: + driver: bridge + frontend: + driver: bridge