Remove default credentials and reset existing credentials if none were provided

apiserver/config/default/secure.conf

@@ -30,8 +30,8 @@
         }
         services_agent {
             role: "admin"
-            user_key: "P4BMJA7RK3TKBXGSY8OAA1FA8TOD11"
-            user_secret: "OjxF-nxfMMZ-pzFNQpLqHKBlca9OxmD8C-ZbQqTx5Ill1kwCVFj2CR2HQGjLlFGvYTc"
+            user_key: ""
+            user_secret: ""
         }
         tests {
             role: "user"

apiserver/mongo/initialize/user.py

@@ -19,6 +19,9 @@ def _ensure_user_credentials(
         return
 
     if not (key and secret):
+        log.info(f"Resetting credentials for existing user {user.id} ({user.name})")
+        user.credentials = []
+        user.save()
         return
 
     new_credentials = Credentials(key=key, secret=secret)
@@ -89,6 +92,7 @@ def _ensure_backend_user(user_id: str, company_id: str, user_name: str):
 
 
 def ensure_fixed_user(user: FixedUser, log: Logger, emails: set):
+    # noinspection PyTypeChecker
     data = attr.asdict(user)
     data["id"] = user.user_id
     email = f"{user.user_id}@example.com"

docker/docker-compose-win10.yml

@@ -19,17 +19,18 @@ services:
     environment:
       CLEARML_ELASTIC_SERVICE_HOST: elasticsearch
       CLEARML_ELASTIC_SERVICE_PORT: 9200
-      CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD}
       CLEARML_MONGODB_SERVICE_HOST: mongo
       CLEARML_MONGODB_SERVICE_PORT: 27017
       CLEARML_REDIS_SERVICE_HOST: redis
       CLEARML_REDIS_SERVICE_PORT: 6379
-      CLEARML_SERVER_DEPLOYMENT_TYPE: ${CLEARML_SERVER_DEPLOYMENT_TYPE:-win10}
+      CLEARML_SERVER_DEPLOYMENT_TYPE: win10
       CLEARML__apiserver__pre_populate__enabled: "true"
       CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate"
       CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver"
       CLEARML__services__async_urls_delete__enabled: "true"
       CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]"
+      CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-}
+      CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-}
     ports:
     - "8008:8008"
     networks:
@@ -41,8 +42,6 @@ services:
       - backend
     container_name: clearml-elastic
     environment:
-      ES_JAVA_OPTS: -Xms2g -Xmx2g -Dlog4j2.formatMsgNoLookups=true
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
       bootstrap.memory_lock: "true"
       cluster.name: clearml
       cluster.routing.allocation.node_initial_primaries_recoveries: "500"
@@ -137,7 +136,6 @@ services:
     environment:
       CLEARML_ELASTIC_SERVICE_HOST: elasticsearch
       CLEARML_ELASTIC_SERVICE_PORT: 9200
-      CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD}
       CLEARML_MONGODB_SERVICE_HOST: mongo
       CLEARML_MONGODB_SERVICE_PORT: 27017
       CLEARML_REDIS_SERVICE_HOST: redis

docker/docker-compose.yml

@@ -19,17 +19,18 @@ services:
     environment:
       CLEARML_ELASTIC_SERVICE_HOST: elasticsearch
       CLEARML_ELASTIC_SERVICE_PORT: 9200
-      CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD}
       CLEARML_MONGODB_SERVICE_HOST: mongo
       CLEARML_MONGODB_SERVICE_PORT: 27017
       CLEARML_REDIS_SERVICE_HOST: redis
       CLEARML_REDIS_SERVICE_PORT: 6379
-      CLEARML_SERVER_DEPLOYMENT_TYPE: ${CLEARML_SERVER_DEPLOYMENT_TYPE:-linux}
+      CLEARML_SERVER_DEPLOYMENT_TYPE: linux
       CLEARML__apiserver__pre_populate__enabled: "true"
       CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate"
       CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver"
       CLEARML__services__async_urls_delete__enabled: "true"
       CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]"
+      CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-}
+      CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-}
     ports:
     - "8008:8008"
     networks:
@@ -41,8 +42,6 @@ services:
       - backend
     container_name: clearml-elastic
     environment:
-      ES_JAVA_OPTS: -Xms2g -Xmx2g -Dlog4j2.formatMsgNoLookups=true
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
       bootstrap.memory_lock: "true"
       cluster.name: clearml
       cluster.routing.allocation.node_initial_primaries_recoveries: "500"
@@ -136,7 +135,6 @@ services:
     environment:
       CLEARML_ELASTIC_SERVICE_HOST: elasticsearch
       CLEARML_ELASTIC_SERVICE_PORT: 9200
-      CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD}
       CLEARML_MONGODB_SERVICE_HOST: mongo
       CLEARML_MONGODB_SERVICE_PORT: 27017
       CLEARML_REDIS_SERVICE_HOST: redis
@@ -167,8 +165,8 @@ services:
       CLEARML_WEB_HOST: ${CLEARML_WEB_HOST:-}
       CLEARML_API_HOST: http://apiserver:8008
       CLEARML_FILES_HOST: ${CLEARML_FILES_HOST:-}
-      CLEARML_API_ACCESS_KEY: ${CLEARML_API_ACCESS_KEY:-}
-      CLEARML_API_SECRET_KEY: ${CLEARML_API_SECRET_KEY:-}
+      CLEARML_API_ACCESS_KEY: ${CLEARML_AGENT_ACCESS_KEY:-}
+      CLEARML_API_SECRET_KEY: ${CLEARML_AGENT_SECRET_KEY:-}
       CLEARML_AGENT_GIT_USER: ${CLEARML_AGENT_GIT_USER}
       CLEARML_AGENT_GIT_PASS: ${CLEARML_AGENT_GIT_PASS}
       CLEARML_AGENT_UPDATE_VERSION: ${CLEARML_AGENT_UPDATE_VERSION:->=0.17.0}

docker/legacy/trains-server/docker-compose-win10.yml

@@ -1,116 +0,0 @@
-version: "3.6"
-services:
-
-  apiserver:
-    command:
-    - apiserver
-    container_name: trains-apiserver
-    image: allegroai/trains:latest
-    restart: unless-stopped
-    volumes:
-    - c:/opt/trains/logs:/var/log/trains
-    - c:/opt/trains/config:/opt/trains/config
-    depends_on:
-      - redis
-      - mongo
-      - elasticsearch
-      - fileserver
-    environment:
-      TRAINS_ELASTIC_SERVICE_HOST: elasticsearch
-      TRAINS_ELASTIC_SERVICE_PORT: 9200
-      TRAINS_MONGODB_SERVICE_HOST: mongo
-      TRAINS_MONGODB_SERVICE_PORT: 27017
-      TRAINS_REDIS_SERVICE_HOST: redis
-      TRAINS_REDIS_SERVICE_PORT: 6379
-      TRAINS_SERVER_DEPLOYMENT_TYPE: ${TRAINS_SERVER_DEPLOYMENT_TYPE:-win10}
-      TRAINS__apiserver__mongo__pre_populate__enabled: "true"
-      TRAINS__apiserver__mongo__pre_populate__zip_file: "/opt/trains/db-pre-populate/export.zip"
-    ports:
-    - "8008:8008"
-    networks:
-      - backend
-
-  elasticsearch:
-    networks:
-      - backend
-    container_name: trains-elastic
-    environment:
-      ES_JAVA_OPTS: -Xms2g -Xmx2g
-      bootstrap.memory_lock: "true"
-      cluster.name: trains
-      cluster.routing.allocation.node_initial_primaries_recoveries: "500"
-      cluster.routing.allocation.disk.watermark.low: 500mb
-      cluster.routing.allocation.disk.watermark.high: 500mb
-      cluster.routing.allocation.disk.watermark.flood_stage: 500mb
-      discovery.zen.minimum_master_nodes: "1"
-      discovery.type: "single-node"
-      http.compression_level: "7"
-      node.ingest: "true"
-      node.name: trains
-      reindex.remote.whitelist: '*.*'
-      xpack.monitoring.enabled: "false"
-      xpack.security.enabled: "false"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-      nofile:
-        soft: 65536
-        hard: 65536
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
-    restart: unless-stopped
-    volumes:
-    - c:/opt/trains/data/elastic_7:/usr/share/elasticsearch/data
-
-  fileserver:
-    networks:
-      - backend
-    command:
-    - fileserver
-    container_name: trains-fileserver
-    image: allegroai/trains:latest
-    restart: unless-stopped
-    volumes:
-    - c:/opt/trains/logs:/var/log/trains
-    - c:/opt/trains/data/fileserver:/mnt/fileserver
-    - c:/opt/trains/config:/opt/trains/config
-
-    ports:
-    - "8081:8081"
-
-  mongo:
-    networks:
-      - backend
-    container_name: trains-mongo
-    image: mongo:3.6.5
-    restart: unless-stopped
-    command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200
-    volumes:
-    - c:/opt/trains/data/mongo/db:/data/db
-    - c:/opt/trains/data/mongo/configdb:/data/configdb
-
-  redis:
-    networks:
-      - backend
-    container_name: trains-redis
-    image: redis:5.0
-    restart: unless-stopped
-    volumes:
-    - c:/opt/trains/data/redis:/data
-
-  webserver:
-    command:
-    - webserver
-    container_name: trains-webserver
-    image: allegroai/trains:latest
-    restart: unless-stopped
-    volumes:
-    - c:/trains/logs:/var/log/trains
-    depends_on:
-      - apiserver
-    ports:
-    - "8080:80"
-
-networks:
-  backend:
-    driver: bridge

docker/legacy/trains-server/docker-compose.yml

@@ -1,153 +0,0 @@
-version: "3.6"
-services:
-
-  apiserver:
-    command:
-    - apiserver
-    container_name: trains-apiserver
-    image: allegroai/clearml:latest
-    restart: unless-stopped
-    volumes:
-    - /opt/trains/logs:/var/log/trains
-    - /opt/trains/config:/opt/trains/config
-    - /opt/trains/data/fileserver:/mnt/fileserver
-    depends_on:
-      - redis
-      - mongo
-      - elasticsearch
-      - fileserver
-    environment:
-      TRAINS_ELASTIC_SERVICE_HOST: elasticsearch
-      TRAINS_ELASTIC_SERVICE_PORT: 9200
-      TRAINS_MONGODB_SERVICE_HOST: mongo
-      TRAINS_MONGODB_SERVICE_PORT: 27017
-      TRAINS_REDIS_SERVICE_HOST: redis
-      TRAINS_REDIS_SERVICE_PORT: 6379
-      TRAINS_SERVER_DEPLOYMENT_TYPE: ${TRAINS_SERVER_DEPLOYMENT_TYPE:-linux}
-      TRAINS__apiserver__pre_populate__enabled: "true"
-      TRAINS__apiserver__pre_populate__zip_files: "/opt/trains/db-pre-populate"
-      TRAINS__apiserver__pre_populate__artifacts_path: "/mnt/fileserver"
-    ports:
-    - "8008:8008"
-    networks:
-      - backend
-      - frontend
-
-  elasticsearch:
-    networks:
-      - backend
-    container_name: trains-elastic
-    environment:
-      ES_JAVA_OPTS: -Xms2g -Xmx2g
-      bootstrap.memory_lock: "true"
-      cluster.name: trains
-      cluster.routing.allocation.node_initial_primaries_recoveries: "500"
-      cluster.routing.allocation.disk.watermark.low: 500mb
-      cluster.routing.allocation.disk.watermark.high: 500mb
-      cluster.routing.allocation.disk.watermark.flood_stage: 500mb
-      discovery.zen.minimum_master_nodes: "1"
-      discovery.type: "single-node"
-      http.compression_level: "7"
-      node.ingest: "true"
-      node.name: trains
-      reindex.remote.whitelist: '*.*'
-      xpack.monitoring.enabled: "false"
-      xpack.security.enabled: "false"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-      nofile:
-        soft: 65536
-        hard: 65536
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
-    restart: unless-stopped
-    volumes:
-    - /opt/trains/data/elastic_7:/usr/share/elasticsearch/data
-
-  fileserver:
-    networks:
-      - backend
-    command:
-    - fileserver
-    container_name: trains-fileserver
-    image: allegroai/clearml:latest
-    restart: unless-stopped
-    volumes:
-    - /opt/trains/logs:/var/log/trains
-    - /opt/trains/data/fileserver:/mnt/fileserver
-    - /opt/trains/config:/opt/trains/config
-    ports:
-    - "8081:8081"
-
-  mongo:
-    networks:
-      - backend
-    container_name: trains-mongo
-    image: mongo:3.6.5
-    restart: unless-stopped
-    command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200
-    volumes:
-    - /opt/trains/data/mongo/db:/data/db
-    - /opt/trains/data/mongo/configdb:/data/configdb
-
-  redis:
-    networks:
-      - backend
-    container_name: trains-redis
-    image: redis:5.0
-    restart: unless-stopped
-    volumes:
-    - /opt/trains/data/redis:/data
-
-  webserver:
-    command:
-    - webserver
-    container_name: trains-webserver
-    image: allegroai/clearml:latest
-    restart: unless-stopped
-    depends_on:
-      - apiserver
-    ports:
-    - "8080:80"
-    networks:
-      - backend
-      - frontend
-
-  agent-services:
-    networks:
-      - backend
-    container_name: trains-agent-services
-    image: allegroai/trains-agent-services:latest
-    restart: unless-stopped
-    privileged: true
-    environment:
-      TRAINS_HOST_IP: ${TRAINS_HOST_IP}
-      TRAINS_WEB_HOST: ${TRAINS_WEB_HOST:-}
-      TRAINS_API_HOST: http://apiserver:8008
-      TRAINS_FILES_HOST: ${TRAINS_FILES_HOST:-}
-      TRAINS_API_ACCESS_KEY: ${TRAINS_API_ACCESS_KEY:-}
-      TRAINS_API_SECRET_KEY: ${TRAINS_API_SECRET_KEY:-}
-      TRAINS_AGENT_GIT_USER: ${TRAINS_AGENT_GIT_USER}
-      TRAINS_AGENT_GIT_PASS: ${TRAINS_AGENT_GIT_PASS}
-      TRAINS_AGENT_UPDATE_VERSION: ${TRAINS_AGENT_UPDATE_VERSION:->=0.15.0}
-      TRAINS_AGENT_DEFAULT_BASE_DOCKER: "ubuntu:18.04"
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
-      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-}
-      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT:-}
-      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY:-}
-      GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-}
-      TRAINS_WORKER_ID: "trains-services"
-      TRAINS_AGENT_DOCKER_HOST_MOUNT: "/opt/trains/agent:/root/.trains"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /opt/trains/agent:/root/.trains
-    depends_on:
-      - apiserver
-
-networks:
-  backend:
-    driver: bridge
-  frontend:
-    driver: bridge