diff --git a/apiserver/config/default/secure.conf b/apiserver/config/default/secure.conf index 68fb18a..cb1dee6 100644 --- a/apiserver/config/default/secure.conf +++ b/apiserver/config/default/secure.conf @@ -30,8 +30,8 @@ } services_agent { role: "admin" - user_key: "P4BMJA7RK3TKBXGSY8OAA1FA8TOD11" - user_secret: "OjxF-nxfMMZ-pzFNQpLqHKBlca9OxmD8C-ZbQqTx5Ill1kwCVFj2CR2HQGjLlFGvYTc" + user_key: "" + user_secret: "" } tests { role: "user" diff --git a/apiserver/mongo/initialize/user.py b/apiserver/mongo/initialize/user.py index e7353d0..90ca85f 100644 --- a/apiserver/mongo/initialize/user.py +++ b/apiserver/mongo/initialize/user.py @@ -19,6 +19,9 @@ def _ensure_user_credentials( return if not (key and secret): + log.info(f"Resetting credentials for existing user {user.id} ({user.name})") + user.credentials = [] + user.save() return new_credentials = Credentials(key=key, secret=secret) @@ -89,6 +92,7 @@ def _ensure_backend_user(user_id: str, company_id: str, user_name: str): def ensure_fixed_user(user: FixedUser, log: Logger, emails: set): + # noinspection PyTypeChecker data = attr.asdict(user) data["id"] = user.user_id email = f"{user.user_id}@example.com" diff --git a/docker/docker-compose-win10.yml b/docker/docker-compose-win10.yml index 0589bfc..a789844 100644 --- a/docker/docker-compose-win10.yml +++ b/docker/docker-compose-win10.yml @@ -19,17 +19,18 @@ services: environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch CLEARML_ELASTIC_SERVICE_PORT: 9200 - CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD} CLEARML_MONGODB_SERVICE_HOST: mongo CLEARML_MONGODB_SERVICE_PORT: 27017 CLEARML_REDIS_SERVICE_HOST: redis CLEARML_REDIS_SERVICE_PORT: 6379 - CLEARML_SERVER_DEPLOYMENT_TYPE: ${CLEARML_SERVER_DEPLOYMENT_TYPE:-win10} + CLEARML_SERVER_DEPLOYMENT_TYPE: win10 CLEARML__apiserver__pre_populate__enabled: "true" CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" CLEARML__services__async_urls_delete__enabled: "true" CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-} + CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-} ports: - "8008:8008" networks: @@ -41,8 +42,6 @@ services: - backend container_name: clearml-elastic environment: - ES_JAVA_OPTS: -Xms2g -Xmx2g -Dlog4j2.formatMsgNoLookups=true - ELASTIC_PASSWORD: ${ELASTIC_PASSWORD} bootstrap.memory_lock: "true" cluster.name: clearml cluster.routing.allocation.node_initial_primaries_recoveries: "500" @@ -137,7 +136,6 @@ services: environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch CLEARML_ELASTIC_SERVICE_PORT: 9200 - CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD} CLEARML_MONGODB_SERVICE_HOST: mongo CLEARML_MONGODB_SERVICE_PORT: 27017 CLEARML_REDIS_SERVICE_HOST: redis diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 03ca702..970b47b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -19,17 +19,18 @@ services: environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch CLEARML_ELASTIC_SERVICE_PORT: 9200 - CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD} CLEARML_MONGODB_SERVICE_HOST: mongo CLEARML_MONGODB_SERVICE_PORT: 27017 CLEARML_REDIS_SERVICE_HOST: redis CLEARML_REDIS_SERVICE_PORT: 6379 - CLEARML_SERVER_DEPLOYMENT_TYPE: ${CLEARML_SERVER_DEPLOYMENT_TYPE:-linux} + CLEARML_SERVER_DEPLOYMENT_TYPE: linux CLEARML__apiserver__pre_populate__enabled: "true" CLEARML__apiserver__pre_populate__zip_files: "/opt/clearml/db-pre-populate" CLEARML__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" CLEARML__services__async_urls_delete__enabled: "true" CLEARML__services__async_urls_delete__fileserver__url_prefixes: "[${CLEARML_FILES_HOST:-}]" + CLEARML__secure__credentials__services_agent__user_key: ${CLEARML_AGENT_ACCESS_KEY:-} + CLEARML__secure__credentials__services_agent__user_secret: ${CLEARML_AGENT_SECRET_KEY:-} ports: - "8008:8008" networks: @@ -41,8 +42,6 @@ services: - backend container_name: clearml-elastic environment: - ES_JAVA_OPTS: -Xms2g -Xmx2g -Dlog4j2.formatMsgNoLookups=true - ELASTIC_PASSWORD: ${ELASTIC_PASSWORD} bootstrap.memory_lock: "true" cluster.name: clearml cluster.routing.allocation.node_initial_primaries_recoveries: "500" @@ -136,7 +135,6 @@ services: environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch CLEARML_ELASTIC_SERVICE_PORT: 9200 - CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD} CLEARML_MONGODB_SERVICE_HOST: mongo CLEARML_MONGODB_SERVICE_PORT: 27017 CLEARML_REDIS_SERVICE_HOST: redis @@ -167,8 +165,8 @@ services: CLEARML_WEB_HOST: ${CLEARML_WEB_HOST:-} CLEARML_API_HOST: http://apiserver:8008 CLEARML_FILES_HOST: ${CLEARML_FILES_HOST:-} - CLEARML_API_ACCESS_KEY: ${CLEARML_API_ACCESS_KEY:-} - CLEARML_API_SECRET_KEY: ${CLEARML_API_SECRET_KEY:-} + CLEARML_API_ACCESS_KEY: ${CLEARML_AGENT_ACCESS_KEY:-} + CLEARML_API_SECRET_KEY: ${CLEARML_AGENT_SECRET_KEY:-} CLEARML_AGENT_GIT_USER: ${CLEARML_AGENT_GIT_USER} CLEARML_AGENT_GIT_PASS: ${CLEARML_AGENT_GIT_PASS} CLEARML_AGENT_UPDATE_VERSION: ${CLEARML_AGENT_UPDATE_VERSION:->=0.17.0} diff --git a/docker/legacy/trains-server/docker-compose-win10.yml b/docker/legacy/trains-server/docker-compose-win10.yml deleted file mode 100644 index c779638..0000000 --- a/docker/legacy/trains-server/docker-compose-win10.yml +++ /dev/null @@ -1,116 +0,0 @@ -version: "3.6" -services: - - apiserver: - command: - - apiserver - container_name: trains-apiserver - image: allegroai/trains:latest - restart: unless-stopped - volumes: - - c:/opt/trains/logs:/var/log/trains - - c:/opt/trains/config:/opt/trains/config - depends_on: - - redis - - mongo - - elasticsearch - - fileserver - environment: - TRAINS_ELASTIC_SERVICE_HOST: elasticsearch - TRAINS_ELASTIC_SERVICE_PORT: 9200 - TRAINS_MONGODB_SERVICE_HOST: mongo - TRAINS_MONGODB_SERVICE_PORT: 27017 - TRAINS_REDIS_SERVICE_HOST: redis - TRAINS_REDIS_SERVICE_PORT: 6379 - TRAINS_SERVER_DEPLOYMENT_TYPE: ${TRAINS_SERVER_DEPLOYMENT_TYPE:-win10} - TRAINS__apiserver__mongo__pre_populate__enabled: "true" - TRAINS__apiserver__mongo__pre_populate__zip_file: "/opt/trains/db-pre-populate/export.zip" - ports: - - "8008:8008" - networks: - - backend - - elasticsearch: - networks: - - backend - container_name: trains-elastic - environment: - ES_JAVA_OPTS: -Xms2g -Xmx2g - bootstrap.memory_lock: "true" - cluster.name: trains - cluster.routing.allocation.node_initial_primaries_recoveries: "500" - cluster.routing.allocation.disk.watermark.low: 500mb - cluster.routing.allocation.disk.watermark.high: 500mb - cluster.routing.allocation.disk.watermark.flood_stage: 500mb - discovery.zen.minimum_master_nodes: "1" - discovery.type: "single-node" - http.compression_level: "7" - node.ingest: "true" - node.name: trains - reindex.remote.whitelist: '*.*' - xpack.monitoring.enabled: "false" - xpack.security.enabled: "false" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 - restart: unless-stopped - volumes: - - c:/opt/trains/data/elastic_7:/usr/share/elasticsearch/data - - fileserver: - networks: - - backend - command: - - fileserver - container_name: trains-fileserver - image: allegroai/trains:latest - restart: unless-stopped - volumes: - - c:/opt/trains/logs:/var/log/trains - - c:/opt/trains/data/fileserver:/mnt/fileserver - - c:/opt/trains/config:/opt/trains/config - - ports: - - "8081:8081" - - mongo: - networks: - - backend - container_name: trains-mongo - image: mongo:3.6.5 - restart: unless-stopped - command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200 - volumes: - - c:/opt/trains/data/mongo/db:/data/db - - c:/opt/trains/data/mongo/configdb:/data/configdb - - redis: - networks: - - backend - container_name: trains-redis - image: redis:5.0 - restart: unless-stopped - volumes: - - c:/opt/trains/data/redis:/data - - webserver: - command: - - webserver - container_name: trains-webserver - image: allegroai/trains:latest - restart: unless-stopped - volumes: - - c:/trains/logs:/var/log/trains - depends_on: - - apiserver - ports: - - "8080:80" - -networks: - backend: - driver: bridge diff --git a/docker/legacy/trains-server/docker-compose.yml b/docker/legacy/trains-server/docker-compose.yml deleted file mode 100644 index a309fe9..0000000 --- a/docker/legacy/trains-server/docker-compose.yml +++ /dev/null @@ -1,153 +0,0 @@ -version: "3.6" -services: - - apiserver: - command: - - apiserver - container_name: trains-apiserver - image: allegroai/clearml:latest - restart: unless-stopped - volumes: - - /opt/trains/logs:/var/log/trains - - /opt/trains/config:/opt/trains/config - - /opt/trains/data/fileserver:/mnt/fileserver - depends_on: - - redis - - mongo - - elasticsearch - - fileserver - environment: - TRAINS_ELASTIC_SERVICE_HOST: elasticsearch - TRAINS_ELASTIC_SERVICE_PORT: 9200 - TRAINS_MONGODB_SERVICE_HOST: mongo - TRAINS_MONGODB_SERVICE_PORT: 27017 - TRAINS_REDIS_SERVICE_HOST: redis - TRAINS_REDIS_SERVICE_PORT: 6379 - TRAINS_SERVER_DEPLOYMENT_TYPE: ${TRAINS_SERVER_DEPLOYMENT_TYPE:-linux} - TRAINS__apiserver__pre_populate__enabled: "true" - TRAINS__apiserver__pre_populate__zip_files: "/opt/trains/db-pre-populate" - TRAINS__apiserver__pre_populate__artifacts_path: "/mnt/fileserver" - ports: - - "8008:8008" - networks: - - backend - - frontend - - elasticsearch: - networks: - - backend - container_name: trains-elastic - environment: - ES_JAVA_OPTS: -Xms2g -Xmx2g - bootstrap.memory_lock: "true" - cluster.name: trains - cluster.routing.allocation.node_initial_primaries_recoveries: "500" - cluster.routing.allocation.disk.watermark.low: 500mb - cluster.routing.allocation.disk.watermark.high: 500mb - cluster.routing.allocation.disk.watermark.flood_stage: 500mb - discovery.zen.minimum_master_nodes: "1" - discovery.type: "single-node" - http.compression_level: "7" - node.ingest: "true" - node.name: trains - reindex.remote.whitelist: '*.*' - xpack.monitoring.enabled: "false" - xpack.security.enabled: "false" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 - restart: unless-stopped - volumes: - - /opt/trains/data/elastic_7:/usr/share/elasticsearch/data - - fileserver: - networks: - - backend - command: - - fileserver - container_name: trains-fileserver - image: allegroai/clearml:latest - restart: unless-stopped - volumes: - - /opt/trains/logs:/var/log/trains - - /opt/trains/data/fileserver:/mnt/fileserver - - /opt/trains/config:/opt/trains/config - ports: - - "8081:8081" - - mongo: - networks: - - backend - container_name: trains-mongo - image: mongo:3.6.5 - restart: unless-stopped - command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200 - volumes: - - /opt/trains/data/mongo/db:/data/db - - /opt/trains/data/mongo/configdb:/data/configdb - - redis: - networks: - - backend - container_name: trains-redis - image: redis:5.0 - restart: unless-stopped - volumes: - - /opt/trains/data/redis:/data - - webserver: - command: - - webserver - container_name: trains-webserver - image: allegroai/clearml:latest - restart: unless-stopped - depends_on: - - apiserver - ports: - - "8080:80" - networks: - - backend - - frontend - - agent-services: - networks: - - backend - container_name: trains-agent-services - image: allegroai/trains-agent-services:latest - restart: unless-stopped - privileged: true - environment: - TRAINS_HOST_IP: ${TRAINS_HOST_IP} - TRAINS_WEB_HOST: ${TRAINS_WEB_HOST:-} - TRAINS_API_HOST: http://apiserver:8008 - TRAINS_FILES_HOST: ${TRAINS_FILES_HOST:-} - TRAINS_API_ACCESS_KEY: ${TRAINS_API_ACCESS_KEY:-} - TRAINS_API_SECRET_KEY: ${TRAINS_API_SECRET_KEY:-} - TRAINS_AGENT_GIT_USER: ${TRAINS_AGENT_GIT_USER} - TRAINS_AGENT_GIT_PASS: ${TRAINS_AGENT_GIT_PASS} - TRAINS_AGENT_UPDATE_VERSION: ${TRAINS_AGENT_UPDATE_VERSION:->=0.15.0} - TRAINS_AGENT_DEFAULT_BASE_DOCKER: "ubuntu:18.04" - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} - AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-} - AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT:-} - AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY:-} - GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-} - TRAINS_WORKER_ID: "trains-services" - TRAINS_AGENT_DOCKER_HOST_MOUNT: "/opt/trains/agent:/root/.trains" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /opt/trains/agent:/root/.trains - depends_on: - - apiserver - -networks: - backend: - driver: bridge - frontend: - driver: bridge