From 5e4c495d6249231e108253dc1202bcbb0cffede3 Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Thu, 12 Aug 2021 13:38:26 +0300
Subject: [PATCH] Add support for naming docker containers. Use
agent.docker_container_name_format to configure the name format (disabled by
default) (issue clearml/#412) Add missing entries in docs/clearml.conf
---
.../backend_api/config/default/agent.conf | 6 ++++
clearml_agent/commands/worker.py | 29 +++++++++++++++++++
docs/clearml.conf | 18 ++++++++++++
3 files changed, 53 insertions(+)
diff --git a/clearml_agent/backend_api/config/default/agent.conf b/clearml_agent/backend_api/config/default/agent.conf
index fa37231..84fb5ad 100644
--- a/clearml_agent/backend_api/config/default/agent.conf
+++ b/clearml_agent/backend_api/config/default/agent.conf
@@ -197,4 +197,10 @@
enabled: true
extra_keys: []
}
+
+ # Name docker containers created by the daemon using the following string format (supported from Docker 0.6.5)
+ # Allowed variables are task_id, worker_id and rand_string (random lower-case letters string, up to 32 charaters)
+ # Note: resulting name must start with an alpha-numeric character and continue with a alpha-numeric characters,
+ # underscores (_), dots (.) and/or dashes (-)
+ #docker_container_name_format: "clearml-id-{task_id}-{rand_string:.8}"
}
diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py
index e452dca..f9060a0 100644
--- a/clearml_agent/commands/worker.py
+++ b/clearml_agent/commands/worker.py
@@ -5,10 +5,12 @@ import json
import logging
import os
import os.path
+import random
import re
import shlex
import shutil
import signal
+import string
import subprocess
import sys
import traceback
@@ -125,6 +127,9 @@ DOCKER_ROOT_CONF_FILE = "/root/clearml.conf"
DOCKER_DEFAULT_CONF_FILE = "/root/default_clearml.conf"
+sys_random = random.SystemRandom()
+
+
@attr.s
class LiteralScriptManager(object):
"""
@@ -687,6 +692,23 @@ class Worker(ServiceCommandSection):
if self._services_mode:
# if this is services mode, give the docker a unique worker id, as it will register itself.
docker_params["worker_id"] = worker_id
+
+ name_format = self._session.config.get('agent.docker_container_name_format', None)
+ if name_format:
+ try:
+ name = name_format.format(
+ task_id=re.sub(r'[^a-zA-Z0-9._-]', '-', task_id),
+ worker_id=re.sub(r'[^a-zA-Z0-9._-]', '-', worker_id),
+ rand_string="".join(sys_random.choice(string.ascii_lowercase) for _ in range(32))
+ )
+ except Exception as ex:
+ print("Warning: failed generating docker container name: {}".format(ex))
+ else:
+ if self._valid_docker_container_name(name):
+ docker_params["name"] = name
+ else:
+ print("Warning: generated docker container name is invalid: {}".format(name))
+
full_docker_cmd = self.docker_image_func(**docker_params)
# if we are using the default docker, update back the Task:
@@ -3119,6 +3141,7 @@ class Worker(ServiceCommandSection):
docker_bash_setup_script=None,
auth_token=None,
worker_tags=None,
+ name=None,
):
docker = 'docker'
@@ -3294,6 +3317,7 @@ class Worker(ServiceCommandSection):
' ; '
base_cmd += (
+ (['--name', name] if name else []) +
['-v', conf_file+':'+DOCKER_ROOT_CONF_FILE] +
(['-v', host_ssh_cache+':/root/.ssh'] if host_ssh_cache else []) +
(['-v', host_apt_cache+':/var/cache/apt/archives'] if host_apt_cache else []) +
@@ -3500,6 +3524,11 @@ class Worker(ServiceCommandSection):
return result
+ @staticmethod
+ def _valid_docker_container_name(name):
+ # type: (str) -> bool
+ return re.fullmatch(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$", name) is not None
+
if __name__ == "__main__":
pass
diff --git a/docs/clearml.conf b/docs/clearml.conf
index 05e0922..bf97d71 100644
--- a/docs/clearml.conf
+++ b/docs/clearml.conf
@@ -163,6 +163,24 @@ agent {
# it Should be detected automatically. Override with os environment CUDA_VERSION / CUDNN_VERSION
# cuda_version: 10.1
# cudnn_version: 7.6
+
+ # Hide docker environment variables containing secrets when printing out the docker command by replacing their
+ # values with "********". Turning this feature on will hide the following environment variables values:
+ # CLEARML_API_SECRET_KEY, CLEARML_AGENT_GIT_PASS, AWS_SECRET_ACCESS_KEY, AZURE_STORAGE_KEY
+ # To include more environment variables, add their keys to the "extra_keys" list. E.g. to make sure the value of
+ # your custom environment variable named MY_SPECIAL_PASSWORD will not show in the logs when included in the
+ # docker command, set:
+ # extra_keys: ["MY_SPECIAL_PASSWORD"]
+ hide_docker_command_env_vars {
+ enabled: true
+ extra_keys: []
+ }
+
+ # Name docker containers created by the daemon using the following string format (supported from Docker 0.6.5)
+ # Allowed variables are task_id, worker_id and rand_string (random lower-case letters string, up to 32 charaters)
+ # Note: resulting name must start with an alpha-numeric character and continue with a alpha-numeric characters,
+ # underscores (_), dots (.) and/or dashes (-)
+ #docker_container_name_format: "clearml-id-{task_id}-{rand_string:.8}"
}
sdk {