Add support for naming docker containers. Use agent.docker_container_name_format to configure the name format (disabled by default) (issue clearml/#412)

Add missing entries in docs/clearml.conf

clearml_agent/backend_api/config/default/agent.conf

@@ -197,4 +197,10 @@
         enabled: true
         extra_keys: []
     }
+
+    # Name docker containers created by the daemon using the following string format (supported from Docker 0.6.5)
+    # Allowed variables are task_id, worker_id and rand_string (random lower-case letters string, up to 32 charaters)
+    # Note: resulting name must start with an alpha-numeric character and continue with a alpha-numeric characters,
+    #  underscores (_), dots (.) and/or dashes (-)
+    #docker_container_name_format: "clearml-id-{task_id}-{rand_string:.8}"
 }

clearml_agent/commands/worker.py

@@ -5,10 +5,12 @@ import json
 import logging
 import os
 import os.path
+import random
 import re
 import shlex
 import shutil
 import signal
+import string
 import subprocess
 import sys
 import traceback
@@ -125,6 +127,9 @@ DOCKER_ROOT_CONF_FILE = "/root/clearml.conf"
 DOCKER_DEFAULT_CONF_FILE = "/root/default_clearml.conf"
 
 
+sys_random = random.SystemRandom()
+
+
 @attr.s
 class LiteralScriptManager(object):
     """
@@ -687,6 +692,23 @@ class Worker(ServiceCommandSection):
             if self._services_mode:
                 # if this is services mode, give the docker a unique worker id, as it will register itself.
                 docker_params["worker_id"] = worker_id
+
+            name_format = self._session.config.get('agent.docker_container_name_format', None)
+            if name_format:
+                try:
+                    name = name_format.format(
+                        task_id=re.sub(r'[^a-zA-Z0-9._-]', '-', task_id),
+                        worker_id=re.sub(r'[^a-zA-Z0-9._-]', '-', worker_id),
+                        rand_string="".join(sys_random.choice(string.ascii_lowercase) for _ in range(32))
+                    )
+                except Exception as ex:
+                    print("Warning: failed generating docker container name: {}".format(ex))
+                else:
+                    if self._valid_docker_container_name(name):
+                        docker_params["name"] = name
+                    else:
+                        print("Warning: generated docker container name is invalid: {}".format(name))
+
             full_docker_cmd = self.docker_image_func(**docker_params)
 
             # if we are using the default docker, update back the Task:
@@ -3119,6 +3141,7 @@ class Worker(ServiceCommandSection):
             docker_bash_setup_script=None,
             auth_token=None,
             worker_tags=None,
+            name=None,
     ):
         docker = 'docker'
 
@@ -3294,6 +3317,7 @@ class Worker(ServiceCommandSection):
                 ' ; '
 
         base_cmd += (
+            (['--name', name] if name else []) +
             ['-v', conf_file+':'+DOCKER_ROOT_CONF_FILE] +
             (['-v', host_ssh_cache+':/root/.ssh'] if host_ssh_cache else []) +
             (['-v', host_apt_cache+':/var/cache/apt/archives'] if host_apt_cache else []) +
@@ -3500,6 +3524,11 @@ class Worker(ServiceCommandSection):
 
         return result
 
+    @staticmethod
+    def _valid_docker_container_name(name):
+        # type: (str) -> bool
+        return re.fullmatch(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$", name) is not None
+
 
 if __name__ == "__main__":
     pass

docs/clearml.conf

@@ -163,6 +163,24 @@ agent {
     # it Should be detected automatically. Override with os environment CUDA_VERSION / CUDNN_VERSION
     # cuda_version: 10.1
     # cudnn_version: 7.6
+
+    # Hide docker environment variables containing secrets when printing out the docker command by replacing their
+    # values with "********". Turning this feature on will hide the following environment variables values:
+    #   CLEARML_API_SECRET_KEY, CLEARML_AGENT_GIT_PASS, AWS_SECRET_ACCESS_KEY, AZURE_STORAGE_KEY
+    # To include more environment variables, add their keys to the "extra_keys" list. E.g. to make sure the value of
+    # your custom environment variable named MY_SPECIAL_PASSWORD will not show in the logs when included in the
+    # docker command, set:
+    #   extra_keys: ["MY_SPECIAL_PASSWORD"]
+    hide_docker_command_env_vars {
+        enabled: true
+        extra_keys: []
+    }
+
+    # Name docker containers created by the daemon using the following string format (supported from Docker 0.6.5)
+    # Allowed variables are task_id, worker_id and rand_string (random lower-case letters string, up to 32 charaters)
+    # Note: resulting name must start with an alpha-numeric character and continue with a alpha-numeric characters,
+    #  underscores (_), dots (.) and/or dashes (-)
+    #docker_container_name_format: "clearml-id-{task_id}-{rand_string:.8}"
 }
 
 sdk {