diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index 9658a05..389bdb2 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -3820,6 +3820,60 @@ class Worker(ServiceCommandSection): pass return results + @staticmethod + def _resolve_docker_env_args(docker_args): + # type: (List[str]) -> List[str] + """ + Resolve -e / --env docker environment args matching $VAR or ${VAR} from the host environment + + :argument docker_args: List of docker argument strings (flags and values) + """ + non_list_args = ( + "rm", "read-only", "sig-proxy", "tty", "privileged", "publish-all", "interactive", "init", "help", "detach" + ) + non_list_args_single = ( + "t", "P", "i", "d", + ) + + # if no filtering, do nothing + if not docker_args: + return docker_args + + args = docker_args[:] + skip_arg = False + for i, cmd in enumerate(docker_args): + if skip_arg and not cmd.startswith("-"): + continue + + skip_arg = False + + if cmd.startswith("--"): + # jump over single command + if cmd[2:] in non_list_args: + continue + elif cmd.startswith("-"): + # jump over single character non args + if cmd[1:] in non_list_args_single: + continue + + # if we are here we have a command to bypass and the list after it + if cmd in ('-e', '--env'): + skip_arg = True + for j in range(i+1, len(args)): + if args[j].startswith("-"): + break + + parts = args[j].split("=", 1) + if len(parts) != 2: + continue + + args[j] = "{}={}".format(parts[0], os.path.expandvars(parts[1])) + + elif cmd.startswith("-"): + skip_arg = True + + return args + def _get_docker_cmd( self, worker_id, parent_worker_id, @@ -3883,9 +3937,14 @@ class Worker(ServiceCommandSection): docker_arguments = list(docker_arguments) \ if isinstance(docker_arguments, (list, tuple)) else [docker_arguments] docker_arguments = self._filter_docker_args(docker_arguments) + if self._session.config.get("agent.docker_allow_host_environ", None): + docker_arguments = self._resolve_docker_env_args(docker_arguments) base_cmd += [a for a in docker_arguments if a] if extra_docker_arguments: + # we always resolve environments in the `extra_docker_arguments` becuase the admin set them (not users) + extra_docker_arguments = self._resolve_docker_env_args(extra_docker_arguments) + extra_docker_arguments = [extra_docker_arguments] \ if isinstance(extra_docker_arguments, six.string_types) else extra_docker_arguments base_cmd += [str(a) for a in extra_docker_arguments if a] diff --git a/docs/clearml.conf b/docs/clearml.conf index 20749a2..c334ecb 100644 --- a/docs/clearml.conf +++ b/docs/clearml.conf @@ -185,6 +185,7 @@ agent { # optional arguments to pass to docker image # these are local for this agent and will not be updated in the experiment's docker_cmd section + # You can also pass host environments into the container with ["-e", "HOST_NAME=$HOST_NAME"] # extra_docker_arguments: ["--ipc=host", "-v", "/mnt/host/data:/mnt/data"] # optional shell script to run in docker when started before the experiment is started @@ -195,6 +196,12 @@ agent { # change to false to skip installation and decrease docker spin up time # docker_install_opencv_libs: true + # Allow passing host environments into docker container with Task's docker container args + # Example "-e HOST_NAME=$HOST_NAME" + # NOTICE this might introduce security risk allowing access to keys/secret on the host machine1 + # Use with care! + # docker_allow_host_environ: false + # set to true in order to force "docker pull" before running an experiment using a docker image. # This makes sure the docker image is updated. docker_force_pull: false