diff --git a/clearml_agent/backend_api/config/default/agent.conf b/clearml_agent/backend_api/config/default/agent.conf index a66d675..fa37231 100644 --- a/clearml_agent/backend_api/config/default/agent.conf +++ b/clearml_agent/backend_api/config/default/agent.conf @@ -185,4 +185,16 @@ # should be detected automatically. Override with os environment CUDA_VERSION / CUDNN_VERSION # cuda_version: 10.1 # cudnn_version: 7.6 + + # Hide docker environment variables containing secrets when printing out the docker command by replacing their + # values with "********". Turning this feature on will hide the following environment variables values: + # CLEARML_API_SECRET_KEY, CLEARML_AGENT_GIT_PASS, AWS_SECRET_ACCESS_KEY, AZURE_STORAGE_KEY + # To include more environment variables, add their keys to the "extra_keys" list. E.g. to make sure the value of + # your custom environment variable named MY_SPECIAL_PASSWORD will not show in the logs when included in the + # docker command, set: + # extra_keys: ["MY_SPECIAL_PASSWORD"] + hide_docker_command_env_vars { + enabled: true + extra_keys: [] + } } diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index a93d900..8e58f14 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -20,7 +20,7 @@ from functools import partial, cmp_to_key from itertools import chain from tempfile import mkdtemp, NamedTemporaryFile from time import sleep, time -from typing import Text, Optional, Any, Tuple +from typing import Text, Optional, Any, Tuple, List import attr import psutil @@ -44,7 +44,13 @@ from clearml_agent.definitions import ( ENV_DOCKER_HOST_MOUNT, ENV_TASK_EXTRA_PYTHON_PATH, ENV_AGENT_GIT_USER, - ENV_AGENT_GIT_PASS, ENV_WORKER_ID, ENV_DOCKER_SKIP_GPUS_FLAG, ) + ENV_AGENT_GIT_PASS, + ENV_WORKER_ID, + ENV_DOCKER_SKIP_GPUS_FLAG, + ENV_AGENT_SECRET_KEY, + ENV_AWS_SECRET_KEY, + ENV_AZURE_ACCOUNT_KEY, +) from clearml_agent.definitions import WORKING_REPOSITORY_DIR, PIP_EXTRA_INDICES from clearml_agent.errors import APIError, CommandFailedError, Sigterm from clearml_agent.helper.base import ( @@ -68,7 +74,9 @@ from clearml_agent.helper.base import ( get_python_path, is_linux_platform, rm_file, - add_python_path, safe_remove_tree, ) + add_python_path, + safe_remove_tree, +) from clearml_agent.helper.console import ensure_text, print_text, decode_binary_lines from clearml_agent.helper.os.daemonize import daemonize_process from clearml_agent.helper.package.base import PackageManager @@ -91,7 +99,10 @@ from clearml_agent.helper.process import ( get_bash_output, shutdown_docker_process, get_docker_id, - commit_docker, terminate_process, check_if_command_exists, terminate_all_child_processes, + commit_docker, + terminate_process, + check_if_command_exists, + terminate_all_child_processes, ) from clearml_agent.helper.package.priority_req import PriorityPackageRequirement, PackageCollectorRequirement from clearml_agent.helper.repo import clone_repository_cached, RepoInfo, VCS, fix_package_import_diff_patch @@ -621,10 +632,13 @@ class Worker(ServiceCommandSection): '--standalone-mode' if self._standalone_mode else '', task_id) - # send the actual used command line to the backend - self.send_logs(task_id=task_id, lines=['Executing: {}\n'.format(full_docker_cmd)], level="INFO") + display_docker_command = self._sanitize_docker_command(full_docker_cmd) + + # send the actual used command line to the backend + self.send_logs(task_id=task_id, lines=['Executing: {}\n'.format(display_docker_command)], level="INFO") + + cmd = Argv(*full_docker_cmd, display_argv=display_docker_command) - cmd = Argv(*full_docker_cmd) print('Running Docker:\n{}\n'.format(str(cmd))) else: cmd = worker_args.get_argv_for_command("execute") + ( @@ -3161,6 +3175,33 @@ class Worker(ServiceCommandSection): queue_ids.append(q_id) return queue_ids + def _sanitize_docker_command(self, docker_command): + # type: (List[str]) -> List[str] + if not self._session.config.get('agent.hide_docker_command_env_vars.enabled', False): + return docker_command + + keys = set(self._session.config.get('agent.hide_docker_command_env_vars.extra_keys', [])) + keys.update( + ENV_AGENT_GIT_PASS.vars, + ENV_AGENT_SECRET_KEY.vars, + ENV_AWS_SECRET_KEY.vars, + ENV_AZURE_ACCOUNT_KEY.vars + ) + + result = docker_command[:] + for i, item in enumerate(docker_command): + try: + if item not in ("-e", "--env"): + continue + key, sep, _ = result[i + 1].partition("=") + if key not in keys or not sep: + continue + result[i + 1] = "{}={}".format(key, "********") + except KeyError: + pass + + return result + if __name__ == "__main__": pass diff --git a/clearml_agent/definitions.py b/clearml_agent/definitions.py index c71db5f..6aa0451 100644 --- a/clearml_agent/definitions.py +++ b/clearml_agent/definitions.py @@ -62,6 +62,10 @@ class EnvironmentConfig(object): return None +ENV_AGENT_SECRET_KEY = EnvironmentConfig("CLEARML_API_SECRET_KEY", "TRAINS_API_SECRET_KEY") +ENV_AWS_SECRET_KEY = EnvironmentConfig("AWS_SECRET_ACCESS_KEY") +ENV_AZURE_ACCOUNT_KEY = EnvironmentConfig("AZURE_STORAGE_KEY") + ENVIRONMENT_CONFIG = { "api.api_server": EnvironmentConfig("CLEARML_API_HOST", "TRAINS_API_HOST", ), "api.files_server": EnvironmentConfig("CLEARML_FILES_HOST", "TRAINS_FILES_HOST", ), @@ -69,9 +73,7 @@ ENVIRONMENT_CONFIG = { "api.credentials.access_key": EnvironmentConfig( "CLEARML_API_ACCESS_KEY", "TRAINS_API_ACCESS_KEY", ), - "api.credentials.secret_key": EnvironmentConfig( - "CLEARML_API_SECRET_KEY", "TRAINS_API_SECRET_KEY", - ), + "api.credentials.secret_key": ENV_AGENT_SECRET_KEY, "agent.worker_name": EnvironmentConfig("CLEARML_WORKER_NAME", "TRAINS_WORKER_NAME", ), "agent.worker_id": EnvironmentConfig("CLEARML_WORKER_ID", "TRAINS_WORKER_ID", ), "agent.cuda_version": EnvironmentConfig( @@ -84,10 +86,10 @@ ENVIRONMENT_CONFIG = { names=("CLEARML_CPU_ONLY", "TRAINS_CPU_ONLY", "CPU_ONLY"), type=bool ), "sdk.aws.s3.key": EnvironmentConfig("AWS_ACCESS_KEY_ID"), - "sdk.aws.s3.secret": EnvironmentConfig("AWS_SECRET_ACCESS_KEY"), + "sdk.aws.s3.secret": ENV_AWS_SECRET_KEY, "sdk.aws.s3.region": EnvironmentConfig("AWS_DEFAULT_REGION"), "sdk.azure.storage.containers.0": {'account_name': EnvironmentConfig("AZURE_STORAGE_ACCOUNT"), - 'account_key': EnvironmentConfig("AZURE_STORAGE_KEY")}, + 'account_key': ENV_AZURE_ACCOUNT_KEY}, "sdk.google.storage.credentials_json": EnvironmentConfig("GOOGLE_APPLICATION_CREDENTIALS"), } diff --git a/clearml_agent/helper/process.py b/clearml_agent/helper/process.py index c92c3cf..fc86785 100644 --- a/clearml_agent/helper/process.py +++ b/clearml_agent/helper/process.py @@ -221,6 +221,7 @@ class Argv(Executable): """ self.argv = argv self._log = kwargs.pop("log", None) + self._display_argv = kwargs.pop("display_argv", argv) if not self._log: self._log = logging.getLogger(__name__) self._log.propagate = False @@ -245,10 +246,10 @@ class Argv(Executable): return self.argv def __repr__(self): - return "".format(self.argv) + return "".format(self._display_argv) def __str__(self): - return "Executing: {}".format(self.argv) + return "Executing: {}".format(self._display_argv) def __iter__(self): if is_windows_platform():