ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move configuration sanitization settings to the default config file

Browse Source
This commit is contained in:
allegroai 2024-02-29 13:51:40 +02:00
parent 6fbd70786e
commit 85049d8705
2 changed files with 38 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
clearml_agent/backend_api/config/default/agent.conf
View File

@ -249,6 +249,20 @@
# cuda_version: 10.1
# cudnn_version: 7.6
# Sanitize configuration printout using these settings
sanitize_config_printout {
# Hide values of configuration keys matching these regexps
hide_secrets: ["^sanitize_config_printout$", "secret", "pass", "token", "account_key", "contents"]
# As above, only show field's value keys if value is a dictionary
hide_secrets_recursive: ["^environment$"]
# Do not hide for keys matching these regexps
dont_hide_secrets: ["^enable_git_ask_pass$"]
# Hide secrets in docker commands, according to the 'agent.hide_docker_command_env_vars' settings
docker_commands: ["^extra_docker_arguments$"]
# Hide password in URLs found in keys matching these regexps (handles single URLs, lists and dictionaries)
urls: ["^extra_index_url$"]
}
# Hide docker environment variables containing secrets when printing out the docker command by replacing their
# values with "********". Turning this feature on will hide the following environment variables values:
# CLEARML_API_SECRET_KEY, CLEARML_AGENT_GIT_PASS, AWS_SECRET_ACCESS_KEY, AZURE_STORAGE_KEY

33
clearml_agent/session.py
View File

@ -4,6 +4,7 @@ import json
import logging
import os
import platform
import re
import sys
from copy import deepcopy
from typing import Any, Callable
@ -240,28 +241,28 @@ class Session(_Session):
except:
pass
def print_configuration(
self,
remove_secret_keys=("secret", "pass", "token", "account_key", "contents"),
remove_secret_keys_exceptions=("enable_git_ask_pass",),
skip_value_keys=("environment",),
docker_args_sanitize_keys=("extra_docker_arguments", ),
sanitize_urls_keys=("extra_index_url", ),
):
def print_configuration(self):
def load_config(key, default):
return [re.compile(x) for x in self.config.get(f"agent.sanitize_config_printout.{key}", default=default)]
dont_hide_secret_keys = load_config("dont_hide_secrets", ("^enable_git_ask_pass$",))
hide_secret_keys = load_config("hide_secrets", ("secret", "pass", "token", "account_key", "contents"))
hide_secret_section_keys = load_config("hide_secrets_recursive", ("^environment$",))
docker_cmd_keys = load_config("docker_commands", ("^extra_docker_arguments$",))
urls_keys = load_config("urls", ("^extra_index_url$",))
# remove all the secrets from the print
def recursive_remove_secrets(dictionary):
for k in list(dictionary):
for s in remove_secret_keys:
if s in k and k not in remove_secret_keys_exceptions:
if not any(r.search(k) for r in dont_hide_secret_keys):
if any(r.search(k) for r in hide_secret_keys):
dictionary[k] = '****'
break
for s in skip_value_keys:
if s == k:
if any(r.search(k) for r in hide_secret_section_keys):
dictionary[k] = {key: '****' for key in dictionary[k]} \
if isinstance(dictionary[k], dict) else '****'
break
for s in sanitize_urls_keys:
if s == k:
if any(r.search(k) for r in urls_keys):
value = dictionary.get(k, None)
if isinstance(value, str):
dictionary[k] = sanitize_urls(value)[0]
@ -272,7 +273,7 @@ class Session(_Session):
if isinstance(dictionary.get(k, None), dict):
recursive_remove_secrets(dictionary[k])
elif isinstance(dictionary.get(k, None), (list, tuple)):
if k in (docker_args_sanitize_keys or []):
if any(r.match(k) for r in docker_cmd_keys):
dictionary[k] = DockerArgsSanitizer.sanitize_docker_command(self, dictionary[k])
for item in dictionary[k]:
if isinstance(item, dict):
@ -281,7 +282,7 @@ class Session(_Session):
config = deepcopy(self.config.to_dict())
# remove the env variable, it's not important
config.pop('env', None)
if remove_secret_keys or skip_value_keys or docker_args_sanitize_keys or sanitize_urls_keys:
if hide_secret_keys or hide_secret_section_keys or docker_cmd_keys or urls_keys:
recursive_remove_secrets(config)
# remove logging.loggers.urllib3.level from the print
try: