ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add CLEARML_AGENT_DISABLE_SSH_MOUNT allowing disabling the auto .ssh mount into the docker

Browse Source
This commit is contained in:
allegroai 2021-06-02 13:16:58 +03:00
parent 7e90ebd5db
commit 8c56777125
2 changed files with 29 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
clearml_agent/commands/worker.py
View File

@ -50,6 +50,7 @@ from clearml_agent.definitions import (
ENV_AGENT_SECRET_KEY, ENV_AGENT_SECRET_KEY,
ENV_AWS_SECRET_KEY, ENV_AWS_SECRET_KEY,
ENV_AZURE_ACCOUNT_KEY, ENV_AZURE_ACCOUNT_KEY,
ENV_AGENT_DISABLE_SSH_MOUNT,
) )
from clearml_agent.definitions import WORKING_REPOSITORY_DIR, PIP_EXTRA_INDICES from clearml_agent.definitions import WORKING_REPOSITORY_DIR, PIP_EXTRA_INDICES
from clearml_agent.errors import APIError, CommandFailedError, Sigterm from clearml_agent.errors import APIError, CommandFailedError, Sigterm
@ -2725,6 +2726,9 @@ class Worker(ServiceCommandSection):
if temp_config.get("agent.venvs_cache.path", None): if temp_config.get("agent.venvs_cache.path", None):
temp_config.put("agent.venvs_cache.path", '/root/.clearml/venvs-cache') temp_config.put("agent.venvs_cache.path", '/root/.clearml/venvs-cache')
if ENV_AGENT_DISABLE_SSH_MOUNT.get():
self._host_ssh_cache = None
else:
self._host_ssh_cache = mkdtemp(prefix='clearml_agent.ssh.') self._host_ssh_cache = mkdtemp(prefix='clearml_agent.ssh.')
self._temp_cleanup_list.append(self._host_ssh_cache) self._temp_cleanup_list.append(self._host_ssh_cache)
@ -2748,15 +2752,22 @@ class Worker(ServiceCommandSection):
"agent.docker_pip_cache", '~/.clearml/pip-cache'))).expanduser().as_posix() "agent.docker_pip_cache", '~/.clearml/pip-cache'))).expanduser().as_posix()
# make sure all folders are valid # make sure all folders are valid
if host_apt_cache:
Path(host_apt_cache).mkdir(parents=True, exist_ok=True) Path(host_apt_cache).mkdir(parents=True, exist_ok=True)
if host_pip_cache:
Path(host_pip_cache).mkdir(parents=True, exist_ok=True) Path(host_pip_cache).mkdir(parents=True, exist_ok=True)
if host_cache:
Path(host_cache).mkdir(parents=True, exist_ok=True) Path(host_cache).mkdir(parents=True, exist_ok=True)
if host_pip_dl:
Path(host_pip_dl).mkdir(parents=True, exist_ok=True) Path(host_pip_dl).mkdir(parents=True, exist_ok=True)
if host_vcs_cache:
Path(host_vcs_cache).mkdir(parents=True, exist_ok=True) Path(host_vcs_cache).mkdir(parents=True, exist_ok=True)
if host_ssh_cache:
Path(host_ssh_cache).mkdir(parents=True, exist_ok=True) Path(host_ssh_cache).mkdir(parents=True, exist_ok=True)
if host_venvs_cache: if host_venvs_cache:
Path(host_venvs_cache).mkdir(parents=True, exist_ok=True) Path(host_venvs_cache).mkdir(parents=True, exist_ok=True)
if host_ssh_cache:
# copy the .ssh folder to a temp folder, to be mapped into docker # copy the .ssh folder to a temp folder, to be mapped into docker
# noinspection PyBroadException # noinspection PyBroadException
try: try:

1
clearml_agent/definitions.py
View File

@ -134,6 +134,7 @@ ENV_DOCKER_SKIP_GPUS_FLAG = EnvironmentConfig('CLEARML_DOCKER_SKIP_GPUS_FLAG', '
ENV_AGENT_GIT_USER = EnvironmentConfig('CLEARML_AGENT_GIT_USER', 'TRAINS_AGENT_GIT_USER') ENV_AGENT_GIT_USER = EnvironmentConfig('CLEARML_AGENT_GIT_USER', 'TRAINS_AGENT_GIT_USER')
ENV_AGENT_GIT_PASS = EnvironmentConfig('CLEARML_AGENT_GIT_PASS', 'TRAINS_AGENT_GIT_PASS') ENV_AGENT_GIT_PASS = EnvironmentConfig('CLEARML_AGENT_GIT_PASS', 'TRAINS_AGENT_GIT_PASS')
ENV_AGENT_GIT_HOST = EnvironmentConfig('CLEARML_AGENT_GIT_HOST', 'TRAINS_AGENT_GIT_HOST') ENV_AGENT_GIT_HOST = EnvironmentConfig('CLEARML_AGENT_GIT_HOST', 'TRAINS_AGENT_GIT_HOST')
ENV_AGENT_DISABLE_SSH_MOUNT = EnvironmentConfig('CLEARML_AGENT_DISABLE_SSH_MOUNT', type=bool)
ENV_TASK_EXECUTE_AS_USER = EnvironmentConfig('CLEARML_AGENT_EXEC_USER', 'TRAINS_AGENT_EXEC_USER') ENV_TASK_EXECUTE_AS_USER = EnvironmentConfig('CLEARML_AGENT_EXEC_USER', 'TRAINS_AGENT_EXEC_USER')
ENV_TASK_EXTRA_PYTHON_PATH = EnvironmentConfig('CLEARML_AGENT_EXTRA_PYTHON_PATH', 'TRAINS_AGENT_EXTRA_PYTHON_PATH') ENV_TASK_EXTRA_PYTHON_PATH = EnvironmentConfig('CLEARML_AGENT_EXTRA_PYTHON_PATH', 'TRAINS_AGENT_EXTRA_PYTHON_PATH')
ENV_DOCKER_HOST_MOUNT = EnvironmentConfig('CLEARML_AGENT_K8S_HOST_MOUNT', 'CLEARML_AGENT_DOCKER_HOST_MOUNT', ENV_DOCKER_HOST_MOUNT = EnvironmentConfig('CLEARML_AGENT_K8S_HOST_MOUNT', 'CLEARML_AGENT_DOCKER_HOST_MOUNT',