ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-03-03 10:42:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add CLEARML_AGENT_DISABLE_SSH_MOUNT allowing disabling the auto .ssh mount into the docker

Browse Source
This commit is contained in:
allegroai 2021-06-02 13:16:58 +03:00
parent 7e90ebd5db
commit 8c56777125
2 changed files with 29 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
clearml_agent/commands/worker.py
View File

@ -50,6 +50,7 @@ from clearml_agent.definitions import (
ENV_AGENT_SECRET_KEY,
ENV_AWS_SECRET_KEY,
ENV_AZURE_ACCOUNT_KEY,
ENV_AGENT_DISABLE_SSH_MOUNT,
)
from clearml_agent.definitions import WORKING_REPOSITORY_DIR, PIP_EXTRA_INDICES
from clearml_agent.errors import APIError, CommandFailedError, Sigterm
@ -2725,8 +2726,11 @@ class Worker(ServiceCommandSection):
if temp_config.get("agent.venvs_cache.path", None):
temp_config.put("agent.venvs_cache.path", '/root/.clearml/venvs-cache')
self._host_ssh_cache = mkdtemp(prefix='clearml_agent.ssh.')
self._temp_cleanup_list.append(self._host_ssh_cache)
if ENV_AGENT_DISABLE_SSH_MOUNT.get():
self._host_ssh_cache = None
else:
self._host_ssh_cache = mkdtemp(prefix='clearml_agent.ssh.')
self._temp_cleanup_list.append(self._host_ssh_cache)
return temp_config, partial(self._get_docker_config_cmd, temp_config=temp_config)
@ -2748,24 +2752,31 @@ class Worker(ServiceCommandSection):
"agent.docker_pip_cache", '~/.clearml/pip-cache'))).expanduser().as_posix()
# make sure all folders are valid
Path(host_apt_cache).mkdir(parents=True, exist_ok=True)
Path(host_pip_cache).mkdir(parents=True, exist_ok=True)
Path(host_cache).mkdir(parents=True, exist_ok=True)
Path(host_pip_dl).mkdir(parents=True, exist_ok=True)
Path(host_vcs_cache).mkdir(parents=True, exist_ok=True)
Path(host_ssh_cache).mkdir(parents=True, exist_ok=True)
if host_apt_cache:
Path(host_apt_cache).mkdir(parents=True, exist_ok=True)
if host_pip_cache:
Path(host_pip_cache).mkdir(parents=True, exist_ok=True)
if host_cache:
Path(host_cache).mkdir(parents=True, exist_ok=True)
if host_pip_dl:
Path(host_pip_dl).mkdir(parents=True, exist_ok=True)
if host_vcs_cache:
Path(host_vcs_cache).mkdir(parents=True, exist_ok=True)
if host_ssh_cache:
Path(host_ssh_cache).mkdir(parents=True, exist_ok=True)
if host_venvs_cache:
Path(host_venvs_cache).mkdir(parents=True, exist_ok=True)
# copy the .ssh folder to a temp folder, to be mapped into docker
# noinspection PyBroadException
try:
if Path(host_ssh_cache).is_dir():
shutil.rmtree(host_ssh_cache, ignore_errors=True)
shutil.copytree(Path('~/.ssh').expanduser().as_posix(), host_ssh_cache)
except Exception:
host_ssh_cache = None
self.log.warning('Failed creating temporary copy of ~/.ssh for git credential')
if host_ssh_cache:
# copy the .ssh folder to a temp folder, to be mapped into docker
# noinspection PyBroadException
try:
if Path(host_ssh_cache).is_dir():
shutil.rmtree(host_ssh_cache, ignore_errors=True)
shutil.copytree(Path('~/.ssh').expanduser().as_posix(), host_ssh_cache)
except Exception:
host_ssh_cache = None
self.log.warning('Failed creating temporary copy of ~/.ssh for git credential')
# check if the .git credentials exist:
try:

1
clearml_agent/definitions.py
View File

@ -134,6 +134,7 @@ ENV_DOCKER_SKIP_GPUS_FLAG = EnvironmentConfig('CLEARML_DOCKER_SKIP_GPUS_FLAG', '
ENV_AGENT_GIT_USER = EnvironmentConfig('CLEARML_AGENT_GIT_USER', 'TRAINS_AGENT_GIT_USER')
ENV_AGENT_GIT_PASS = EnvironmentConfig('CLEARML_AGENT_GIT_PASS', 'TRAINS_AGENT_GIT_PASS')
ENV_AGENT_GIT_HOST = EnvironmentConfig('CLEARML_AGENT_GIT_HOST', 'TRAINS_AGENT_GIT_HOST')
ENV_AGENT_DISABLE_SSH_MOUNT = EnvironmentConfig('CLEARML_AGENT_DISABLE_SSH_MOUNT', type=bool)
ENV_TASK_EXECUTE_AS_USER = EnvironmentConfig('CLEARML_AGENT_EXEC_USER', 'TRAINS_AGENT_EXEC_USER')
ENV_TASK_EXTRA_PYTHON_PATH = EnvironmentConfig('CLEARML_AGENT_EXTRA_PYTHON_PATH', 'TRAINS_AGENT_EXTRA_PYTHON_PATH')
ENV_DOCKER_HOST_MOUNT = EnvironmentConfig('CLEARML_AGENT_K8S_HOST_MOUNT', 'CLEARML_AGENT_DOCKER_HOST_MOUNT',