ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add venvs_cache.path mount override for non-root containers (use: agent.docker_internal_mounts.venvs_cache)

Browse Source
This commit is contained in:
allegroai 2024-07-24 17:59:18 +03:00
parent f89a92556f
commit 0bb267115b
4 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
clearml_agent/backend_api/config/default/agent.conf
View File

@ -359,6 +359,7 @@
pip_cache: "/root/.cache/pip" pip_cache: "/root/.cache/pip"
poetry_cache: "/root/.cache/pypoetry" poetry_cache: "/root/.cache/pypoetry"
vcs_cache: "/root/.clearml/vcs-cache" vcs_cache: "/root/.clearml/vcs-cache"
venvs_cache: "/root/.clearml/venvs-cache"
venv_build: "~/.clearml/venvs-builds" venv_build: "~/.clearml/venvs-builds"
pip_download: "/root/.clearml/pip-download-cache" pip_download: "/root/.clearml/pip-download-cache"
} }

12
clearml_agent/commands/worker.py
View File

@ -3328,7 +3328,6 @@ class Worker(ServiceCommandSection):
if not self.poetry.enabled: if not self.poetry.enabled:
# add to cache # add to cache
if add_venv_folder_cache and not self._standalone_mode: if add_venv_folder_cache and not self._standalone_mode:
print('Adding venv into cache: {}'.format(add_venv_folder_cache))
self.package_api.add_cached_venv( self.package_api.add_cached_venv(
requirements=[freeze, previous_reqs], requirements=[freeze, previous_reqs],
docker_cmd=execution_info.docker_cmd if execution_info else None, docker_cmd=execution_info.docker_cmd if execution_info else None,
@ -4036,8 +4035,15 @@ class Worker(ServiceCommandSection):
if force_system_site_packages: if force_system_site_packages:
temp_config.put("agent.package_manager.system_site_packages", True) temp_config.put("agent.package_manager.system_site_packages", True)
# if venvs_cache is NOT disabled,
# we have to update the mount path and the path inside the container
if temp_config.get("agent.venvs_cache.path", None): if temp_config.get("agent.venvs_cache.path", None):
temp_config.put("agent.venvs_cache.path", '/root/.clearml/venvs-cache') venvs_cache_path = temp_config.get("agent.docker_internal_mounts.venvs_cache", None)
if not venvs_cache_path:
venvs_cache_path = "/root/.clearml/venvs-cache"
temp_config.put("agent.docker_internal_mounts.venvs_cache", venvs_cache_path)
# update the venvs cache path to the container scope
temp_config.put("agent.venvs_cache.path", venvs_cache_path)
if (ENV_SSH_AUTH_SOCK.get() or '').strip(): if (ENV_SSH_AUTH_SOCK.get() or '').strip():
self._host_ssh_cache = None self._host_ssh_cache = None
@ -4157,7 +4163,7 @@ class Worker(ServiceCommandSection):
mounted_cache_dir = temp_config.get("sdk.storage.cache.default_base_dir") mounted_cache_dir = temp_config.get("sdk.storage.cache.default_base_dir")
mounted_pip_dl_dir = temp_config.get("agent.pip_download_cache.path") mounted_pip_dl_dir = temp_config.get("agent.pip_download_cache.path")
mounted_vcs_cache = temp_config.get("agent.vcs_cache.path") mounted_vcs_cache = temp_config.get("agent.vcs_cache.path")
mounted_venvs_cache = temp_config.get("agent.venvs_cache.path", "") mounted_venvs_cache = temp_config.get("agent.docker_internal_mounts.venvs_cache", "")
mount_ssh = temp_config.get("agent.docker_internal_mounts.ssh_folder", None) mount_ssh = temp_config.get("agent.docker_internal_mounts.ssh_folder", None)
mount_ssh_ro = temp_config.get("agent.docker_internal_mounts.ssh_ro_folder", None) mount_ssh_ro = temp_config.get("agent.docker_internal_mounts.ssh_ro_folder", None)
mount_apt_cache = temp_config.get("agent.docker_internal_mounts.apt_cache", None) mount_apt_cache = temp_config.get("agent.docker_internal_mounts.apt_cache", None)

2
clearml_agent/helper/package/base.py
View File

@ -219,6 +219,8 @@ class PackageManager(object):
if not self._get_cache_manager(): if not self._get_cache_manager():
return return
print('Adding venv into cache: {}'.format(source_folder))
try: try:
keys = self._generate_reqs_hash_keys(requirements, docker_cmd, python_version, cuda_version) keys = self._generate_reqs_hash_keys(requirements, docker_cmd, python_version, cuda_version)
return self._get_cache_manager().add_entry( return self._get_cache_manager().add_entry(

1
docs/clearml.conf
View File

@ -308,6 +308,7 @@ agent {
# pip_cache: "/root/.cache/pip" # pip_cache: "/root/.cache/pip"
# poetry_cache: "/root/.cache/pypoetry" # poetry_cache: "/root/.cache/pypoetry"
# vcs_cache: "/root/.clearml/vcs-cache" # vcs_cache: "/root/.clearml/vcs-cache"
# venvs_cache: "/root/.clearml/venvs-cache"
# venv_build: "~/.clearml/venvs-builds" # venv_build: "~/.clearml/venvs-builds"
# pip_download: "/root/.clearml/pip-download-cache" # pip_download: "/root/.clearml/pip-download-cache"
# } # }