From b2da6395824334134b8ad407ec968fc1b8095933 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Sat, 10 Dec 2022 20:00:46 +0200 Subject: [PATCH] Add `CLEARML_AGENT_FORCE_SYSTEM_SITE_PACKAGES` env var (default true) to allow overriding default "system_site_packages: true" behavior when running tasks in containers (docker mode and k8s-glue) --- clearml_agent/commands/worker.py | 10 ++++++---- clearml_agent/definitions.py | 5 +++++ clearml_agent/glue/k8s.py | 16 ++++++++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index c090dc2..94b6521 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -57,10 +57,7 @@ from clearml_agent.definitions import ( ENV_WORKER_ID, ENV_WORKER_TAGS, ENV_DOCKER_SKIP_GPUS_FLAG, - ENV_AGENT_SECRET_KEY, ENV_AGENT_AUTH_TOKEN, - ENV_AWS_SECRET_KEY, - ENV_AZURE_ACCOUNT_KEY, ENV_AGENT_DISABLE_SSH_MOUNT, ENV_SSH_AUTH_SOCK, ENV_AGENT_SKIP_PIP_VENV_INSTALL, @@ -71,6 +68,7 @@ from clearml_agent.definitions import ( ENV_DEBUG_INFO, ENV_CHILD_AGENTS_COUNT_CMD, ENV_DOCKER_ARGS_FILTERS, + ENV_FORCE_SYSTEM_SITE_PACKAGES, ) from clearml_agent.definitions import WORKING_REPOSITORY_DIR, PIP_EXTRA_INDICES from clearml_agent.errors import ( @@ -3458,7 +3456,6 @@ class Worker(ServiceCommandSection): temp_config.put("sdk.storage.cache.default_base_dir", mounted_cache_dir) temp_config.put("agent.pip_download_cache.path", mounted_pip_dl_dir) temp_config.put("agent.vcs_cache.path", mounted_vcs_cache) - temp_config.put("agent.package_manager.system_site_packages", True) temp_config.put("agent.package_manager.conda_env_as_base_docker", False) temp_config.put("agent.default_python", "") temp_config.put("agent.python_binary", "") @@ -3470,6 +3467,11 @@ class Worker(ServiceCommandSection): temp_config.put("agent.git_pass", (ENV_AGENT_GIT_PASS.get() or self._session.config.get("agent.git_pass", None))) + force_system_site_packages = ENV_FORCE_SYSTEM_SITE_PACKAGES.get() + force_system_site_packages = force_system_site_packages if force_system_site_packages is not None else True + if force_system_site_packages: + temp_config.put("agent.package_manager.system_site_packages", True) + if temp_config.get("agent.venvs_cache.path", None): temp_config.put("agent.venvs_cache.path", '/root/.clearml/venvs-cache') diff --git a/clearml_agent/definitions.py b/clearml_agent/definitions.py index 1c4dbd2..a44f612 100644 --- a/clearml_agent/definitions.py +++ b/clearml_agent/definitions.py @@ -154,6 +154,11 @@ ENV_CHILD_AGENTS_COUNT_CMD = EnvironmentConfig('CLEARML_AGENT_CHILD_AGENTS_COUNT ENV_DOCKER_ARGS_FILTERS = EnvironmentConfig('CLEARML_AGENT_DOCKER_ARGS_FILTERS') ENV_DOCKER_ARGS_HIDE_ENV = EnvironmentConfig('CLEARML_AGENT_DOCKER_ARGS_HIDE_ENV') +ENV_FORCE_SYSTEM_SITE_PACKAGES = EnvironmentConfig('CLEARML_AGENT_FORCE_SYSTEM_SITE_PACKAGES', type=bool) +""" Force system_site_packages: true when running tasks in containers (i.e. docker mode or k8s glue) """ + + + ENV_CUSTOM_BUILD_SCRIPT = EnvironmentConfig('CLEARML_AGENT_CUSTOM_BUILD_SCRIPT') """ Specifies a custom environment setup script to be executed instead of installing a virtual environment. diff --git a/clearml_agent/glue/k8s.py b/clearml_agent/glue/k8s.py index 7c87790..8e763c1 100644 --- a/clearml_agent/glue/k8s.py +++ b/clearml_agent/glue/k8s.py @@ -22,7 +22,12 @@ import yaml from clearml_agent.backend_api.session import Request from clearml_agent.commands.events import Events from clearml_agent.commands.worker import Worker, get_task_container, set_task_container, get_next_task -from clearml_agent.definitions import ENV_DOCKER_IMAGE, ENV_AGENT_GIT_USER, ENV_AGENT_GIT_PASS +from clearml_agent.definitions import ( + ENV_DOCKER_IMAGE, + ENV_AGENT_GIT_USER, + ENV_AGENT_GIT_PASS, + ENV_FORCE_SYSTEM_SITE_PACKAGES, +) from clearml_agent.errors import APIError from clearml_agent.glue.definitions import ENV_START_AGENT_SCRIPT_PATH from clearml_agent.helper.base import safe_remove_file @@ -136,8 +141,11 @@ class K8sIntegration(Worker): self.k8s_pending_queue_name = k8s_pending_queue_name or self.K8S_PENDING_QUEUE self.k8s_pending_queue_id = None self.container_bash_script = container_bash_script or self.CONTAINER_BASH_SCRIPT - # Always do system packages, because by we will be running inside a docker - self._session.config.put("agent.package_manager.system_site_packages", True) + force_system_packages = ENV_FORCE_SYSTEM_SITE_PACKAGES.get() + self._force_system_site_packages = force_system_packages if force_system_packages is not None else True + if self._force_system_site_packages: + # Use system packages, because by we will be running inside a docker + self._session.config.put("agent.package_manager.system_site_packages", True) # Add debug logging if debug: self.log.logger.disabled = False @@ -455,7 +463,7 @@ class K8sIntegration(Worker): git_user = ENV_AGENT_GIT_USER.get() or self._session.config.get("agent.git_user", None) git_pass = ENV_AGENT_GIT_PASS.get() or self._session.config.get("agent.git_pass", None) extra_config_values = [ - 'agent.package_manager.system_site_packages: true', + 'agent.package_manager.system_site_packages: true' if self._force_system_site_packages else '', 'agent.git_user: "{}"'.format(git_user) if git_user else '', 'agent.git_pass: "{}"'.format(git_pass) if git_pass else '', ]