From d8366dedc6a7224f82d28070268ac471ffe2119d Mon Sep 17 00:00:00 2001 From: clearml <> Date: Thu, 12 Dec 2024 23:38:42 +0200 Subject: [PATCH] Fix UV priority Fix UV cache is disabled, UV handles its own cache Fix UV freeze Fix make sure we do not use pip cache if poetry/uv is used (even if we reverted to pip we can't know if someone changed the repository and now in a new version, a lock file exists) --- clearml_agent/commands/worker.py | 26 ++++++++++++-------- clearml_agent/helper/package/priority_req.py | 2 +- clearml_agent/helper/package/uv_api.py | 18 +++++++++----- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index 855131b..af2af20 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -3002,9 +3002,11 @@ class Worker(ServiceCommandSection): # Add the script CWD to the python path if repo_info and repo_info.root and self._session.config.get('agent.force_git_root_python_path', None): - python_path = get_python_path(repo_info.root, None, self.package_api, is_conda_env=self.is_conda) + python_path = get_python_path(repo_info.root, None, self.package_api, + is_conda_env=self.is_conda or self.uv.enabled) else: - python_path = get_python_path(script_dir, execution.entry_point, self.package_api, is_conda_env=self.is_conda) + python_path = get_python_path(script_dir, execution.entry_point, self.package_api, + is_conda_env=self.is_conda or self.uv.enabled) if ENV_TASK_EXTRA_PYTHON_PATH.get(): python_path = add_python_path(python_path, ENV_TASK_EXTRA_PYTHON_PATH.get()) if python_path: @@ -3385,7 +3387,7 @@ class Worker(ServiceCommandSection): # disable caching with poetry because we cannot make it install into a specific folder # Todo: add support for poetry caching - if not self.poetry.enabled: + if not self.poetry.enabled and not self.uv.enabled: # disable caching if we skipped the venv creation or the entire python setup if add_venv_folder_cache and not self._standalone_mode and ( not ENV_AGENT_SKIP_PIP_VENV_INSTALL.get() and @@ -3490,7 +3492,9 @@ class Worker(ServiceCommandSection): package_api.cwd = cwd api = self._install_poetry_requirements(repo_info, execution.working_dir) - api = self._install_uv_requirements(repo_info, execution.working_dir) + if not api: + api = self._install_uv_requirements(repo_info, execution.working_dir) + if api: # update back the package manager, this hack should be fixed if package_api == self.package_api: @@ -3948,12 +3952,14 @@ class Worker(ServiceCommandSection): 'To accelerate spin-up time set `agent.venvs_cache.path=~/.clearml/venvs-cache` :::\n') # check if we have a cached folder - if cached_requirements and not skip_pip_venv_install and self.package_api.get_cached_venv( - requirements=cached_requirements, - docker_cmd=execution_info.docker_cmd if execution_info else None, - python_version=self.package_api.python, - cuda_version=self._session.config.get("agent.cuda_version"), - destination_folder=Path(venv_dir) + if (cached_requirements and not skip_pip_venv_install and + not self.poetry.enabled and not self.uv.enabled and + self.package_api.get_cached_venv( + requirements=cached_requirements, + docker_cmd=execution_info.docker_cmd if execution_info else None, + python_version=self.package_api.python, + cuda_version=self._session.config.get("agent.cuda_version"), + destination_folder=Path(venv_dir)) ): print('::: Using Cached environment {} :::'.format(self.package_api.get_last_used_entry_cache())) return venv_dir, requirements_manager, True diff --git a/clearml_agent/helper/package/priority_req.py b/clearml_agent/helper/package/priority_req.py index b4e02ce..5a2f041 100644 --- a/clearml_agent/helper/package/priority_req.py +++ b/clearml_agent/helper/package/priority_req.py @@ -7,7 +7,7 @@ from .requirements import SimpleSubstitution class PriorityPackageRequirement(SimpleSubstitution): - name = ("cython", "numpy", "setuptools", "pip", ) + name = ("cython", "numpy", "setuptools", "pip", "uv", ) optional_package_names = tuple() def __init__(self, *args, **kwargs): diff --git a/clearml_agent/helper/package/uv_api.py b/clearml_agent/helper/package/uv_api.py index 55576a2..430c1e9 100644 --- a/clearml_agent/helper/package/uv_api.py +++ b/clearml_agent/helper/package/uv_api.py @@ -7,6 +7,7 @@ import os from pathlib2 import Path from clearml_agent.definitions import ENV_AGENT_FORCE_UV +from clearml_agent.helper.base import select_for_platform from clearml_agent.helper.process import Argv, DEVNULL, check_if_command_exists from clearml_agent.session import Session, UV @@ -199,13 +200,18 @@ class UvAPI(object): ) def freeze(self, freeze_full_environment=False): - lines = self.config.run("pip", "show", cwd=str(self.path)).splitlines() - lines = [[p for p in line.split(" ") if p] for line in lines] + python = Path(self.path) / ".venv" / select_for_platform(linux="bin/python", windows="scripts/python.exe") + lines = self.config.run("pip", "freeze", "--python", str(python), cwd=str(self.path)).splitlines() + # fix local filesystem reference in freeze + from clearml_agent.external.requirements_parser.requirement import Requirement + packages = [Requirement.parse(p) for p in lines] + for p in packages: + if p.local_file and p.editable: + p.path = str(Path(p.path).relative_to(self.path)) + p.line = "-e {}".format(p.path) + return { - "pip": [ - parts[0] + "==" + parts[1] + " # " + " ".join(parts[2:]) - for parts in lines - ] + "pip": [p.line for p in packages] } def get_python_command(self, extra):