Fix UV priority

Fix UV cache is disabled, UV handles its own cache
Fix UV freeze
Fix make sure we do not use pip cache if poetry/uv is used (even if we reverted to pip we can't know if someone changed the repository and now in a new version, a lock file exists)
This commit is contained in:
clearml 2024-12-12 23:38:42 +02:00
parent cc656e2969
commit d8366dedc6
3 changed files with 29 additions and 17 deletions

View File

@ -3002,9 +3002,11 @@ class Worker(ServiceCommandSection):
# Add the script CWD to the python path
if repo_info and repo_info.root and self._session.config.get('agent.force_git_root_python_path', None):
python_path = get_python_path(repo_info.root, None, self.package_api, is_conda_env=self.is_conda)
python_path = get_python_path(repo_info.root, None, self.package_api,
is_conda_env=self.is_conda or self.uv.enabled)
else:
python_path = get_python_path(script_dir, execution.entry_point, self.package_api, is_conda_env=self.is_conda)
python_path = get_python_path(script_dir, execution.entry_point, self.package_api,
is_conda_env=self.is_conda or self.uv.enabled)
if ENV_TASK_EXTRA_PYTHON_PATH.get():
python_path = add_python_path(python_path, ENV_TASK_EXTRA_PYTHON_PATH.get())
if python_path:
@ -3385,7 +3387,7 @@ class Worker(ServiceCommandSection):
# disable caching with poetry because we cannot make it install into a specific folder
# Todo: add support for poetry caching
if not self.poetry.enabled:
if not self.poetry.enabled and not self.uv.enabled:
# disable caching if we skipped the venv creation or the entire python setup
if add_venv_folder_cache and not self._standalone_mode and (
not ENV_AGENT_SKIP_PIP_VENV_INSTALL.get() and
@ -3490,7 +3492,9 @@ class Worker(ServiceCommandSection):
package_api.cwd = cwd
api = self._install_poetry_requirements(repo_info, execution.working_dir)
api = self._install_uv_requirements(repo_info, execution.working_dir)
if not api:
api = self._install_uv_requirements(repo_info, execution.working_dir)
if api:
# update back the package manager, this hack should be fixed
if package_api == self.package_api:
@ -3948,12 +3952,14 @@ class Worker(ServiceCommandSection):
'To accelerate spin-up time set `agent.venvs_cache.path=~/.clearml/venvs-cache` :::\n')
# check if we have a cached folder
if cached_requirements and not skip_pip_venv_install and self.package_api.get_cached_venv(
requirements=cached_requirements,
docker_cmd=execution_info.docker_cmd if execution_info else None,
python_version=self.package_api.python,
cuda_version=self._session.config.get("agent.cuda_version"),
destination_folder=Path(venv_dir)
if (cached_requirements and not skip_pip_venv_install and
not self.poetry.enabled and not self.uv.enabled and
self.package_api.get_cached_venv(
requirements=cached_requirements,
docker_cmd=execution_info.docker_cmd if execution_info else None,
python_version=self.package_api.python,
cuda_version=self._session.config.get("agent.cuda_version"),
destination_folder=Path(venv_dir))
):
print('::: Using Cached environment {} :::'.format(self.package_api.get_last_used_entry_cache()))
return venv_dir, requirements_manager, True

View File

@ -7,7 +7,7 @@ from .requirements import SimpleSubstitution
class PriorityPackageRequirement(SimpleSubstitution):
name = ("cython", "numpy", "setuptools", "pip", )
name = ("cython", "numpy", "setuptools", "pip", "uv", )
optional_package_names = tuple()
def __init__(self, *args, **kwargs):

View File

@ -7,6 +7,7 @@ import os
from pathlib2 import Path
from clearml_agent.definitions import ENV_AGENT_FORCE_UV
from clearml_agent.helper.base import select_for_platform
from clearml_agent.helper.process import Argv, DEVNULL, check_if_command_exists
from clearml_agent.session import Session, UV
@ -199,13 +200,18 @@ class UvAPI(object):
)
def freeze(self, freeze_full_environment=False):
lines = self.config.run("pip", "show", cwd=str(self.path)).splitlines()
lines = [[p for p in line.split(" ") if p] for line in lines]
python = Path(self.path) / ".venv" / select_for_platform(linux="bin/python", windows="scripts/python.exe")
lines = self.config.run("pip", "freeze", "--python", str(python), cwd=str(self.path)).splitlines()
# fix local filesystem reference in freeze
from clearml_agent.external.requirements_parser.requirement import Requirement
packages = [Requirement.parse(p) for p in lines]
for p in packages:
if p.local_file and p.editable:
p.path = str(Path(p.path).relative_to(self.path))
p.line = "-e {}".format(p.path)
return {
"pip": [
parts[0] + "==" + parts[1] + " # " + " ".join(parts[2:])
for parts in lines
]
"pip": [p.line for p in packages]
}
def get_python_command(self, extra):