mirror of
https://github.com/clearml/clearml-agent
synced 2025-01-31 00:56:53 +00:00
Fix UV priority
Fix UV cache is disabled, UV handles its own cache Fix UV freeze Fix make sure we do not use pip cache if poetry/uv is used (even if we reverted to pip we can't know if someone changed the repository and now in a new version, a lock file exists)
This commit is contained in:
parent
cc656e2969
commit
d8366dedc6
@ -3002,9 +3002,11 @@ class Worker(ServiceCommandSection):
|
||||
|
||||
# Add the script CWD to the python path
|
||||
if repo_info and repo_info.root and self._session.config.get('agent.force_git_root_python_path', None):
|
||||
python_path = get_python_path(repo_info.root, None, self.package_api, is_conda_env=self.is_conda)
|
||||
python_path = get_python_path(repo_info.root, None, self.package_api,
|
||||
is_conda_env=self.is_conda or self.uv.enabled)
|
||||
else:
|
||||
python_path = get_python_path(script_dir, execution.entry_point, self.package_api, is_conda_env=self.is_conda)
|
||||
python_path = get_python_path(script_dir, execution.entry_point, self.package_api,
|
||||
is_conda_env=self.is_conda or self.uv.enabled)
|
||||
if ENV_TASK_EXTRA_PYTHON_PATH.get():
|
||||
python_path = add_python_path(python_path, ENV_TASK_EXTRA_PYTHON_PATH.get())
|
||||
if python_path:
|
||||
@ -3385,7 +3387,7 @@ class Worker(ServiceCommandSection):
|
||||
|
||||
# disable caching with poetry because we cannot make it install into a specific folder
|
||||
# Todo: add support for poetry caching
|
||||
if not self.poetry.enabled:
|
||||
if not self.poetry.enabled and not self.uv.enabled:
|
||||
# disable caching if we skipped the venv creation or the entire python setup
|
||||
if add_venv_folder_cache and not self._standalone_mode and (
|
||||
not ENV_AGENT_SKIP_PIP_VENV_INSTALL.get() and
|
||||
@ -3490,7 +3492,9 @@ class Worker(ServiceCommandSection):
|
||||
package_api.cwd = cwd
|
||||
|
||||
api = self._install_poetry_requirements(repo_info, execution.working_dir)
|
||||
api = self._install_uv_requirements(repo_info, execution.working_dir)
|
||||
if not api:
|
||||
api = self._install_uv_requirements(repo_info, execution.working_dir)
|
||||
|
||||
if api:
|
||||
# update back the package manager, this hack should be fixed
|
||||
if package_api == self.package_api:
|
||||
@ -3948,12 +3952,14 @@ class Worker(ServiceCommandSection):
|
||||
'To accelerate spin-up time set `agent.venvs_cache.path=~/.clearml/venvs-cache` :::\n')
|
||||
|
||||
# check if we have a cached folder
|
||||
if cached_requirements and not skip_pip_venv_install and self.package_api.get_cached_venv(
|
||||
requirements=cached_requirements,
|
||||
docker_cmd=execution_info.docker_cmd if execution_info else None,
|
||||
python_version=self.package_api.python,
|
||||
cuda_version=self._session.config.get("agent.cuda_version"),
|
||||
destination_folder=Path(venv_dir)
|
||||
if (cached_requirements and not skip_pip_venv_install and
|
||||
not self.poetry.enabled and not self.uv.enabled and
|
||||
self.package_api.get_cached_venv(
|
||||
requirements=cached_requirements,
|
||||
docker_cmd=execution_info.docker_cmd if execution_info else None,
|
||||
python_version=self.package_api.python,
|
||||
cuda_version=self._session.config.get("agent.cuda_version"),
|
||||
destination_folder=Path(venv_dir))
|
||||
):
|
||||
print('::: Using Cached environment {} :::'.format(self.package_api.get_last_used_entry_cache()))
|
||||
return venv_dir, requirements_manager, True
|
||||
|
@ -7,7 +7,7 @@ from .requirements import SimpleSubstitution
|
||||
|
||||
class PriorityPackageRequirement(SimpleSubstitution):
|
||||
|
||||
name = ("cython", "numpy", "setuptools", "pip", )
|
||||
name = ("cython", "numpy", "setuptools", "pip", "uv", )
|
||||
optional_package_names = tuple()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
@ -7,6 +7,7 @@ import os
|
||||
from pathlib2 import Path
|
||||
|
||||
from clearml_agent.definitions import ENV_AGENT_FORCE_UV
|
||||
from clearml_agent.helper.base import select_for_platform
|
||||
from clearml_agent.helper.process import Argv, DEVNULL, check_if_command_exists
|
||||
from clearml_agent.session import Session, UV
|
||||
|
||||
@ -199,13 +200,18 @@ class UvAPI(object):
|
||||
)
|
||||
|
||||
def freeze(self, freeze_full_environment=False):
|
||||
lines = self.config.run("pip", "show", cwd=str(self.path)).splitlines()
|
||||
lines = [[p for p in line.split(" ") if p] for line in lines]
|
||||
python = Path(self.path) / ".venv" / select_for_platform(linux="bin/python", windows="scripts/python.exe")
|
||||
lines = self.config.run("pip", "freeze", "--python", str(python), cwd=str(self.path)).splitlines()
|
||||
# fix local filesystem reference in freeze
|
||||
from clearml_agent.external.requirements_parser.requirement import Requirement
|
||||
packages = [Requirement.parse(p) for p in lines]
|
||||
for p in packages:
|
||||
if p.local_file and p.editable:
|
||||
p.path = str(Path(p.path).relative_to(self.path))
|
||||
p.line = "-e {}".format(p.path)
|
||||
|
||||
return {
|
||||
"pip": [
|
||||
parts[0] + "==" + parts[1] + " # " + " ".join(parts[2:])
|
||||
for parts in lines
|
||||
]
|
||||
"pip": [p.line for p in packages]
|
||||
}
|
||||
|
||||
def get_python_command(self, extra):
|
||||
|
Loading…
Reference in New Issue
Block a user