mirror of
https://github.com/clearml/clearml-agent
synced 2025-04-06 05:25:07 +00:00
Fix uv_replace_pip feature
Fix UV cache based on sync/pip-replacement Fix if UV fails but lock file is missing, revert to UV as pip drop in replacement Fix use UV bin instead of UV python package to avoid nested VIRTUAL_ENV issues
This commit is contained in:
clearml
parent
eee261685f
commit
3b70e1c4a0
@ -74,6 +74,7 @@
|
||||
# poetry_install_extra_args: ["-v"]
|
||||
# uv_version: ">0.4",
|
||||
# uv_sync_extra_args: ["--all-extras"]
|
||||
# uv_replace_pip: false
|
||||
|
||||
# virtual environment inherits packages from system
|
||||
system_site_packages: false,
|
||||
|
||||
@ -116,7 +116,7 @@ from clearml_agent.helper.check_update import start_check_update_daemon
|
||||
from clearml_agent.helper.console import ensure_text, print_text, decode_binary_lines
|
||||
from clearml_agent.helper.environment.converters import strtobool
|
||||
from clearml_agent.helper.os.daemonize import daemonize_process
|
||||
from clearml_agent.helper.package.base import PackageManager
|
||||
from clearml_agent.helper.package.base import PackageManager, get_specific_package_version
|
||||
from clearml_agent.helper.package.conda_api import CondaAPI
|
||||
from clearml_agent.helper.package.external_req import ExternalRequirements, OnlyExternalRequirements
|
||||
from clearml_agent.helper.package.pip_api.system import SystemPip
|
||||
@ -2776,7 +2776,8 @@ class Worker(ServiceCommandSection):
|
||||
|
||||
return
|
||||
|
||||
def _get_task_python_version(self, task):
|
||||
@staticmethod
|
||||
def _get_task_python_version(task):
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
python_ver = task.script.binary
|
||||
@ -2786,12 +2787,13 @@ class Worker(ServiceCommandSection):
|
||||
|
||||
python_ver = python_ver.replace('python', '')
|
||||
# if we can cast it, we are good
|
||||
return '{}.{}'.format(
|
||||
int(python_ver.partition(".")[0]),
|
||||
int(python_ver.partition(".")[-1].partition(".")[0] or 0)
|
||||
)
|
||||
parts = python_ver.split('.')
|
||||
if len(parts) < 2:
|
||||
return '{}'.format(int(parts[0]))
|
||||
else:
|
||||
return '{}.{}'.format(int(parts[0]), int(parts[1]))
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
@resolve_names
|
||||
def execute(
|
||||
@ -3232,6 +3234,9 @@ class Worker(ServiceCommandSection):
|
||||
# check if we need to update backwards compatible OS environment
|
||||
if not os.environ.get("TRAINS_CONFIG_FILE") and os.environ.get("CLEARML_CONFIG_FILE"):
|
||||
os.environ["TRAINS_CONFIG_FILE"] = os.environ.get("CLEARML_CONFIG_FILE")
|
||||
# remove our internal env
|
||||
os.environ.pop("CLEARML_APT_INSTALL", None)
|
||||
os.environ.pop("TRAINS_APT_INSTALL", None)
|
||||
|
||||
print("Starting Task Execution:\n".format(current_task.id))
|
||||
exit_code = -1
|
||||
@ -3642,29 +3647,37 @@ class Worker(ServiceCommandSection):
|
||||
self.log.error("failed installing poetry requirements: {}".format(ex))
|
||||
return None
|
||||
|
||||
def _install_uv_requirements(self, repo_info, working_dir=None):
|
||||
# type: (Optional[RepoInfo], Optional[str]) -> Optional[UvAPI]
|
||||
def _install_uv_requirements(self, repo_info, working_dir=None, cached_requirements=None):
|
||||
# type: (Optional[RepoInfo], Optional[str], Optional[Dict[str, list]]) -> Optional[UvAPI]
|
||||
if not repo_info:
|
||||
return None
|
||||
|
||||
if not isinstance(self.package_api, UvAPI):
|
||||
return None
|
||||
|
||||
files_from_working_dir = self._session.config.get(
|
||||
"agent.package_manager.uv_files_from_repo_working_dir", False)
|
||||
lockfile_path = Path(repo_info.root) / ((working_dir or "") if files_from_working_dir else "")
|
||||
|
||||
try:
|
||||
if not self.uv.enabled:
|
||||
self.package_api.set_lockfile_path(lockfile_path)
|
||||
|
||||
if self.package_api.enabled:
|
||||
print('UV Enabled: Ignoring requested python packages, using repository uv lock file!')
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
self.package_api.install()
|
||||
return self.package_api
|
||||
except Exception as ex:
|
||||
self.log.error("Failed installing uv requirements from lock file: {}".format(ex))
|
||||
# if we have a lock file we actually fail, otherwise we revert to pip behaviour
|
||||
if self.package_api.lock_file_exists:
|
||||
raise
|
||||
self.log.error("Reverting to UV as pip replacement - installing "
|
||||
"from Task requirements or git requirements.txt")
|
||||
self.package_api._enabled = False
|
||||
return None
|
||||
|
||||
self.uv.initialize(cwd=lockfile_path)
|
||||
api = self.uv.get_api(lockfile_path)
|
||||
if api.enabled:
|
||||
print('UV Enabled: Ignoring requested python packages, using repository uv lock file!')
|
||||
api.install()
|
||||
return api
|
||||
|
||||
print(f"Could not find pyproject.toml or uv.lock file in {lockfile_path} \n")
|
||||
except Exception as ex:
|
||||
self.log.error("failed installing uv requirements: {}".format(ex))
|
||||
# we will create the "regular" venv style using UV
|
||||
return None
|
||||
|
||||
def install_requirements(
|
||||
@ -3696,7 +3709,16 @@ class Worker(ServiceCommandSection):
|
||||
|
||||
api = self._install_poetry_requirements(repo_info, execution.working_dir)
|
||||
if not api:
|
||||
api = self._install_uv_requirements(repo_info, execution.working_dir)
|
||||
api = self._install_uv_requirements(repo_info, execution.working_dir, cached_requirements)
|
||||
# if we could not find a lock file, but UV is installed and enabled, use it instead of pip
|
||||
if not api and not self._session.config.get("agent.package_manager.uv_replace_pip", False):
|
||||
if package_api == self.package_api and isinstance(self.package_api, UvAPI):
|
||||
# revert to venv that we used inside UV
|
||||
api = None
|
||||
self.package_api = package_api = package_api.get_venv_manager()
|
||||
elif not api:
|
||||
# this means `agent.package_manager.uv_replace_pip` is set to true
|
||||
print("INFO: using UV as pip drop-in replacement")
|
||||
|
||||
if api:
|
||||
# update back the package manager, this hack should be fixed
|
||||
@ -4128,6 +4150,15 @@ class Worker(ServiceCommandSection):
|
||||
Path(self._session.config["agent.venvs_dir"], executable_version_suffix)
|
||||
venv_dir = Path(os.path.expanduser(os.path.expandvars(venv_dir.as_posix())))
|
||||
|
||||
pip_version = get_specific_package_version(cached_requirements, package_name="pip")
|
||||
if pip_version:
|
||||
PackageManager.set_pip_version(pip_version)
|
||||
if self.uv.enabled:
|
||||
self.uv.set_python_version(requested_python_version)
|
||||
uv_version = get_specific_package_version(cached_requirements, package_name="uv")
|
||||
if uv_version:
|
||||
self.uv.set_uv_version(uv_version)
|
||||
|
||||
first_time = not standalone_mode and (
|
||||
is_windows_platform()
|
||||
or self.is_conda
|
||||
@ -4221,11 +4252,16 @@ class Worker(ServiceCommandSection):
|
||||
url=self._session.config["agent.venv_update.url"] or DEFAULT_VENV_UPDATE_URL,
|
||||
**package_manager_params
|
||||
)
|
||||
elif self.uv.enabled:
|
||||
self.uv.initialize()
|
||||
self.package_api = self.uv.get_api(**package_manager_params)
|
||||
else:
|
||||
self.package_api = VirtualenvPip(**package_manager_params)
|
||||
|
||||
if first_time:
|
||||
self.package_api.remove()
|
||||
call_package_manager_create = True
|
||||
|
||||
self.global_package_api = SystemPip(**global_package_manager_params)
|
||||
else:
|
||||
if standalone_mode:
|
||||
|
||||
@ -17,17 +17,21 @@ def parse(reqstr, cwd=None):
|
||||
:param cwd: Optional current working dir for -r file.txt loading
|
||||
:returns: a *generator* of Requirement objects
|
||||
"""
|
||||
filename = getattr(reqstr, 'name', None)
|
||||
try:
|
||||
# Python 2.x compatibility
|
||||
if not isinstance(reqstr, basestring): # noqa
|
||||
reqstr = reqstr.read()
|
||||
except NameError:
|
||||
# Python 3.x only
|
||||
if not isinstance(reqstr, str):
|
||||
reqstr = reqstr.read()
|
||||
if isinstance(reqstr, list) and reqstr and isinstance(reqstr[0], str):
|
||||
lines = reqstr
|
||||
else:
|
||||
filename = getattr(reqstr, 'name', None)
|
||||
try:
|
||||
# Python 2.x compatibility
|
||||
if not isinstance(reqstr, basestring): # noqa
|
||||
reqstr = reqstr.read()
|
||||
except NameError:
|
||||
# Python 3.x only
|
||||
if not isinstance(reqstr, str):
|
||||
reqstr = reqstr.read()
|
||||
lines = reqstr.splitlines()
|
||||
|
||||
for line in reqstr.splitlines():
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line == '':
|
||||
continue
|
||||
|
||||
@ -399,3 +399,26 @@ class PackageManager(object):
|
||||
return None
|
||||
|
||||
return self._cache_manager
|
||||
|
||||
|
||||
def get_specific_package_version(cached_requirements, package_name):
|
||||
pkg_version = None
|
||||
try:
|
||||
from clearml_agent.external.requirements_parser.requirement import Requirement
|
||||
from clearml_agent.external.requirements_parser import parse
|
||||
requirements = []
|
||||
if cached_requirements.get("pip", ""):
|
||||
requirements += cached_requirements.get("pip", "").split("\n") \
|
||||
if isinstance(cached_requirements.get("pip", ""), str) else cached_requirements.get("pip", [])
|
||||
|
||||
if cached_requirements.get("org_pip", ""):
|
||||
requirements += cached_requirements.get("org_pip", "").split("\n") \
|
||||
if isinstance(cached_requirements.get("org_pip", ""), str) else (
|
||||
cached_requirements.get("org_pip", []))
|
||||
|
||||
pkg_version = [p for p in parse(requirements) if p.name == package_name]
|
||||
if pkg_version:
|
||||
pkg_version = pkg_version[0].specs[0][1]
|
||||
except Exception as ex:
|
||||
print("Failed parsing {} package version ({})".format(package_name, ex))
|
||||
return pkg_version
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from copy import deepcopy
|
||||
from copy import deepcopy, copy
|
||||
from functools import wraps
|
||||
|
||||
from ..._vendor import attr
|
||||
@ -6,10 +6,14 @@ import sys
|
||||
import os
|
||||
from ..._vendor.pathlib2 import Path
|
||||
|
||||
import shutil
|
||||
|
||||
from clearml_agent.definitions import ENV_AGENT_FORCE_UV
|
||||
from clearml_agent.helper.base import select_for_platform
|
||||
from clearml_agent.helper.base import python_version_string, rm_tree
|
||||
from clearml_agent.helper.package.base import get_specific_package_version
|
||||
from clearml_agent.helper.package.pip_api.venv import VirtualenvPip
|
||||
from clearml_agent.helper.process import Argv, DEVNULL, check_if_command_exists
|
||||
from clearml_agent.session import Session, UV
|
||||
from clearml_agent.session import UV
|
||||
|
||||
|
||||
def prop_guard(prop, log_prop=None):
|
||||
@ -38,14 +42,41 @@ def prop_guard(prop, log_prop=None):
|
||||
|
||||
|
||||
class UvConfig:
|
||||
USE_UV_BIN = False
|
||||
|
||||
def __init__(self, session):
|
||||
# type: (Session, str) -> None
|
||||
# type: (str) -> None
|
||||
self.session = session
|
||||
self._log = session.get_logger(__name__)
|
||||
self._python = (
|
||||
sys.executable
|
||||
) # default, overwritten from session config in initialize()
|
||||
self._initialized = False
|
||||
self._api = None
|
||||
self._cwd = None
|
||||
self._is_sync = False
|
||||
self._uv_version = None
|
||||
self._uv_bin = None
|
||||
self._venv_python = None
|
||||
self._req_python_version = None
|
||||
|
||||
def set_uv_version(self, version):
|
||||
self._uv_version = version
|
||||
|
||||
def set_uv_bin(self, uv_bin_fullpath):
|
||||
self._uv_bin = uv_bin_fullpath
|
||||
|
||||
def get_uv_bin(self):
|
||||
return self._uv_bin or shutil.which("uv")
|
||||
|
||||
def set_python_version(self, python_version):
|
||||
self._req_python_version = python_version
|
||||
|
||||
def get_python_version(self):
|
||||
return self._req_python_version
|
||||
|
||||
def get_uv_version(self):
|
||||
return self._uv_version
|
||||
|
||||
@property
|
||||
def log(self):
|
||||
@ -60,8 +91,34 @@ class UvConfig:
|
||||
|
||||
_guard_enabled = prop_guard(enabled, log)
|
||||
|
||||
def set_binary(self, binary_path):
|
||||
self._python = binary_path or self._python
|
||||
|
||||
def get_binary(self):
|
||||
return self._python
|
||||
|
||||
def set_venv_binary(self, binary_path):
|
||||
self._venv_python = binary_path or self._python
|
||||
|
||||
def get_venv_binary(self):
|
||||
return self._venv_python
|
||||
|
||||
def is_binary_updated(self):
|
||||
return self._python != sys.executable
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
func = kwargs.pop("func", Argv.get_output)
|
||||
argv = self.get_run_argv(*args, **kwargs)
|
||||
|
||||
self.log.debug("running: %s", argv)
|
||||
|
||||
#synced = self._is_sync
|
||||
#self._is_sync = False
|
||||
ret = func(argv, **kwargs)
|
||||
#self._is_sync = synced
|
||||
return ret
|
||||
|
||||
def get_run_argv(self, *args, **kwargs):
|
||||
kwargs.setdefault("stdin", DEVNULL)
|
||||
kwargs["env"] = deepcopy(os.environ)
|
||||
if "VIRTUAL_ENV" in kwargs["env"] or "CONDA_PREFIX" in kwargs["env"]:
|
||||
@ -74,161 +131,346 @@ class UvConfig:
|
||||
kwargs["env"]["PATH"] = path
|
||||
|
||||
if self.session and self.session.config and args and args[0] == "sync":
|
||||
# Set the cache dir to venvs dir
|
||||
cache_dir = self.session.config.get("agent.venvs_dir", None)
|
||||
if cache_dir is not None:
|
||||
os.environ["UV_CACHE_DIR"] = cache_dir
|
||||
|
||||
extra_args = self.session.config.get(
|
||||
"agent.package_manager.uv_sync_extra_args", None
|
||||
)
|
||||
if extra_args:
|
||||
args = args + tuple(extra_args)
|
||||
self._is_sync = True
|
||||
|
||||
if check_if_command_exists("uv"):
|
||||
argv = Argv("uv", *args)
|
||||
# Set the cache dir to venvs dir is SYNCed otherwise use the pip download as cache
|
||||
if self._is_sync:
|
||||
cache_dir = self.session.config.get("agent.venvs_dir", None)
|
||||
if cache_dir is not None:
|
||||
kwargs["env"]["UV_CACHE_DIR"] = cache_dir
|
||||
else:
|
||||
argv = Argv(self._python, "-m", "uv", *args)
|
||||
self.log.debug("running: %s", argv)
|
||||
return func(argv, **kwargs)
|
||||
cache_dir = self.session.config.get("agent.pip_download_cache.path", None)
|
||||
if cache_dir is not None:
|
||||
kwargs["env"]["UV_CACHE_DIR"] = cache_dir
|
||||
|
||||
# if we need synced it then we cannot specify the python binary
|
||||
if not self._is_sync and self._venv_python:
|
||||
# if we have not synced then use the preinstalled venv python,
|
||||
# otherwise do not specify it
|
||||
args_i = next(i for i, a in enumerate(args+("-", )) if a.startswith("-") or a == "python")
|
||||
args = tuple(args[:args_i]) + ("--python", str(self._venv_python),) + tuple(args[args_i:])
|
||||
# elif "cwd" in kwargs:
|
||||
# cwd = Path(kwargs["cwd"])/".venv"
|
||||
# if cwd.exists():
|
||||
# args_i = next(i for i, a in enumerate(args+("-", )) if a.startswith("-") or a == "python")
|
||||
# args = tuple(args[:args_i]) + ("--python", str(cwd), ) + tuple(args[args_i:])
|
||||
|
||||
# if check_if_command_exists("uv"):
|
||||
# argv = Argv("uv", *args)
|
||||
# else:
|
||||
# argv = Argv(self._python, "-m", "uv", *args)
|
||||
|
||||
if self.USE_UV_BIN:
|
||||
argv = Argv(self.get_uv_bin(), *args, **kwargs)
|
||||
else:
|
||||
argv = Argv(self._python, "-m", "uv", *args, **kwargs)
|
||||
|
||||
return argv
|
||||
|
||||
@_guard_enabled
|
||||
def initialize(self, cwd=None):
|
||||
def initialize(self, cwd=None,):
|
||||
if not self._initialized:
|
||||
# use correct python version -- detected in Worker.install_virtualenv() and written to
|
||||
# session
|
||||
if self.session.config.get("agent.python_binary", None):
|
||||
self._python = self.session.config.get("agent.python_binary")
|
||||
if cwd:
|
||||
self._cwd = cwd
|
||||
|
||||
if (
|
||||
self.session.config.get("agent.package_manager.uv_version", None)
|
||||
is not None
|
||||
):
|
||||
version = str(
|
||||
self.session.config.get("agent.package_manager.uv_version")
|
||||
)
|
||||
|
||||
# get uv version
|
||||
version = version.replace(" ", "")
|
||||
if (
|
||||
("=" in version)
|
||||
or ("~" in version)
|
||||
or ("<" in version)
|
||||
or (">" in version)
|
||||
):
|
||||
version = version
|
||||
elif version:
|
||||
version = "==" + version
|
||||
# (we are not running it yet)
|
||||
argv = Argv(
|
||||
self._python,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"uv{}".format(version),
|
||||
"--upgrade",
|
||||
"--disable-pip-version-check",
|
||||
)
|
||||
# this is just for beauty and checks, we already set the verion in the Argv
|
||||
if not version:
|
||||
version = "latest"
|
||||
else:
|
||||
# mark to install uv if not already installed (we are not running it yet)
|
||||
argv = Argv(
|
||||
self._python,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"uv",
|
||||
"--disable-pip-version-check",
|
||||
)
|
||||
version = ""
|
||||
|
||||
# first upgrade pip if we need to
|
||||
try:
|
||||
from clearml_agent.helper.package.pip_api.venv import VirtualenvPip
|
||||
|
||||
pip = VirtualenvPip(
|
||||
session=self.session,
|
||||
python=self._python,
|
||||
requirements_manager=None,
|
||||
path=None,
|
||||
interpreter=self._python,
|
||||
)
|
||||
pip.upgrade_pip()
|
||||
except Exception as ex:
|
||||
self.log.warning("failed upgrading pip: {}".format(ex))
|
||||
|
||||
# check if we do not have a specific version and uv is found skip installation
|
||||
if not version and check_if_command_exists("uv"):
|
||||
print(
|
||||
"Notice: uv was found, no specific version required, skipping uv installation"
|
||||
)
|
||||
else:
|
||||
print("Installing / Upgrading uv package to {}".format(version))
|
||||
# now install uv
|
||||
try:
|
||||
print(argv.get_output())
|
||||
except Exception as ex:
|
||||
self.log.warning("failed installing uv: {}".format(ex))
|
||||
|
||||
# all done.
|
||||
self._initialized = True
|
||||
|
||||
def get_api(self, path):
|
||||
# type: (Path) -> UvAPI
|
||||
return UvAPI(self, path)
|
||||
def get_api(self, session, python, requirements_manager, path, *args, **kwargs):
|
||||
if not self._api:
|
||||
|
||||
self._api = UvAPI(
|
||||
lockfile_path=self._cwd, lock_config=self,
|
||||
session=session,
|
||||
python=python or self._python,
|
||||
requirements_manager=requirements_manager, path=path, *args, **kwargs)
|
||||
return self._api
|
||||
|
||||
|
||||
@attr.s
|
||||
class UvAPI(object):
|
||||
class UvAPI(VirtualenvPip):
|
||||
config = attr.ib(type=UvConfig)
|
||||
path = attr.ib(type=Path, converter=Path)
|
||||
|
||||
INDICATOR_FILES = "pyproject.toml", "uv.lock"
|
||||
VENV_SUFFIX = "_uv"
|
||||
|
||||
def __init__(self, lockfile_path, lock_config, session, python, requirements_manager,
|
||||
path, interpreter=None, execution_info=None, **kwargs):
|
||||
self.lockfile_path = Path(lockfile_path) if lockfile_path else None
|
||||
self.lock_config = lock_config
|
||||
self._installed = False
|
||||
self._enabled = None
|
||||
self._created = False
|
||||
self._uv_install_path = None
|
||||
super(UvAPI, self).__init__(
|
||||
session, python, requirements_manager,
|
||||
path, interpreter=interpreter, execution_info=execution_info, **kwargs)
|
||||
|
||||
def set_lockfile_path(self, lockfile_path):
|
||||
if lockfile_path:
|
||||
self.lockfile_path = Path(lockfile_path)
|
||||
|
||||
def install(self, lockfile_path=None):
|
||||
# type: (str) -> bool
|
||||
self.set_lockfile_path(lockfile_path)
|
||||
|
||||
def install(self):
|
||||
# type: () -> bool
|
||||
if self.enabled:
|
||||
self.config.run("sync", "--locked", cwd=str(self.path), func=Argv.check_call)
|
||||
self.lock_config.run("sync", "--locked", cwd=str(self.lockfile_path), func=Argv.check_call)
|
||||
self._installed = True
|
||||
# self.lock_config.set_binary(Path(self.lockfile_path) / ".venv" / "bin" / "python")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@property
|
||||
def is_installed(self):
|
||||
return self._installed
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self.config.enabled and (
|
||||
any((self.path / indicator).exists() for indicator in self.INDICATOR_FILES)
|
||||
)
|
||||
if self._enabled is None:
|
||||
self._enabled = self.lockfile_path and self.lock_config.enabled and (
|
||||
any((self.lockfile_path / indicator).exists() for indicator in self.INDICATOR_FILES)
|
||||
)
|
||||
return self._enabled
|
||||
|
||||
@property
|
||||
def lock_file_exists(self):
|
||||
return (self.lockfile_path and self.lock_config.enabled and
|
||||
(self.lockfile_path / self.INDICATOR_FILES[1]).exists())
|
||||
|
||||
def freeze(self, freeze_full_environment=False):
|
||||
python = Path(self.path) / ".venv" / select_for_platform(linux="bin/python", windows="scripts/python.exe")
|
||||
lines = self.config.run("pip", "freeze", "--python", str(python), cwd=str(self.path)).splitlines()
|
||||
if not self.is_installed or not self.lockfile_path or not self.lock_config.enabled:
|
||||
# there is a bug so we have to call pip to get the freeze because UV will return the wrong list
|
||||
# packages = self.run_with_env(('freeze',), output=True).splitlines()
|
||||
packages = self.lock_config.get_run_argv(
|
||||
"pip", "freeze", "--python", str(Path(self.path) / "bin" / "python"), cwd=self.lockfile_path).get_output().splitlines()
|
||||
# list clearml_agent as well
|
||||
# packages_without_program = [package for package in packages if PROGRAM_NAME not in package]
|
||||
return {'pip': packages}
|
||||
|
||||
lines = self.lock_config.run(
|
||||
"pip", "freeze",
|
||||
cwd=str(self.lockfile_path or self._cwd or self.path)
|
||||
).splitlines()
|
||||
# fix local filesystem reference in freeze
|
||||
from clearml_agent.external.requirements_parser.requirement import Requirement
|
||||
packages = [Requirement.parse(p) for p in lines]
|
||||
for p in packages:
|
||||
if p.local_file and p.editable:
|
||||
p.path = str(Path(p.path).relative_to(self.path))
|
||||
p.path = str(Path(p.path).relative_to(self.lockfile_path))
|
||||
p.line = "-e {}".format(p.path)
|
||||
|
||||
return {
|
||||
"pip": [p.line for p in packages]
|
||||
}
|
||||
|
||||
def get_python_command(self, extra):
|
||||
if check_if_command_exists("uv"):
|
||||
return Argv("uv", "run", "python", *extra)
|
||||
def get_python_command(self, extra=()):
|
||||
if self.lock_config and self.lockfile_path and self.is_installed:
|
||||
return self.lock_config.get_run_argv(
|
||||
"run", "--python", str(self.lockfile_path / ".venv" / "bin" / "python"), "python", *extra, cwd=self.lockfile_path)
|
||||
|
||||
# if not self.lock_config.get_venv_binary() and check_if_command_exists("uv"):
|
||||
# return Argv("uv", "run", "--no-project", "--python", self.lock_config.get_venv_binary(), "python", *extra)
|
||||
# else:
|
||||
# if UvConfig.USE_UV_BIN:
|
||||
# return Argv(shutil.which("uv"), "run", "--no-project", "--python", self.lock_config.get_venv_binary(), "python", *extra)
|
||||
# else:
|
||||
# return Argv(self.bin, "-m", "uv", "run", "--no-project", "--python", self.lock_config.get_venv_binary(), "python", *extra)
|
||||
#
|
||||
return Argv(self.lock_config.get_venv_binary(), *extra)
|
||||
|
||||
def _make_command(self, command):
|
||||
return self.lock_config.get_run_argv("pip", *command)
|
||||
|
||||
def _add_legacy_resolver_flag(self, pip_pkg_version):
|
||||
# no need for legacy flags
|
||||
pass
|
||||
|
||||
def get_venv_manager(self):
|
||||
# Create a new instance of the parent class dynamically
|
||||
parent_class = self.__class__.__bases__[0]
|
||||
parent_instance = parent_class.__new__(parent_class) # noqa
|
||||
parent_instance.__dict__ = copy(self.__dict__)
|
||||
return parent_instance
|
||||
|
||||
def create(self):
|
||||
"""
|
||||
Create virtualenv.
|
||||
Only valid if instantiated with path.
|
||||
Use self.python as self.bin does not exist.
|
||||
"""
|
||||
if self._created:
|
||||
return
|
||||
|
||||
# if found a lock file, we will create the entire environment when we can "install"
|
||||
if self.enabled:
|
||||
# create virtualenv for the UV package
|
||||
super(UvAPI, self).create()
|
||||
self.lock_config.set_binary(self.bin)
|
||||
return self
|
||||
|
||||
# no lock file create a venv
|
||||
pip_venv = self.install_uv_package()
|
||||
self.lock_config.set_venv_binary(self._bin)
|
||||
self._bin = pip_venv.bin
|
||||
|
||||
# Otherwise, we create a new venv here
|
||||
# if we want UV to create the venv we first need to install it, so we create a "temp" UV venv
|
||||
|
||||
# get python version
|
||||
python_version = self.lock_config.get_python_version()
|
||||
if self.python and not python_version:
|
||||
python_version = self.python.split("/")[-1].lower().replace("python", "").replace(".exe", "")
|
||||
try:
|
||||
float(python_version)
|
||||
except: # noqa
|
||||
python_version = None
|
||||
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
# if no python version requested or it's the same as ours create a new venv from the currenbt one
|
||||
if not python_version or python_version_string() == python_version:
|
||||
if UvConfig.USE_UV_BIN:
|
||||
command = Argv(self.lock_config.get_uv_bin(), "venv",
|
||||
"--python", sys.executable, *self.create_flags(), str(self.path))
|
||||
else:
|
||||
command = pip_venv.get_python_command(
|
||||
extra=("-m", "uv", "venv",
|
||||
"--python", sys.executable, *self.create_flags(), str(self.path))
|
||||
)
|
||||
else:
|
||||
# create and download the new python version
|
||||
if UvConfig.USE_UV_BIN:
|
||||
command = Argv(self.lock_config.get_uv_bin(), "venv",
|
||||
"--python", python_version, *self.create_flags(), str(self.path))
|
||||
else:
|
||||
command = pip_venv.get_python_command(
|
||||
extra=("-m", "uv", "venv",
|
||||
"--python", python_version, *self.create_flags(), str(self.path))
|
||||
)
|
||||
|
||||
print(python_version, python_version_string(), command)
|
||||
command.get_output()
|
||||
except Exception as ex:
|
||||
print("ERROR: UV venv creation failed: {}".format(ex))
|
||||
raise ex
|
||||
|
||||
self._created = True
|
||||
|
||||
return self
|
||||
|
||||
def install_uv_package(self, uv_version=None):
|
||||
if not uv_version:
|
||||
if self.lock_config:
|
||||
uv_version = self.lock_config.get_uv_version()
|
||||
uv_version = uv_version or self.session.config.get("agent.package_manager.uv_version", None)
|
||||
|
||||
# check the installed version
|
||||
existing_uv_version = None
|
||||
pip_venv = VirtualenvPip(
|
||||
session=self.session, python=self.python, requirements_manager=None,
|
||||
path=self.path, interpreter=self.lock_config.get_binary())
|
||||
packages = (pip_venv.freeze(freeze_full_environment=True) or dict()).get("pip")
|
||||
if packages:
|
||||
existing_uv_version = get_specific_package_version({"pip": packages}, package_name="uv")
|
||||
|
||||
argv = None
|
||||
version = None
|
||||
need_install = True
|
||||
|
||||
if uv_version is not None:
|
||||
version = str(uv_version)
|
||||
|
||||
# get uv version
|
||||
version = version.replace(" ", "")
|
||||
if (
|
||||
("=" in version)
|
||||
or ("~" in version)
|
||||
or ("<" in version)
|
||||
or (">" in version)
|
||||
):
|
||||
version = version
|
||||
elif version:
|
||||
version = "==" + version
|
||||
|
||||
if existing_uv_version:
|
||||
from clearml_agent.helper.package.requirements import SimpleVersion
|
||||
need_install = not SimpleVersion.compare_versions(
|
||||
existing_uv_version,
|
||||
*SimpleVersion.split_op_version(version))
|
||||
|
||||
if need_install:
|
||||
# (we are not running it yet)
|
||||
argv = (
|
||||
"install",
|
||||
"uv{}".format(version),
|
||||
"--upgrade",
|
||||
)
|
||||
|
||||
# this is just for beauty and checks, we already set the version in the Argv
|
||||
if not version:
|
||||
version = "latest"
|
||||
elif not existing_uv_version:
|
||||
# mark to install uv if not already installed (we are not running it yet)
|
||||
argv = (
|
||||
"install",
|
||||
"uv",
|
||||
)
|
||||
version = ""
|
||||
|
||||
# check if we do not have a specific version and uv is found skip installation
|
||||
if not version and (existing_uv_version or check_if_command_exists("uv")):
|
||||
print(
|
||||
"Notice: `uv`{} was found, no specific version required, "
|
||||
"skipping uv installation".format(existing_uv_version or "")
|
||||
)
|
||||
UvConfig.USE_UV_BIN = True
|
||||
elif argv:
|
||||
if version:
|
||||
print("Installing / Upgrading `uv` package to {}".format(version))
|
||||
else:
|
||||
print("Installing `uv`")
|
||||
|
||||
self._uv_install_path = str(self.path)[:-1] if str(self.path)[-1] == os.pathsep else str(self.path)
|
||||
self._uv_install_path += self.VENV_SUFFIX
|
||||
pip_venv = VirtualenvPip(
|
||||
session=self.session, python=self.python,
|
||||
requirements_manager=None, path=self._uv_install_path)
|
||||
pip_venv.create()
|
||||
|
||||
# now install uv
|
||||
try:
|
||||
pip_venv.run_with_env(argv)
|
||||
except Exception as ex:
|
||||
self.lock_config.log.warning("failed installing uv: {}".format(ex))
|
||||
|
||||
self.lock_config.set_binary(pip_venv.bin)
|
||||
if (Path(self._uv_install_path) / "bin" / "uv").exists():
|
||||
self.lock_config.set_uv_bin(Path(self._uv_install_path) / "bin" / "uv")
|
||||
UvConfig.USE_UV_BIN = True
|
||||
else:
|
||||
return Argv(self.config._python, "-m", "uv", "run", "python", *extra)
|
||||
print(
|
||||
"Notice: `uv` {}was found, version required is {}, skipping uv installation".format(
|
||||
existing_uv_version + " ", version)
|
||||
)
|
||||
|
||||
return pip_venv
|
||||
|
||||
def upgrade_pip(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def set_selected_package_manager(self, *args, **kwargs):
|
||||
pass
|
||||
def remove(self):
|
||||
"""
|
||||
Delete virtualenv.
|
||||
Only valid if instantiated with path.
|
||||
"""
|
||||
super(UvAPI, self).remove()
|
||||
uv_path = str(self.path)
|
||||
if uv_path and uv_path[-1] == os.pathsep:
|
||||
uv_path = uv_path[:-1]
|
||||
rm_tree(uv_path + self.VENV_SUFFIX)
|
||||
|
||||
def out_of_scope_install_package(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def install_from_file(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
@ -87,6 +87,8 @@ agent {
|
||||
# poetry_install_extra_args: ["-v"]
|
||||
# uv_version: ">0.4",
|
||||
# uv_sync_extra_args: ["--all-extras"]
|
||||
# # experimental, use UV as a pip replacement even when a lock-file is missing
|
||||
# uv_replace_pip: false
|
||||
|
||||
# virtual environment inheres packages from system
|
||||
system_site_packages: false,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user