From 92a1e07b3311e64228bc4762ca07f54df427ce79 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Fri, 26 Mar 2021 12:16:05 +0300 Subject: [PATCH] Fix local path replace back when using cache --- clearml_agent/commands/worker.py | 18 ++++++-- clearml_agent/helper/package/conda_api.py | 6 ++- clearml_agent/helper/package/external_req.py | 46 +++++++++++++++---- .../helper/package/pip_api/system.py | 6 ++- 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/clearml_agent/commands/worker.py b/clearml_agent/commands/worker.py index 16a74ed..12e351c 100644 --- a/clearml_agent/commands/worker.py +++ b/clearml_agent/commands/worker.py @@ -1469,25 +1469,30 @@ class Worker(ServiceCommandSection): directory, vcs, repo_info = self.get_repo_info(execution, current_task, venv_folder.as_posix()) + cwd = vcs.location if vcs and vcs.location else directory + if is_cached: # reinstalling git / local packages package_api = copy(self.package_api) + OnlyExternalRequirements.cwd = package_api.cwd = cwd package_api.requirements_manager = self._get_requirements_manager( base_interpreter=package_api.requirements_manager.get_interpreter(), - requirement_substitutions=[OnlyExternalRequirements] + requirement_substitutions=[OnlyExternalRequirements], ) # make sure we run the handlers cached_requirements = \ {k: package_api.requirements_manager.replace(requirements[k] or '') for k in requirements} package_api.load_requirements(cached_requirements) + # make sure we call the correct freeze + requirements_manager = package_api.requirements_manager else: self.install_requirements( execution, repo_info, requirements_manager=requirements_manager, cached_requirements=requirements, - cwd=vcs.location if vcs and vcs.location else directory, + cwd=cwd, package_api=self.global_package_api if install_globally else None, ) @@ -1735,14 +1740,16 @@ class Worker(ServiceCommandSection): print("\n") + cwd = vcs.location if vcs and vcs.location else directory + if is_cached and not standalone_mode: # reinstalling git / local packages package_api = copy(self.package_api) + OnlyExternalRequirements.cwd = package_api.cwd = cwd package_api.requirements_manager = self._get_requirements_manager( base_interpreter=package_api.requirements_manager.get_interpreter(), requirement_substitutions=[OnlyExternalRequirements] ) - package_api.cwd = vcs.location if vcs and vcs.location else directory # make sure we run the handlers cached_requirements = \ {k: package_api.requirements_manager.replace(requirements[k] or '') @@ -1750,6 +1757,8 @@ class Worker(ServiceCommandSection): if str(cached_requirements.get('pip', '')).strip() \ or str(cached_requirements.get('conda', '')).strip(): package_api.load_requirements(cached_requirements) + # make sure we call the correct freeze + requirements_manager = package_api.requirements_manager elif not is_cached and not standalone_mode: self.install_requirements( @@ -1757,7 +1766,7 @@ class Worker(ServiceCommandSection): repo_info, requirements_manager=requirements_manager, cached_requirements=requirements, - cwd=vcs.location if vcs and vcs.location else directory, + cwd=cwd, ) # do not update the task packages if we are using conda, @@ -2169,6 +2178,7 @@ class Worker(ServiceCommandSection): def install_requirements( self, execution, repo_info, requirements_manager, cached_requirements=None, cwd=None, package_api=None ): + ExternalRequirements.cwd = cwd return self.install_requirements_for_package_api(execution, repo_info, requirements_manager, cached_requirements=cached_requirements, cwd=cwd, package_api=package_api if package_api else self.package_api) diff --git a/clearml_agent/helper/package/conda_api.py b/clearml_agent/helper/package/conda_api.py index e2bdb3f..91b9fe9 100644 --- a/clearml_agent/helper/package/conda_api.py +++ b/clearml_agent/helper/package/conda_api.py @@ -650,12 +650,16 @@ class CondaAPI(PackageManager): ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]') return ansi_escape.sub('', line) + # make sure we are not running it with our own PYTHONPATH + env = dict(**os.environ) + env.pop('PYTHONPATH', None) + command = Argv(*command) # type: Executable if not raw: command = (self.conda,) + command + ("--quiet", "--json") try: print('Executing Conda: {}'.format(command.serialize())) - result = command.get_output(stdin=DEVNULL, **kwargs) + result = command.get_output(stdin=DEVNULL, env=env, **kwargs) if self.session.debug_mode: print(result) except Exception as e: diff --git a/clearml_agent/helper/package/external_req.py b/clearml_agent/helper/package/external_req.py index 1c1b898..be96e65 100644 --- a/clearml_agent/helper/package/external_req.py +++ b/clearml_agent/helper/package/external_req.py @@ -2,6 +2,8 @@ import re from collections import OrderedDict from typing import Text +from pathlib2 import Path + from .base import PackageManager from .requirements import SimpleSubstitution from ..base import safe_furl as furl @@ -10,22 +12,26 @@ from ..base import safe_furl as furl class ExternalRequirements(SimpleSubstitution): name = "external_link" + cwd = None def __init__(self, *args, **kwargs): super(ExternalRequirements, self).__init__(*args, **kwargs) self.post_install_req = [] self.post_install_req_lookup = OrderedDict() + self.post_install_local_req_lookup = OrderedDict() def match(self, req): # match local folder building: - # noinspection PyBroadException - try: - if not req.name and req.req and not req.req.editable and not req.req.vcs and \ - req.req.line and req.req.line.strip().split('#')[0] and \ - not req.req.line.strip().split('#')[0].lower().endswith('.whl'): - return True - except Exception: - pass + if self.is_local_folder_package(req): + # noinspection PyBroadException + try: + folder_path = req.req.line.strip().split('#')[0].strip() + if self.cwd and not Path(folder_path).is_absolute(): + folder_path = (Path(self.cwd) / Path(folder_path)).absolute().as_posix() + self.post_install_local_req_lookup['file://{}'.format(folder_path)] = req.req.line + except Exception: + pass + return True # match both editable or code or unparsed if not (not req.name or req.req and (req.req.editable or req.req.vcs)): @@ -113,8 +119,32 @@ class ExternalRequirements(SimpleSubstitution): if r not in self.post_install_req_lookup] list_of_requirements[k] += [self.post_install_req_lookup.get(r, '') for r in self.post_install_req_lookup.keys() if r in original_requirements] + + if self.post_install_local_req_lookup: + original_requirements = list_of_requirements[k] + list_of_requirements[k] = [ + r for r in original_requirements + if len(r.split('@', 1)) != 2 or r.split('@', 1)[1].strip() not in self.post_install_local_req_lookup] + + list_of_requirements[k] += [ + self.post_install_local_req_lookup.get(r.split('@', 1)[1].strip(), '') + for r in original_requirements + if len(r.split('@', 1)) == 2 and r.split('@', 1)[1].strip() in self.post_install_local_req_lookup] + return list_of_requirements + @classmethod + def is_local_folder_package(cls, req): + # noinspection PyBroadException + try: + if not req.name and req.req and not req.req.editable and not req.req.vcs and \ + req.req.line and req.req.line.strip().split('#')[0] and \ + not req.req.line.strip().split('#')[0].lower().endswith('.whl'): + return True + except Exception: + pass + return False + class OnlyExternalRequirements(ExternalRequirements): def __init__(self, *args, **kwargs): diff --git a/clearml_agent/helper/package/pip_api/system.py b/clearml_agent/helper/package/pip_api/system.py index 7a7b2e9..67d27b0 100644 --- a/clearml_agent/helper/package/pip_api/system.py +++ b/clearml_agent/helper/package/pip_api/system.py @@ -1,3 +1,4 @@ +import os import sys from itertools import chain from typing import Text, Optional @@ -82,7 +83,10 @@ class SystemPip(PackageManager): :param kwargs: kwargs for get_output/check_output command """ command = self._make_command(command) - return (command.get_output if output else command.check_call)(stdin=DEVNULL, **kwargs) + # make sure we are not running it with our own PYTHONPATH + env = dict(**os.environ) + env.pop('PYTHONPATH', None) + return (command.get_output if output else command.check_call)(stdin=DEVNULL, env=env, **kwargs) def _make_command(self, command): return Argv(self.bin, '-m', 'pip', '--disable-pip-version-check', *command)