Fix local path replace back when using cache

clearml_agent/commands/worker.py

@@ -1469,25 +1469,30 @@ class Worker(ServiceCommandSection):
 
         directory, vcs, repo_info = self.get_repo_info(execution, current_task, venv_folder.as_posix())
 
+        cwd = vcs.location if vcs and vcs.location else directory
+
         if is_cached:
             # reinstalling git / local packages
             package_api = copy(self.package_api)
+            OnlyExternalRequirements.cwd = package_api.cwd = cwd
             package_api.requirements_manager = self._get_requirements_manager(
                 base_interpreter=package_api.requirements_manager.get_interpreter(),
-                requirement_substitutions=[OnlyExternalRequirements]
+                requirement_substitutions=[OnlyExternalRequirements],
             )
             # make sure we run the handlers
             cached_requirements = \
                 {k: package_api.requirements_manager.replace(requirements[k] or '')
                  for k in requirements}
             package_api.load_requirements(cached_requirements)
+            # make sure we call the correct freeze
+            requirements_manager = package_api.requirements_manager
         else:
             self.install_requirements(
                 execution,
                 repo_info,
                 requirements_manager=requirements_manager,
                 cached_requirements=requirements,
-                cwd=vcs.location if vcs and vcs.location else directory,
+                cwd=cwd,
                 package_api=self.global_package_api if install_globally else None,
             )
 
@@ -1735,14 +1740,16 @@ class Worker(ServiceCommandSection):
 
         print("\n")
 
+        cwd = vcs.location if vcs and vcs.location else directory
+
         if is_cached and not standalone_mode:
             # reinstalling git / local packages
             package_api = copy(self.package_api)
+            OnlyExternalRequirements.cwd = package_api.cwd = cwd
             package_api.requirements_manager = self._get_requirements_manager(
                 base_interpreter=package_api.requirements_manager.get_interpreter(),
                 requirement_substitutions=[OnlyExternalRequirements]
             )
-            package_api.cwd = vcs.location if vcs and vcs.location else directory
             # make sure we run the handlers
             cached_requirements = \
                 {k: package_api.requirements_manager.replace(requirements[k] or '')
@@ -1750,6 +1757,8 @@ class Worker(ServiceCommandSection):
             if str(cached_requirements.get('pip', '')).strip() \
                     or str(cached_requirements.get('conda', '')).strip():
                 package_api.load_requirements(cached_requirements)
+            # make sure we call the correct freeze
+            requirements_manager = package_api.requirements_manager
 
         elif not is_cached and not standalone_mode:
             self.install_requirements(
@@ -1757,7 +1766,7 @@ class Worker(ServiceCommandSection):
                 repo_info,
                 requirements_manager=requirements_manager,
                 cached_requirements=requirements,
-                cwd=vcs.location if vcs and vcs.location else directory,
+                cwd=cwd,
             )
 
         # do not update the task packages if we are using conda,
@@ -2169,6 +2178,7 @@ class Worker(ServiceCommandSection):
     def install_requirements(
         self, execution, repo_info, requirements_manager, cached_requirements=None, cwd=None, package_api=None
     ):
+        ExternalRequirements.cwd = cwd
         return self.install_requirements_for_package_api(execution, repo_info, requirements_manager,
                                                          cached_requirements=cached_requirements, cwd=cwd,
                                                          package_api=package_api if package_api else self.package_api)

clearml_agent/helper/package/conda_api.py

@@ -650,12 +650,16 @@ class CondaAPI(PackageManager):
             ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')
             return ansi_escape.sub('', line)
 
+        # make sure we are not running it with our own PYTHONPATH
+        env = dict(**os.environ)
+        env.pop('PYTHONPATH', None)
+
         command = Argv(*command)  # type: Executable
         if not raw:
             command = (self.conda,) + command + ("--quiet", "--json")
         try:
             print('Executing Conda: {}'.format(command.serialize()))
-            result = command.get_output(stdin=DEVNULL, **kwargs)
+            result = command.get_output(stdin=DEVNULL, env=env, **kwargs)
             if self.session.debug_mode:
                 print(result)
         except Exception as e:

clearml_agent/helper/package/external_req.py

@@ -2,6 +2,8 @@ import re
 from collections import OrderedDict
 from typing import Text
 
+from pathlib2 import Path
+
 from .base import PackageManager
 from .requirements import SimpleSubstitution
 from ..base import safe_furl as furl
@@ -10,22 +12,26 @@ from ..base import safe_furl as furl
 class ExternalRequirements(SimpleSubstitution):
 
     name = "external_link"
+    cwd = None
 
     def __init__(self, *args, **kwargs):
         super(ExternalRequirements, self).__init__(*args, **kwargs)
         self.post_install_req = []
         self.post_install_req_lookup = OrderedDict()
+        self.post_install_local_req_lookup = OrderedDict()
 
     def match(self, req):
         # match local folder building:
+        if self.is_local_folder_package(req):
             # noinspection PyBroadException
             try:
-            if not req.name and req.req and not req.req.editable and not req.req.vcs and \
+                folder_path = req.req.line.strip().split('#')[0].strip()
-                    req.req.line and req.req.line.strip().split('#')[0] and \
+                if self.cwd and not Path(folder_path).is_absolute():
-                    not req.req.line.strip().split('#')[0].lower().endswith('.whl'):
+                    folder_path = (Path(self.cwd) / Path(folder_path)).absolute().as_posix()
-                return True
+                self.post_install_local_req_lookup['file://{}'.format(folder_path)] = req.req.line
             except Exception:
                 pass
+            return True
 
         # match both editable or code or unparsed
         if not (not req.name or req.req and (req.req.editable or req.req.vcs)):
@@ -113,8 +119,32 @@ class ExternalRequirements(SimpleSubstitution):
                                        if r not in self.post_install_req_lookup]
             list_of_requirements[k] += [self.post_install_req_lookup.get(r, '')
                                         for r in self.post_install_req_lookup.keys() if r in original_requirements]
+
+            if self.post_install_local_req_lookup:
+                original_requirements = list_of_requirements[k]
+                list_of_requirements[k] = [
+                    r for r in original_requirements
+                    if len(r.split('@', 1)) != 2 or r.split('@', 1)[1].strip() not in self.post_install_local_req_lookup]
+
+                list_of_requirements[k] += [
+                    self.post_install_local_req_lookup.get(r.split('@', 1)[1].strip(), '')
+                    for r in original_requirements
+                    if len(r.split('@', 1)) == 2 and r.split('@', 1)[1].strip() in self.post_install_local_req_lookup]
+
         return list_of_requirements
 
+    @classmethod
+    def is_local_folder_package(cls, req):
+        # noinspection PyBroadException
+        try:
+            if not req.name and req.req and not req.req.editable and not req.req.vcs and \
+                    req.req.line and req.req.line.strip().split('#')[0] and \
+                    not req.req.line.strip().split('#')[0].lower().endswith('.whl'):
+                return True
+        except Exception:
+            pass
+        return False
+
 
 class OnlyExternalRequirements(ExternalRequirements):
     def __init__(self, *args, **kwargs):

clearml_agent/helper/package/pip_api/system.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from itertools import chain
 from typing import Text, Optional
@@ -82,7 +83,10 @@ class SystemPip(PackageManager):
         :param kwargs: kwargs for get_output/check_output command
         """
         command = self._make_command(command)
-        return (command.get_output if output else command.check_call)(stdin=DEVNULL, **kwargs)
+        # make sure we are not running it with our own PYTHONPATH
+        env = dict(**os.environ)
+        env.pop('PYTHONPATH', None)
+        return (command.get_output if output else command.check_call)(stdin=DEVNULL, env=env, **kwargs)
 
     def _make_command(self, command):
         return Argv(self.bin, '-m', 'pip', '--disable-pip-version-check', *command)