From 0fbbe774fab2f3617333ea57972720b089e51439 Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Thu, 5 Aug 2021 19:19:54 +0300
Subject: [PATCH] Fix support for "-r requirements.txt" in installed packages

---
 clearml_agent/external/requirements_parser/parser.py | 4 +++-
 clearml_agent/helper/package/base.py                 | 3 +++
 clearml_agent/helper/package/external_req.py         | 3 ++-
 clearml_agent/helper/package/requirements.py         | 9 +++++----
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/clearml_agent/external/requirements_parser/parser.py b/clearml_agent/external/requirements_parser/parser.py
index 8669320..cd1d7b1 100644
--- a/clearml_agent/external/requirements_parser/parser.py
+++ b/clearml_agent/external/requirements_parser/parser.py
@@ -31,10 +31,12 @@ def parse(reqstr, cwd=None):
         elif not line or line.startswith('#'):
             # comments are lines that start with # only
             continue
-        elif line.startswith('-r') or line.startswith('--requirement'):
+        elif line.startswith('-r ') or line.startswith('--requirement '):
             _, new_filename = line.split()
             new_file_path = os.path.join(
                 os.path.dirname(filename or '.') if filename or not cwd else cwd, new_filename)
+            if not os.path.exists(new_file_path):
+                continue
             with open(new_file_path) as f:
                 for requirement in parse(f):
                     yield requirement
diff --git a/clearml_agent/helper/package/base.py b/clearml_agent/helper/package/base.py
index fc08b14..662d108 100644
--- a/clearml_agent/helper/package/base.py
+++ b/clearml_agent/helper/package/base.py
@@ -241,6 +241,9 @@ class PackageManager(object):
                                  if p.strip(strip_chars) and not p.strip(strip_chars).startswith('#')])
             if not pip_reqs and not conda_reqs:
                 continue
+            # do not process "-r" or "--requirement" because we cannot know what we have in the git repo.
+            if any(r.strip().startswith('-r ') or r.strip().startswith('--requirement ') for r in pip_reqs):
+                continue
             hash_text = '{class_type}\n{docker_cmd}\n{cuda_ver}\n{python_version}\n{pip_reqs}\n{conda_reqs}'.format(
                 class_type=str(cls),
                 docker_cmd=str(docker_cmd or ''),
diff --git a/clearml_agent/helper/package/external_req.py b/clearml_agent/helper/package/external_req.py
index be96e65..f9daed1 100644
--- a/clearml_agent/helper/package/external_req.py
+++ b/clearml_agent/helper/package/external_req.py
@@ -139,7 +139,8 @@ class ExternalRequirements(SimpleSubstitution):
         try:
             if not req.name and req.req and not req.req.editable and not req.req.vcs and \
                     req.req.line and req.req.line.strip().split('#')[0] and \
-                    not req.req.line.strip().split('#')[0].lower().endswith('.whl'):
+                    not req.req.line.strip().split('#')[0].lower().endswith('.whl') and \
+                    not (req.req.line.strip().startswith('-r ') or req.req.line.strip().startswith('--requirement ')):
                 return True
         except Exception:
             pass
diff --git a/clearml_agent/helper/package/requirements.py b/clearml_agent/helper/package/requirements.py
index 4086100..f8b68e8 100644
--- a/clearml_agent/helper/package/requirements.py
+++ b/clearml_agent/helper/package/requirements.py
@@ -469,16 +469,17 @@ class RequirementsManager(object):
 
     def replace(self, requirements):  # type: (Text) -> Text
         def safe_parse(req_str):
+            # noinspection PyBroadException
             try:
-                return next(parse(req_str, cwd=self._cwd))
+                return list(parse(req_str, cwd=self._cwd))
             except Exception as ex:
-                return Requirement(req_str)
+                return [Requirement(req_str)]
 
         parsed_requirements = tuple(
             map(
                 MarkerRequirement,
-                [safe_parse(line) for line in (requirements.splitlines()
-                                               if isinstance(requirements, six.text_type) else requirements)]
+                [r for line in (requirements.splitlines() if isinstance(requirements, six.text_type) else requirements)
+                 for r in safe_parse(line)]
             )
         )
         if not parsed_requirements: