Fix support for "-r requirements.txt" inside "installed packages"

This commit is contained in:
allegroai 2021-06-24 19:26:35 +03:00
parent 821a0c4a2b
commit aaa6b32f9f
3 changed files with 13 additions and 4 deletions
clearml_agent
commands
external/requirements_parser
helper/package

View File

@ -2370,6 +2370,10 @@ class Worker(ServiceCommandSection):
if not self.is_conda:
package_api.out_of_scope_install_package('Cython')
# add support for -r <file.txt> in requirements
if requirements_manager:
requirements_manager.set_cwd(cwd)
cached_requirements_failed = False
if cached_requirements and (cached_requirements.get('pip') is not None or
cached_requirements.get('conda') is not None):

View File

@ -4,13 +4,14 @@ import warnings
from .requirement import Requirement
def parse(reqstr):
def parse(reqstr, cwd=None):
"""
Parse a requirements file into a list of Requirements
See: pip/req.py:parse_requirements()
:param reqstr: a string or file like object containing requirements
:param cwd: Optional current working dir for -r file.txt loading
:returns: a *generator* of Requirement objects
"""
filename = getattr(reqstr, 'name', None)
@ -32,8 +33,8 @@ def parse(reqstr):
continue
elif line.startswith('-r') or line.startswith('--requirement'):
_, new_filename = line.split()
new_file_path = os.path.join(os.path.dirname(filename or '.'),
new_filename)
new_file_path = os.path.join(
os.path.dirname(filename or '.') if filename or not cwd else cwd, new_filename)
with open(new_file_path) as f:
for requirement in parse(f):
yield requirement

View File

@ -448,10 +448,14 @@ class RequirementsManager(object):
self.translator = RequirementsTranslator(session, interpreter=base_interpreter,
cache_dir=pip_cache_dir.as_posix())
self._base_interpreter = base_interpreter
self._cwd = None
def register(self, cls): # type: (Type[RequirementSubstitution]) -> None
self.handlers.append(cls(self._session))
def set_cwd(self, cwd):
self._cwd = str(cwd) if cwd else None
def _replace_one(self, req): # type: (MarkerRequirement) -> Optional[Text]
match = re.search(r';\s*(.*)', Text(req))
if match:
@ -466,7 +470,7 @@ class RequirementsManager(object):
def replace(self, requirements): # type: (Text) -> Text
def safe_parse(req_str):
try:
return next(parse(req_str))
return next(parse(req_str, cwd=self._cwd))
except Exception as ex:
return Requirement(req_str)