ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix support for "-r requirements.txt" in installed packages

Browse Source
This commit is contained in:
allegroai 2021-08-05 19:19:54 +03:00
parent aede6f4bac
commit 0fbbe774fa
4 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clearml_agent/external/requirements_parser/parser.py vendored
View File

@ -31,10 +31,12 @@ def parse(reqstr, cwd=None):
elif not line or line.startswith('#'): elif not line or line.startswith('#'):
# comments are lines that start with # only # comments are lines that start with # only
continue continue
elif line.startswith('-r') or line.startswith('--requirement'): elif line.startswith('-r ') or line.startswith('--requirement '):
_, new_filename = line.split() _, new_filename = line.split()
new_file_path = os.path.join( new_file_path = os.path.join(
os.path.dirname(filename or '.') if filename or not cwd else cwd, new_filename) os.path.dirname(filename or '.') if filename or not cwd else cwd, new_filename)
if not os.path.exists(new_file_path):
continue
with open(new_file_path) as f: with open(new_file_path) as f:
for requirement in parse(f): for requirement in parse(f):
yield requirement yield requirement

3
clearml_agent/helper/package/base.py
View File

@ -241,6 +241,9 @@ class PackageManager(object):
if p.strip(strip_chars) and not p.strip(strip_chars).startswith('#')]) if p.strip(strip_chars) and not p.strip(strip_chars).startswith('#')])
if not pip_reqs and not conda_reqs: if not pip_reqs and not conda_reqs:
continue continue
# do not process "-r" or "--requirement" because we cannot know what we have in the git repo.
if any(r.strip().startswith('-r ') or r.strip().startswith('--requirement ') for r in pip_reqs):
continue
hash_text = '{class_type}\n{docker_cmd}\n{cuda_ver}\n{python_version}\n{pip_reqs}\n{conda_reqs}'.format( hash_text = '{class_type}\n{docker_cmd}\n{cuda_ver}\n{python_version}\n{pip_reqs}\n{conda_reqs}'.format(
class_type=str(cls), class_type=str(cls),
docker_cmd=str(docker_cmd or ''), docker_cmd=str(docker_cmd or ''),

3
clearml_agent/helper/package/external_req.py
View File

@ -139,7 +139,8 @@ class ExternalRequirements(SimpleSubstitution):
try: try:
if not req.name and req.req and not req.req.editable and not req.req.vcs and \ if not req.name and req.req and not req.req.editable and not req.req.vcs and \
req.req.line and req.req.line.strip().split('#')[0] and \ req.req.line and req.req.line.strip().split('#')[0] and \
not req.req.line.strip().split('#')[0].lower().endswith('.whl'): not req.req.line.strip().split('#')[0].lower().endswith('.whl') and \
not (req.req.line.strip().startswith('-r ') or req.req.line.strip().startswith('--requirement ')):
return True return True
except Exception: except Exception:
pass pass

9
clearml_agent/helper/package/requirements.py
View File

@ -469,16 +469,17 @@ class RequirementsManager(object):
def replace(self, requirements): # type: (Text) -> Text def replace(self, requirements): # type: (Text) -> Text
def safe_parse(req_str): def safe_parse(req_str):
# noinspection PyBroadException
try: try:
return next(parse(req_str, cwd=self._cwd)) return list(parse(req_str, cwd=self._cwd))
except Exception as ex: except Exception as ex:
return Requirement(req_str) return [Requirement(req_str)]
parsed_requirements = tuple( parsed_requirements = tuple(
map( map(
MarkerRequirement, MarkerRequirement,
[safe_parse(line) for line in (requirements.splitlines() [r for line in (requirements.splitlines() if isinstance(requirements, six.text_type) else requirements)
if isinstance(requirements, six.text_type) else requirements)] for r in safe_parse(line)]
) )
) )
if not parsed_requirements: if not parsed_requirements: