ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add requirement parsing optimization using entry point by default

Browse Source
This commit is contained in:
allegroai
2020-04-26 22:58:13 +03:00
parent 7e4fba9d10
commit 9726f782f2
4 changed files with 32 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
trains/backend_interface/task/repo/scriptinfo.py
View File

@@ -26,7 +26,7 @@ class ScriptRequirements(object):
def __init__(self, root_folder):
self._root_folder = root_folder
def get_requirements(self):
def get_requirements(self, entry_point_filename=None):
try:
from ....utilities.pigar.reqs import get_installed_pkgs_detail
from ....utilities.pigar.__main__ import GenerateReqs
@@ -34,7 +34,8 @@ class ScriptRequirements(object):
gr = GenerateReqs(save_path='', project_path=self._root_folder, installed_pkgs=installed_pkgs,
ignores=['.git', '.hg', '.idea', '__pycache__', '.ipynb_checkpoints',
'site-packages', 'dist-packages'])
reqs, try_imports, guess, local_pks = gr.extract_reqs(module_callback=ScriptRequirements.add_trains_used_packages)
reqs, try_imports, guess, local_pks = gr.extract_reqs(
module_callback=ScriptRequirements.add_trains_used_packages, entry_point_filename=entry_point_filename)
return self.create_requirements_txt(reqs, local_pks)
except Exception:
return '', ''
@@ -493,7 +494,6 @@ class ScriptInfo(object):
else:
script_requirements = None
script_info = dict(
repository=furl(repo_info.url).remove(username=True, password=True).tostr(),
branch=repo_info.branch,

6
trains/backend_interface/task/task.py
View File

@@ -231,7 +231,11 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self.reload()
# if jupyter is present, requirements will be created in the background, when saving a snapshot
if result.script and script_requirements:
requirements, conda_requirements = script_requirements.get_requirements()
entry_point_filename = None if config.get('development.force_analyze_entire_repo', False) else \
os.path.join(result.script['working_dir'], result.script['entry_point'])
requirements, conda_requirements = script_requirements.get_requirements(
entry_point_filename=entry_point_filename)
if requirements:
if not result.script['requirements']:
result.script['requirements'] = {}