ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add Task.force_store_standalone_script() to force storing standalone script instead of git repository reference (issue #340)

Browse Source
This commit is contained in:
allegroai 2022-01-05 08:57:33 +02:00
parent de6ea269f8
commit ca20121fb9
2 changed files with 27 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
clearml/backend_interface/task/repo/scriptinfo.py
View File

@ -760,7 +760,7 @@ class ScriptInfo(object):
def _get_script_info( def _get_script_info(
cls, filepaths, check_uncommitted=True, create_requirements=True, log=None, cls, filepaths, check_uncommitted=True, create_requirements=True, log=None,
uncommitted_from_remote=False, detect_jupyter_notebook=True, uncommitted_from_remote=False, detect_jupyter_notebook=True,
add_missing_installed_packages=False, detailed_req_report=None): add_missing_installed_packages=False, detailed_req_report=None, force_single_script=False):
jupyter_filepath = cls._get_jupyter_notebook_filename() if detect_jupyter_notebook else None jupyter_filepath = cls._get_jupyter_notebook_filename() if detect_jupyter_notebook else None
if jupyter_filepath: if jupyter_filepath:
scripts_path = [Path(os.path.normpath(jupyter_filepath)).absolute()] scripts_path = [Path(os.path.normpath(jupyter_filepath)).absolute()]
@ -786,6 +786,10 @@ class ScriptInfo(object):
script_dir = scripts_dir[0] script_dir = scripts_dir[0]
script_path = scripts_path[0] script_path = scripts_path[0]
if force_single_script:
plugin = None
else:
plugin = next((p for p in cls.plugins if p.exists(script_dir)), None) plugin = next((p for p in cls.plugins if p.exists(script_dir)), None)
repo_info = DetectionResult() repo_info = DetectionResult()
@ -880,7 +884,7 @@ class ScriptInfo(object):
@classmethod @classmethod
def get(cls, filepaths=None, check_uncommitted=True, create_requirements=True, log=None, def get(cls, filepaths=None, check_uncommitted=True, create_requirements=True, log=None,
uncommitted_from_remote=False, detect_jupyter_notebook=True, add_missing_installed_packages=False, uncommitted_from_remote=False, detect_jupyter_notebook=True, add_missing_installed_packages=False,
detailed_req_report=None): detailed_req_report=None, force_single_script=False):
try: try:
if not filepaths: if not filepaths:
filepaths = [sys.argv[0], ] filepaths = [sys.argv[0], ]
@ -892,6 +896,7 @@ class ScriptInfo(object):
detect_jupyter_notebook=detect_jupyter_notebook, detect_jupyter_notebook=detect_jupyter_notebook,
add_missing_installed_packages=add_missing_installed_packages, add_missing_installed_packages=add_missing_installed_packages,
detailed_req_report=detailed_req_report, detailed_req_report=detailed_req_report,
force_single_script=force_single_script,
) )
except SystemExit: except SystemExit:
pass pass

21
clearml/backend_interface/task/task.py
View File

@ -75,6 +75,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
_report_subprocess_enabled = deferred_config('development.report_use_subprocess', sys.platform == 'linux') _report_subprocess_enabled = deferred_config('development.report_use_subprocess', sys.platform == 'linux')
_force_use_pip_freeze = deferred_config(multi=[('development.detect_with_pip_freeze', False), _force_use_pip_freeze = deferred_config(multi=[('development.detect_with_pip_freeze', False),
('development.detect_with_conda_freeze', False)]) ('development.detect_with_conda_freeze', False)])
_force_store_standalone_script = False
_offline_filename = 'task.json' _offline_filename = 'task.json'
class TaskTypes(Enum): class TaskTypes(Enum):
@ -246,8 +247,11 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
result, script_requirements = ScriptInfo.get( result, script_requirements = ScriptInfo.get(
filepaths=[self._calling_filename, sys.argv[0], ] filepaths=[self._calling_filename, sys.argv[0], ]
if ScriptInfo.is_running_from_module() else [sys.argv[0], self._calling_filename, ], if ScriptInfo.is_running_from_module() else [sys.argv[0], self._calling_filename, ],
log=self.log, create_requirements=False, log=self.log,
check_uncommitted=self._store_diff, uncommitted_from_remote=self._store_remote_diff create_requirements=False,
check_uncommitted=self._store_diff,
uncommitted_from_remote=self._store_remote_diff,
force_single_script=self._force_store_standalone_script,
) )
for msg in result.warning_messages: for msg in result.warning_messages:
self.get_logger().report_text(msg) self.get_logger().report_text(msg)
@ -1856,6 +1860,19 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
""" """
cls._force_use_pip_freeze = requirements_file if requirements_file else bool(force) cls._force_use_pip_freeze = requirements_file if requirements_file else bool(force)
@classmethod
def force_store_standalone_script(cls, force=True):
# type: (bool) -> None
"""
Force using storing the main python file as a single standalone script, instead of linking with the
local git repository/commit ID.
Notice: Must be called before `Task.init` !
:param force: Set force using `pip freeze` flag on/off
"""
cls._force_store_standalone_script = bool(force)
def _get_default_report_storage_uri(self): def _get_default_report_storage_uri(self):
# type: () -> str # type: () -> str
if self._offline_mode: if self._offline_mode: