ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix python package analysis timed out before sending git repo detials.

Browse Source
This commit is contained in:
allegroai 2019-09-27 13:23:32 +03:00
parent 3eba23f1a3
commit c44638c8d9
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
trains/backend_interface/task/repo/scriptinfo.py
View File

@ -455,9 +455,13 @@ class ScriptInfo(object):
# if this is not jupyter, get the requirements.txt # if this is not jupyter, get the requirements.txt
requirements = '' requirements = ''
# create requirements if backend supports requirements # create requirements if backend supports requirements
if create_requirements and not jupyter_filepath and Session.check_min_api_version('2.2'): # if jupyter is present, requirements will be created in the background, when saving a snapshot
if not jupyter_filepath and Session.check_min_api_version('2.2'):
script_requirements = ScriptRequirements(Path(repo_root).as_posix()) script_requirements = ScriptRequirements(Path(repo_root).as_posix())
requirements = script_requirements.get_requirements() if create_requirements:
requirements = script_requirements.get_requirements()
else:
script_requirements = None
script_info = dict( script_info = dict(
repository=furl(repo_info.url).remove(username=True, password=True).tostr(), repository=furl(repo_info.url).remove(username=True, password=True).tostr(),
@ -480,7 +484,8 @@ class ScriptInfo(object):
if not any(script_info.values()): if not any(script_info.values()):
script_info = None script_info = None
return ScriptInfoResult(script=script_info, warning_messages=messages) return (ScriptInfoResult(script=script_info, warning_messages=messages),
script_requirements)
@classmethod @classmethod
def get(cls, filepath=sys.argv[0], check_uncommitted=True, create_requirements=True, log=None): def get(cls, filepath=sys.argv[0], check_uncommitted=True, create_requirements=True, log=None):
@ -491,7 +496,7 @@ class ScriptInfo(object):
except Exception as ex: except Exception as ex:
if log: if log:
log.warning("Failed auto-detecting task repository: {}".format(ex)) log.warning("Failed auto-detecting task repository: {}".format(ex))
return ScriptInfoResult() return ScriptInfoResult(), None
@attr.s @attr.s

21
trains/backend_interface/task/task.py
View File

@ -8,6 +8,7 @@ from threading import RLock, Thread
import six import six
from six.moves.urllib.parse import quote from six.moves.urllib.parse import quote
from ...backend_interface.task.repo.scriptinfo import ScriptRequirements
from ...backend_interface.task.development.worker import DevWorker from ...backend_interface.task.development.worker import DevWorker
from ...backend_api import Session from ...backend_api import Session
from ...backend_api.services import tasks, models, events, projects from ...backend_api.services import tasks, models, events, projects
@ -199,11 +200,13 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
except Exception: except Exception:
pass pass
# get repository and create requirements.txt from code base
try: try:
check_package_update_thread = Thread(target=check_package_update) check_package_update_thread = Thread(target=check_package_update)
check_package_update_thread.daemon = True check_package_update_thread.daemon = True
check_package_update_thread.start() check_package_update_thread.start()
result = ScriptInfo.get(log=self.log) # do not request requirements, because it might be a long process, and we first want to update the git repo
result, script_requirements = ScriptInfo.get(log=self.log, create_requirements=False)
for msg in result.warning_messages: for msg in result.warning_messages:
self.get_logger().report_text(msg) self.get_logger().report_text(msg)
@ -212,9 +215,19 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
# overwrite it before we have a chance to call edit) # overwrite it before we have a chance to call edit)
self._edit(script=result.script) self._edit(script=result.script)
self.reload() self.reload()
self._update_requirements(result.script.get('requirements') if result.script and # if jupyter is present, requirements will be created in the background, when saving a snapshot
result.script.get('requirements') else '') if result.script and script_requirements:
check_package_update_thread.join() requirements = script_requirements.get_requirements()
if requirements:
if not result.script['requirements']:
result.script['requirements'] = {}
result.script['requirements']['pip'] = requirements
self._update_requirements(result.script.get('requirements') or '')
self.reload()
# we do not want to wait for the check version thread,
# because someone might wait for us to finish the repo detection update
except Exception as e: except Exception as e:
get_logger('task').debug(str(e)) get_logger('task').debug(str(e))