ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix python package analysis timed out before sending git repo detials.

Browse Source
This commit is contained in:
allegroai 2019-09-27 13:23:32 +03:00
parent 3eba23f1a3
commit c44638c8d9
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
trains/backend_interface/task/repo/scriptinfo.py
View File

@ -455,9 +455,13 @@ class ScriptInfo(object):
# if this is not jupyter, get the requirements.txt
requirements = ''
# create requirements if backend supports requirements
if create_requirements and not jupyter_filepath and Session.check_min_api_version('2.2'):
# if jupyter is present, requirements will be created in the background, when saving a snapshot
if not jupyter_filepath and Session.check_min_api_version('2.2'):
script_requirements = ScriptRequirements(Path(repo_root).as_posix())
requirements = script_requirements.get_requirements()
if create_requirements:
requirements = script_requirements.get_requirements()
else:
script_requirements = None
script_info = dict(
repository=furl(repo_info.url).remove(username=True, password=True).tostr(),
@ -480,7 +484,8 @@ class ScriptInfo(object):
if not any(script_info.values()):
script_info = None
return ScriptInfoResult(script=script_info, warning_messages=messages)
return (ScriptInfoResult(script=script_info, warning_messages=messages),
script_requirements)
@classmethod
def get(cls, filepath=sys.argv[0], check_uncommitted=True, create_requirements=True, log=None):
@ -491,7 +496,7 @@ class ScriptInfo(object):
except Exception as ex:
if log:
log.warning("Failed auto-detecting task repository: {}".format(ex))
return ScriptInfoResult()
return ScriptInfoResult(), None
@attr.s

21
trains/backend_interface/task/task.py
View File

@ -8,6 +8,7 @@ from threading import RLock, Thread
import six
from six.moves.urllib.parse import quote
from ...backend_interface.task.repo.scriptinfo import ScriptRequirements
from ...backend_interface.task.development.worker import DevWorker
from ...backend_api import Session
from ...backend_api.services import tasks, models, events, projects
@ -199,11 +200,13 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
except Exception:
pass
# get repository and create requirements.txt from code base
try:
check_package_update_thread = Thread(target=check_package_update)
check_package_update_thread.daemon = True
check_package_update_thread.start()
result = ScriptInfo.get(log=self.log)
# do not request requirements, because it might be a long process, and we first want to update the git repo
result, script_requirements = ScriptInfo.get(log=self.log, create_requirements=False)
for msg in result.warning_messages:
self.get_logger().report_text(msg)
@ -212,9 +215,19 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
# overwrite it before we have a chance to call edit)
self._edit(script=result.script)
self.reload()
self._update_requirements(result.script.get('requirements') if result.script and
result.script.get('requirements') else '')
check_package_update_thread.join()
# if jupyter is present, requirements will be created in the background, when saving a snapshot
if result.script and script_requirements:
requirements = script_requirements.get_requirements()
if requirements:
if not result.script['requirements']:
result.script['requirements'] = {}
result.script['requirements']['pip'] = requirements
self._update_requirements(result.script.get('requirements') or '')
self.reload()
# we do not want to wait for the check version thread,
# because someone might wait for us to finish the repo detection update
except Exception as e:
get_logger('task').debug(str(e))