ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-01-31 09:07:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix Pipeline self._nodes changes during iteration

Browse Source
This commit is contained in:
allegroai 2023-04-30 09:38:18 +03:00
parent e9ecc9a29b
commit 422807555d
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clearml/automation/controller.py
View File

@ -2370,14 +2370,16 @@ class PipelineController(object):
"""
jobs = []
previous_status_map = {}
for name, node in self._nodes.items():
# copy to avoid race condition
nodes = self._nodes.copy()
for name, node in nodes.items():
if not node.job:
continue
# noinspection PyProtectedMember
previous_status_map[name] = node.job._last_status
jobs.append(node.job)
BaseJob.update_status_batch(jobs)
for node in self._nodes.values():
for node in nodes.values():
self._update_node_status(node)
def _update_node_status(self, node):
@ -2465,7 +2467,9 @@ class PipelineController(object):
"""
if time() - self._last_progress_update_time < self._update_progress_interval:
return
job_progress = [(node.job.task.get_progress() or 0) if node.job else 0 for node in self._nodes.values()]
# copy to avoid race condition
nodes = self._nodes.copy()
job_progress = [(node.job.task.get_progress() or 0) if node.job else 0 for node in nodes.values()]
if len(job_progress):
self._task.set_progress(int(sum(job_progress) / len(job_progress)))
self._last_progress_update_time = time()