From 422807555d6e8340ffa028f8cfbcc7b2bd7e8ad7 Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Sun, 30 Apr 2023 09:38:18 +0300
Subject: [PATCH] Fix Pipeline self._nodes changes during iteration

---
 clearml/automation/controller.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/clearml/automation/controller.py b/clearml/automation/controller.py
index 06ecfbd8..1ae8c883 100644
--- a/clearml/automation/controller.py
+++ b/clearml/automation/controller.py
@@ -2370,14 +2370,16 @@ class PipelineController(object):
         """
         jobs = []
         previous_status_map = {}
-        for name, node in self._nodes.items():
+        # copy to avoid race condition
+        nodes = self._nodes.copy()
+        for name, node in nodes.items():
             if not node.job:
                 continue
             # noinspection PyProtectedMember
             previous_status_map[name] = node.job._last_status
             jobs.append(node.job)
         BaseJob.update_status_batch(jobs)
-        for node in self._nodes.values():
+        for node in nodes.values():
             self._update_node_status(node)
 
     def _update_node_status(self, node):
@@ -2465,7 +2467,9 @@ class PipelineController(object):
         """
         if time() - self._last_progress_update_time < self._update_progress_interval:
             return
-        job_progress = [(node.job.task.get_progress() or 0) if node.job else 0 for node in self._nodes.values()]
+        # copy to avoid race condition
+        nodes = self._nodes.copy()
+        job_progress = [(node.job.task.get_progress() or 0) if node.job else 0 for node in nodes.values()]
         if len(job_progress):
             self._task.set_progress(int(sum(job_progress) / len(job_progress)))
         self._last_progress_update_time = time()