From 41218cee9b3796dc95536a346222fd1c267198bb Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Tue, 9 Aug 2022 09:46:44 +0300 Subject: [PATCH] Fix model not created in the current project --- clearml/automation/controller.py | 5 ++++- clearml/backend_interface/model.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/clearml/automation/controller.py b/clearml/automation/controller.py index e61f0e7f..c5ba4691 100644 --- a/clearml/automation/controller.py +++ b/clearml/automation/controller.py @@ -3228,7 +3228,9 @@ class PipelineDecorator(PipelineController): else: sleep(2) continue + if _node.job.is_failed() or _node.job.is_aborted(): + # noinspection PyProtectedMember if cls._singleton._should_relaunch_node(_node): cls._singleton._task.get_logger().report_text( "Relaunching step {} on instance termination".format(_node.name) @@ -3460,7 +3462,8 @@ class PipelineDecorator(PipelineController): pipeline_kwargs[k] = a_pipeline.get_parameters()[k] # run the actual pipeline - if not start_controller_locally and not PipelineDecorator._debug_execute_step_process and pipeline_execution_queue: + if not start_controller_locally and \ + not PipelineDecorator._debug_execute_step_process and pipeline_execution_queue: # rerun the pipeline on a remote machine a_pipeline._task.execute_remotely(queue_name=pipeline_execution_queue) # when we get here it means we are running remotely diff --git a/clearml/backend_interface/model.py b/clearml/backend_interface/model.py index fa4d2423..f36b76c1 100644 --- a/clearml/backend_interface/model.py +++ b/clearml/backend_interface/model.py @@ -173,7 +173,7 @@ class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin): if self.id is None: if upload_storage_uri: self.upload_storage_uri = upload_storage_uri - self._create_empty_model(self.upload_storage_uri) + self._create_empty_model(self.upload_storage_uri, project_id=project_id) elif upload_storage_uri: self.upload_storage_uri = upload_storage_uri @@ -481,11 +481,11 @@ class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin): res = self.send(req) return res.response.id - def _create_empty_model(self, upload_storage_uri=None): + def _create_empty_model(self, upload_storage_uri=None, project_id=None): upload_storage_uri = upload_storage_uri or self.upload_storage_uri name = make_message('Anonymous model %(time)s') uri = '{}/uploading_file'.format(upload_storage_uri or 'file://') - req = models.CreateRequest(uri=uri, name=name, labels={}) + req = models.CreateRequest(uri=uri, name=name, labels={}, project=project_id) res = self.send(req) if not res: return False