ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-24 13:17:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Handle the case where PipelineController.abort_on_failure is set to False and a step has been aborted

Browse Source
This commit is contained in:
allegroai 2024-01-18 17:12:36 +02:00
parent 1b80a77c92
commit e7ec688022
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
clearml/automation/controller.py
View File

@ -2732,7 +2732,7 @@ class PipelineController(object):
self._final_failure[node.name] = True self._final_failure[node.name] = True
completed_jobs.append(j) completed_jobs.append(j)
node.executed = node.job.task_id() if not node_failed else False node.executed = node.job.task_id() if not (node_failed or node.job.is_aborted()) else False
if j in launched_nodes: if j in launched_nodes:
launched_nodes.remove(j) launched_nodes.remove(j)
# check if we need to stop all running steps # check if we need to stop all running steps
@ -3494,7 +3494,7 @@ class PipelineDecorator(PipelineController):
else: else:
self._final_failure[node.name] = True self._final_failure[node.name] = True
completed_jobs.append(j) completed_jobs.append(j)
node.executed = node.job.task_id() if not node_failed else False node.executed = node.job.task_id() if not (node_failed or node.job.is_aborted()) else False
if j in launched_nodes: if j in launched_nodes:
launched_nodes.remove(j) launched_nodes.remove(j)
# check if we need to stop all running steps # check if we need to stop all running steps
@ -4142,7 +4142,7 @@ class PipelineDecorator(PipelineController):
return task.artifacts[return_name].get( return task.artifacts[return_name].get(
deserialization_function=cls._singleton._artifact_deserialization_function deserialization_function=cls._singleton._artifact_deserialization_function
) )
return task.get_parameters(cast=True)[CreateFromFunction.return_section + "/" + return_name] return task.get_parameters(cast=True).get(CreateFromFunction.return_section + "/" + return_name)
return_w = [LazyEvalWrapper( return_w = [LazyEvalWrapper(
callback=functools.partial(result_wrapper, n), callback=functools.partial(result_wrapper, n),
@ -4572,6 +4572,13 @@ class PipelineDecorator(PipelineController):
_node.parents = (_node.parents or []) + [ _node.parents = (_node.parents or []) + [
x for x in cls._evaluated_return_values.get(tid, []) if x in leaves x for x in cls._evaluated_return_values.get(tid, []) if x in leaves
] ]
if not cls._singleton._abort_running_steps_on_failure:
for parent in _node.parents:
if cls._singleton._nodes[parent].status in ["failed", "aborted", "skipped"]:
_node.skip_job = True
return
for k, v in kwargs.items(): for k, v in kwargs.items():
if v is None or isinstance(v, (float, int, bool, six.string_types)): if v is None or isinstance(v, (float, int, bool, six.string_types)):
_node.parameters["{}/{}".format(CreateFromFunction.kwargs_section, k)] = v _node.parameters["{}/{}".format(CreateFromFunction.kwargs_section, k)] = v