From 13f11a68be8fa4dfeab31c5149e8db011cb639ce Mon Sep 17 00:00:00 2001 From: Alex Burlacu Date: Sat, 8 Jul 2023 16:58:41 +0300 Subject: [PATCH] Improve pipeline caching behaviour --- clearml/automation/job.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/clearml/automation/job.py b/clearml/automation/job.py index 3dc23643..7b62f79e 100644 --- a/clearml/automation/job.py +++ b/clearml/automation/job.py @@ -416,7 +416,22 @@ class BaseJob(object): # we need to ignore `requirements` section because ir might be changing from run to run script.pop("requirements", None) - hyper_params = task.get_parameters() if params_override is None else params_override + hyper_params = deepcopy(task.get_parameters() if params_override is None else params_override) + hyper_params_to_change = {} + task_cache = {} + for key, value in hyper_params.items(): + if key.startswith("kwargs_artifacts/"): + # noinspection PyBroadException + try: + # key format is . + task_id, artifact = value.split(".", 1) + task_ = task_cache.setdefault(task_id, Task.get_task(task_id)) + # set the value of the hyper parameter to the hash of the artifact + # because the task ID might differ, but the artifact might be the same + hyper_params_to_change[key] = task_.artifacts[artifact].hash + except Exception: + pass + hyper_params.update(hyper_params_to_change) configs = task.get_configuration_objects() if configurations_override is None else configurations_override # currently we do not add the docker image to the hash (only args and setup script), # because default docker image will cause the step to change