Allow overriding initial iteration offset using environment variable (CLEARML_SET_ITERATION_OFFSET) or Task.init(continue_last_task==<offset>) (issue #496)

clearml/task.py

@@ -27,7 +27,7 @@ from pathlib2 import Path
 from .backend_config.defs import get_active_config_file, get_config_file
 from .backend_api.services import tasks, projects
 from .backend_api.session.session import (
-    Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST)
+    Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST, )
 from .backend_interface.metrics import Metrics
 from .backend_interface.model import Model as BackendModel
 from .backend_interface.task import Task as _Task
@@ -51,7 +51,7 @@ from .binding.hydra_bind import PatchHydra
 from .binding.click_bind import PatchClick
 from .config import (
     config, DEV_TASK_NO_REUSE, get_is_master_node, DEBUG_SIMULATE_REMOTE_TASK, DEV_DEFAULT_OUTPUT_URI,
-    deferred_config, )
+    deferred_config, TASK_SET_ITERATION_OFFSET, )
 from .config import running_remotely, get_remote_task_id
 from .config.cache import SessionCache
 from .debugging.log import LoggerRoot
@@ -60,11 +60,13 @@ from .logger import Logger
 from .model import Model, InputModel, OutputModel
 from .task_parameters import TaskParameters
 from .utilities.config import verify_basic_value
-from .binding.args import argparser_parseargs_called, get_argparser_last_args, \
-    argparser_update_currenttask
+from .binding.args import (
+    argparser_parseargs_called, get_argparser_last_args,
+    argparser_update_currenttask, )
 from .utilities.dicts import ReadOnlyDict, merge_dicts
-from .utilities.proxy_object import ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary, \
-    nested_from_flat_dictionary, naive_nested_from_flat_dictionary
+from .utilities.proxy_object import (
+    ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary,
+    nested_from_flat_dictionary, naive_nested_from_flat_dictionary, )
 from .utilities.resource_monitor import ResourceMonitor
 from .utilities.seed import make_deterministic
 from .utilities.lowlevel.threads import get_current_thread_id
@@ -72,7 +74,6 @@ from .utilities.process.mp import BackgroundMonitor, leave_process
 # noinspection PyProtectedMember
 from .backend_interface.task.args import _Arguments
 
-
 if TYPE_CHECKING:
     import pandas
     import numpy
@@ -199,7 +200,7 @@ class Task(_Task):
             task_type=TaskTypes.training,  # type: Task.TaskTypes
             tags=None,  # type: Optional[Sequence[str]]
             reuse_last_task_id=True,  # type: Union[bool, str]
-        continue_last_task=False,  # type: Union[bool, str]
+            continue_last_task=False,  # type: Union[bool, str, int]
             output_uri=None,  # type: Optional[Union[str, bool]]
             auto_connect_arg_parser=True,  # type: Union[bool, Mapping[str, bool]]
             auto_connect_frameworks=True,  # type: Union[bool, Mapping[str, bool]]
@@ -297,6 +298,9 @@ class Task(_Task):
             - ``False`` - Overwrite the execution of previous Task  (default).
             - A string - You can also specify a Task ID (string) to be continued.
                 This is equivalent to `continue_last_task=True` and `reuse_last_task_id=a_task_id_string`.
+            - An integer - Specify initial iteration offset (override the auto automatic last_iteration_offset)
+                Pass 0, to disable the automatic last_iteration_offset or specify a different initial offset
+                You can specify a Task ID to be used with `reuse_last_task_id='task_id_here'`
 
         :param str output_uri: The default location for output models and other artifacts.
             If True is passed, the default files_server will be used for model storage.
@@ -2662,6 +2666,12 @@ class Task(_Task):
         if continue_last_task and isinstance(continue_last_task, str):
             reuse_last_task_id = continue_last_task
             continue_last_task = True
+        elif isinstance(continue_last_task, int) and continue_last_task is not True:
+            # allow initial offset environment override
+            continue_last_task = continue_last_task
+
+        if TASK_SET_ITERATION_OFFSET.get() is not None:
+            continue_last_task = TASK_SET_ITERATION_OFFSET.get()
 
         # if we force no task reuse from os environment
         if DEV_TASK_NO_REUSE.get() or not reuse_last_task_id or isinstance(reuse_last_task_id, str):
@@ -2696,10 +2706,15 @@ class Task(_Task):
                     )
 
                     # instead of resting the previously used task we are continuing the training with it.
-                    if task and continue_last_task:
+                    if task and (continue_last_task or isinstance(continue_last_task, int)):
                         task.reload()
                         task.mark_started(force=True)
+                        # allow to disable the
+                        if continue_last_task is True:
                             task.set_initial_iteration(task.get_last_iteration() + 1)
+                        else:
+                            task.set_initial_iteration(continue_last_task)
+
                     else:
                         task_tags = task.data.system_tags if hasattr(task.data, 'system_tags') else task.data.tags
                         task_artifacts = task.data.execution.artifacts \