Allow overriding initial iteration offset using environment variable (CLEARML_SET_ITERATION_OFFSET) or Task.init(continue_last_task==<offset>) (issue #496)

This commit is contained in:
allegroai 2021-11-30 21:15:03 +02:00
parent 297f33703f
commit fc0305728c

View File

@ -27,7 +27,7 @@ from pathlib2 import Path
from .backend_config.defs import get_active_config_file, get_config_file
from .backend_api.services import tasks, projects
from .backend_api.session.session import (
Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST)
Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST, )
from .backend_interface.metrics import Metrics
from .backend_interface.model import Model as BackendModel
from .backend_interface.task import Task as _Task
@ -51,7 +51,7 @@ from .binding.hydra_bind import PatchHydra
from .binding.click_bind import PatchClick
from .config import (
config, DEV_TASK_NO_REUSE, get_is_master_node, DEBUG_SIMULATE_REMOTE_TASK, DEV_DEFAULT_OUTPUT_URI,
deferred_config, )
deferred_config, TASK_SET_ITERATION_OFFSET, )
from .config import running_remotely, get_remote_task_id
from .config.cache import SessionCache
from .debugging.log import LoggerRoot
@ -60,11 +60,13 @@ from .logger import Logger
from .model import Model, InputModel, OutputModel
from .task_parameters import TaskParameters
from .utilities.config import verify_basic_value
from .binding.args import argparser_parseargs_called, get_argparser_last_args, \
argparser_update_currenttask
from .binding.args import (
argparser_parseargs_called, get_argparser_last_args,
argparser_update_currenttask, )
from .utilities.dicts import ReadOnlyDict, merge_dicts
from .utilities.proxy_object import ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary, \
nested_from_flat_dictionary, naive_nested_from_flat_dictionary
from .utilities.proxy_object import (
ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary,
nested_from_flat_dictionary, naive_nested_from_flat_dictionary, )
from .utilities.resource_monitor import ResourceMonitor
from .utilities.seed import make_deterministic
from .utilities.lowlevel.threads import get_current_thread_id
@ -72,7 +74,6 @@ from .utilities.process.mp import BackgroundMonitor, leave_process
# noinspection PyProtectedMember
from .backend_interface.task.args import _Arguments
if TYPE_CHECKING:
import pandas
import numpy
@ -199,7 +200,7 @@ class Task(_Task):
task_type=TaskTypes.training, # type: Task.TaskTypes
tags=None, # type: Optional[Sequence[str]]
reuse_last_task_id=True, # type: Union[bool, str]
continue_last_task=False, # type: Union[bool, str]
continue_last_task=False, # type: Union[bool, str, int]
output_uri=None, # type: Optional[Union[str, bool]]
auto_connect_arg_parser=True, # type: Union[bool, Mapping[str, bool]]
auto_connect_frameworks=True, # type: Union[bool, Mapping[str, bool]]
@ -297,6 +298,9 @@ class Task(_Task):
- ``False`` - Overwrite the execution of previous Task (default).
- A string - You can also specify a Task ID (string) to be continued.
This is equivalent to `continue_last_task=True` and `reuse_last_task_id=a_task_id_string`.
- An integer - Specify initial iteration offset (override the auto automatic last_iteration_offset)
Pass 0, to disable the automatic last_iteration_offset or specify a different initial offset
You can specify a Task ID to be used with `reuse_last_task_id='task_id_here'`
:param str output_uri: The default location for output models and other artifacts.
If True is passed, the default files_server will be used for model storage.
@ -2662,6 +2666,12 @@ class Task(_Task):
if continue_last_task and isinstance(continue_last_task, str):
reuse_last_task_id = continue_last_task
continue_last_task = True
elif isinstance(continue_last_task, int) and continue_last_task is not True:
# allow initial offset environment override
continue_last_task = continue_last_task
if TASK_SET_ITERATION_OFFSET.get() is not None:
continue_last_task = TASK_SET_ITERATION_OFFSET.get()
# if we force no task reuse from os environment
if DEV_TASK_NO_REUSE.get() or not reuse_last_task_id or isinstance(reuse_last_task_id, str):
@ -2696,10 +2706,15 @@ class Task(_Task):
)
# instead of resting the previously used task we are continuing the training with it.
if task and continue_last_task:
if task and (continue_last_task or isinstance(continue_last_task, int)):
task.reload()
task.mark_started(force=True)
# allow to disable the
if continue_last_task is True:
task.set_initial_iteration(task.get_last_iteration() + 1)
else:
task.set_initial_iteration(continue_last_task)
else:
task_tags = task.data.system_tags if hasattr(task.data, 'system_tags') else task.data.tags
task_artifacts = task.data.execution.artifacts \