mirror of
https://github.com/clearml/clearml
synced 2025-06-26 18:16:07 +00:00
Allow overriding initial iteration offset using environment variable (CLEARML_SET_ITERATION_OFFSET) or Task.init(continue_last_task==<offset>) (issue #496)
This commit is contained in:
parent
297f33703f
commit
fc0305728c
@ -27,7 +27,7 @@ from pathlib2 import Path
|
||||
from .backend_config.defs import get_active_config_file, get_config_file
|
||||
from .backend_api.services import tasks, projects
|
||||
from .backend_api.session.session import (
|
||||
Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST)
|
||||
Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST, )
|
||||
from .backend_interface.metrics import Metrics
|
||||
from .backend_interface.model import Model as BackendModel
|
||||
from .backend_interface.task import Task as _Task
|
||||
@ -51,7 +51,7 @@ from .binding.hydra_bind import PatchHydra
|
||||
from .binding.click_bind import PatchClick
|
||||
from .config import (
|
||||
config, DEV_TASK_NO_REUSE, get_is_master_node, DEBUG_SIMULATE_REMOTE_TASK, DEV_DEFAULT_OUTPUT_URI,
|
||||
deferred_config, )
|
||||
deferred_config, TASK_SET_ITERATION_OFFSET, )
|
||||
from .config import running_remotely, get_remote_task_id
|
||||
from .config.cache import SessionCache
|
||||
from .debugging.log import LoggerRoot
|
||||
@ -60,11 +60,13 @@ from .logger import Logger
|
||||
from .model import Model, InputModel, OutputModel
|
||||
from .task_parameters import TaskParameters
|
||||
from .utilities.config import verify_basic_value
|
||||
from .binding.args import argparser_parseargs_called, get_argparser_last_args, \
|
||||
argparser_update_currenttask
|
||||
from .binding.args import (
|
||||
argparser_parseargs_called, get_argparser_last_args,
|
||||
argparser_update_currenttask, )
|
||||
from .utilities.dicts import ReadOnlyDict, merge_dicts
|
||||
from .utilities.proxy_object import ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary, \
|
||||
nested_from_flat_dictionary, naive_nested_from_flat_dictionary
|
||||
from .utilities.proxy_object import (
|
||||
ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary,
|
||||
nested_from_flat_dictionary, naive_nested_from_flat_dictionary, )
|
||||
from .utilities.resource_monitor import ResourceMonitor
|
||||
from .utilities.seed import make_deterministic
|
||||
from .utilities.lowlevel.threads import get_current_thread_id
|
||||
@ -72,7 +74,6 @@ from .utilities.process.mp import BackgroundMonitor, leave_process
|
||||
# noinspection PyProtectedMember
|
||||
from .backend_interface.task.args import _Arguments
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pandas
|
||||
import numpy
|
||||
@ -199,7 +200,7 @@ class Task(_Task):
|
||||
task_type=TaskTypes.training, # type: Task.TaskTypes
|
||||
tags=None, # type: Optional[Sequence[str]]
|
||||
reuse_last_task_id=True, # type: Union[bool, str]
|
||||
continue_last_task=False, # type: Union[bool, str]
|
||||
continue_last_task=False, # type: Union[bool, str, int]
|
||||
output_uri=None, # type: Optional[Union[str, bool]]
|
||||
auto_connect_arg_parser=True, # type: Union[bool, Mapping[str, bool]]
|
||||
auto_connect_frameworks=True, # type: Union[bool, Mapping[str, bool]]
|
||||
@ -297,6 +298,9 @@ class Task(_Task):
|
||||
- ``False`` - Overwrite the execution of previous Task (default).
|
||||
- A string - You can also specify a Task ID (string) to be continued.
|
||||
This is equivalent to `continue_last_task=True` and `reuse_last_task_id=a_task_id_string`.
|
||||
- An integer - Specify initial iteration offset (override the auto automatic last_iteration_offset)
|
||||
Pass 0, to disable the automatic last_iteration_offset or specify a different initial offset
|
||||
You can specify a Task ID to be used with `reuse_last_task_id='task_id_here'`
|
||||
|
||||
:param str output_uri: The default location for output models and other artifacts.
|
||||
If True is passed, the default files_server will be used for model storage.
|
||||
@ -2662,6 +2666,12 @@ class Task(_Task):
|
||||
if continue_last_task and isinstance(continue_last_task, str):
|
||||
reuse_last_task_id = continue_last_task
|
||||
continue_last_task = True
|
||||
elif isinstance(continue_last_task, int) and continue_last_task is not True:
|
||||
# allow initial offset environment override
|
||||
continue_last_task = continue_last_task
|
||||
|
||||
if TASK_SET_ITERATION_OFFSET.get() is not None:
|
||||
continue_last_task = TASK_SET_ITERATION_OFFSET.get()
|
||||
|
||||
# if we force no task reuse from os environment
|
||||
if DEV_TASK_NO_REUSE.get() or not reuse_last_task_id or isinstance(reuse_last_task_id, str):
|
||||
@ -2696,10 +2706,15 @@ class Task(_Task):
|
||||
)
|
||||
|
||||
# instead of resting the previously used task we are continuing the training with it.
|
||||
if task and continue_last_task:
|
||||
if task and (continue_last_task or isinstance(continue_last_task, int)):
|
||||
task.reload()
|
||||
task.mark_started(force=True)
|
||||
# allow to disable the
|
||||
if continue_last_task is True:
|
||||
task.set_initial_iteration(task.get_last_iteration() + 1)
|
||||
else:
|
||||
task.set_initial_iteration(continue_last_task)
|
||||
|
||||
else:
|
||||
task_tags = task.data.system_tags if hasattr(task.data, 'system_tags') else task.data.tags
|
||||
task_artifacts = task.data.execution.artifacts \
|
||||
|
Loading…
Reference in New Issue
Block a user