ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update docstrings

Browse Source
This commit is contained in:
allegroai 2020-04-13 18:58:39 +03:00
parent 648779380c
commit 4b9c5c235c
5 changed files with 1364 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
trains/backend_interface/task/access.py
View File

@ -57,8 +57,8 @@ class AccessMixin(object):
return self._get_task_property('execution.parameters') return self._get_task_property('execution.parameters')
def get_label_num_description(self): def get_label_num_description(self):
""" Get a dict of label number to a string representing all labels associated with this number on the """ Get a dictionary of label number to a string pairs representing all labels associated with this number
model labels on the model labels.
""" """
model_labels = self._get_task_property('execution.model_labels') model_labels = self._get_task_property('execution.model_labels')
label_getter = operator.itemgetter(0) label_getter = operator.itemgetter(0)

181
trains/backend_interface/task/task.py
View File

@ -77,7 +77,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
:param task_name: Optional task name, used only if a new task is created. :param task_name: Optional task name, used only if a new task is created.
:type project_name: str :type project_name: str
:param task_type: Optional task type, used only if a new task is created. Default is training task. :param task_type: Optional task type, used only if a new task is created. Default is training task.
:type project_name: str (see tasks.TaskTypeEnum) :type task_type: str (see tasks.TaskTypeEnum)
:param log_to_backend: If True, all calls to the task's log will be logged to the backend using the API. :param log_to_backend: If True, all calls to the task's log will be logged to the backend using the API.
This value can be overridden using the environment variable TRAINS_LOG_TASK_TO_BACKEND. This value can be overridden using the environment variable TRAINS_LOG_TASK_TO_BACKEND.
:type log_to_backend: bool :type log_to_backend: bool
@ -332,12 +332,15 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
@property @property
def cache_dir(self): def cache_dir(self):
""" Cache dir used to store task related files """ """ The cache directory which is used to store the Task related files. """
return Path(get_cache_dir()) / self.id return Path(get_cache_dir()) / self.id
@property @property
def status(self): def status(self):
""" The task's status. In order to stay updated, we always reload the task info when this value is accessed. """ """
The Task's status. To keep the Task updated, Trains reloads the Task information when this value
is accessed.
"""
self.reload() self.reload()
return self._status return self._status
@ -433,26 +436,26 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self.reload() self.reload()
def started(self, ignore_errors=True): def started(self, ignore_errors=True):
""" Signal that this task has started """ """ The signal that this Task started. """
return self.send(tasks.StartedRequest(self.id), ignore_errors=ignore_errors) return self.send(tasks.StartedRequest(self.id), ignore_errors=ignore_errors)
def stopped(self, ignore_errors=True): def stopped(self, ignore_errors=True):
""" Signal that this task has stopped """ """ The signal that this Task stopped. """
return self.send(tasks.StoppedRequest(self.id), ignore_errors=ignore_errors) return self.send(tasks.StoppedRequest(self.id), ignore_errors=ignore_errors)
def completed(self, ignore_errors=True): def completed(self, ignore_errors=True):
""" Signal that this task has been completed """ """ The signal indicating that this Task completed. """
if hasattr(tasks, 'CompletedRequest'): if hasattr(tasks, 'CompletedRequest'):
return self.send(tasks.CompletedRequest(self.id, status_reason='completed'), ignore_errors=ignore_errors) return self.send(tasks.CompletedRequest(self.id, status_reason='completed'), ignore_errors=ignore_errors)
return self.send(tasks.StoppedRequest(self.id, status_reason='completed'), ignore_errors=ignore_errors) return self.send(tasks.StoppedRequest(self.id, status_reason='completed'), ignore_errors=ignore_errors)
def mark_failed(self, ignore_errors=True, status_reason=None, status_message=None): def mark_failed(self, ignore_errors=True, status_reason=None, status_message=None):
""" Signal that this task has stopped """ """ The signal that this Task stopped. """
return self.send(tasks.FailedRequest(self.id, status_reason=status_reason, status_message=status_message), return self.send(tasks.FailedRequest(self.id, status_reason=status_reason, status_message=status_message),
ignore_errors=ignore_errors) ignore_errors=ignore_errors)
def publish(self, ignore_errors=True): def publish(self, ignore_errors=True):
""" Signal that this task will be published """ """ The signal that this Task will be published """
if str(self.status) != str(tasks.TaskStatusEnum.stopped): if str(self.status) != str(tasks.TaskStatusEnum.stopped):
raise ValueError("Can't publish, Task is not stopped") raise ValueError("Can't publish, Task is not stopped")
resp = self.send(tasks.PublishRequest(self.id), ignore_errors=ignore_errors) resp = self.send(tasks.PublishRequest(self.id), ignore_errors=ignore_errors)
@ -460,7 +463,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
return resp return resp
def update_model_desc(self, new_model_desc_file=None): def update_model_desc(self, new_model_desc_file=None):
""" Change the task's model_desc """ """ Change the Task's model description. """
with self._edit_lock: with self._edit_lock:
self.reload() self.reload()
execution = self._get_task_property('execution') execution = self._get_task_property('execution')
@ -476,18 +479,19 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def update_output_model(self, model_uri, name=None, comment=None, tags=None): def update_output_model(self, model_uri, name=None, comment=None, tags=None):
""" """
Update the task's output model. Update the Task's output model. Use this method to update the output model when you have a local model URI,
Note that this method only updates the model's metadata using the API and does not upload any data. Use this for example, storing the weights file locally, and specifying a ``file://path/to/file`` URI)
method to update the output model when you have a local model URI (e.g. storing the weights file locally and
providing a file://path/to/file URI)
:param model_uri: URI for the updated model weights file .. important::
This method only updates the model's metadata using the API. It does not upload any data.
:param model_uri: The URI of the updated model weights file.
:type model_uri: str :type model_uri: str
:param name: Optional updated model name :param name: The updated model name. (Optional)
:type name: str :type name: str
:param comment: Optional updated model description :param comment: The updated model description. (Optional)
:type comment: str :type comment: str
:param tags: Optional updated model tags :param tags: The updated model tags. (Optional)
:type tags: [str] :type tags: [str]
""" """
self._conditionally_start_task() self._conditionally_start_task()
@ -496,26 +500,31 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def update_output_model_and_upload( def update_output_model_and_upload(
self, model_file, name=None, comment=None, tags=None, async_enable=False, cb=None, iteration=None): self, model_file, name=None, comment=None, tags=None, async_enable=False, cb=None, iteration=None):
""" """
Update the task's output model weights file. File is first uploaded to the preconfigured output destination (see Update the Task's output model weights file. First, Trains uploads the file to the preconfigured output
task's output.destination property or call setup_upload()), than the model object associated with the task is destination (see the Task's ``output.destination`` property or call the ``setup_upload()`` method),
updated using an API call with the URI of the uploaded file (and other values provided by additional arguments) then Trains updates the model object associated with the Task an API call. The API call uses with the URI
of the uploaded file, and other values provided by additional arguments.
:param model_file: Path to the updated model weights file :param model_file: The path to the updated model weights file.
:type model_file: str :type model_file: str
:param name: Optional updated model name :param name: The updated model name. (Optional)
:type name: str :type name: str
:param comment: Optional updated model description :param comment: The updated model description. (Optional)
:type comment: str :type comment: str
:param tags: Optional updated model tags :param tags: The updated model tags. (Optional)
:type tags: [str] :type tags: [str]
:param async_enable: Request asynchronous upload. If False, the call blocks until upload is completed and the :param async_enable: Request asynchronous upload?
API call updating the model returns. If True, the call returns immediately, while upload and update are
scheduled in another thread. Default is False. - ``True`` - The API call returns immediately, while the upload and update are scheduled in another thread.
- ``False`` - The API call blocks until the upload completes, and the API call updating the model returns.
(Default)
:type async_enable: bool :type async_enable: bool
:param cb: Asynchronous callback. If async=True, this callback will be invoked once the asynchronous upload and :param cb: Asynchronous callback. A callback. If ``async_enable`` is set to ``True``, this is a callback that
update have completed. is invoked once the asynchronous upload and update complete.
:return: The URI of the uploaded weights file. If async=True, this is the expected URI as the upload is
probably still in progress. :return: The URI of the uploaded weights file. If ``async_enable`` is set to ``True``, this is the expected URI,
as the upload is probably still in progress.
""" """
self._conditionally_start_task() self._conditionally_start_task()
uri = self.output_model.update_for_task_and_upload( uri = self.output_model.update_for_task_and_upload(
@ -544,13 +553,22 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def set_input_model(self, model_id=None, model_name=None, update_task_design=True, update_task_labels=True): def set_input_model(self, model_id=None, model_name=None, update_task_design=True, update_task_labels=True):
""" """
Set a new input model for this task. Model must be 'ready' in order to be used as the Task's input model. Set a new input model for the Task. The model must be "ready" (status is ``Published``) to be used as the
Task's input model.
:param model_id: ID for a model that exists in the backend. Required if model_name is not provided. :param model_id: The Id of the model on the **Trains Server** (backend). If ``model_name`` is not specified,
:param model_name: Model name. Required if model_id is not provided. If provided, this name will be used to then ``model_id`` must be specified.
locate an existing model in the backend. :param model_name: The model name. The name is used to locate an existing model in the **Trains Server**
:param update_task_design: if True, the task's model design will be copied from the input model (backend). If ``model_id`` is not specified, then ``model_name`` must be specified.
:param update_task_labels: if True, the task's label enumeration will be copied from the input model :param update_task_design: Update the Task's design?
- ``True`` - Trains copies the Task's model design from the input model.
- ``False`` - Trains does not copy the Task's model design from the input model.
:param update_task_labels: Update the Task's label enumeration?
- ``True`` - Trains copies the Task's label enumeration from the input model.
- ``False`` - Trains does not copy the Task's label enumeration from the input model.
""" """
if model_id is None and not model_name: if model_id is None and not model_name:
raise ValueError('Expected one of [model_id, model_name]') raise ValueError('Expected one of [model_id, model_name]')
@ -596,12 +614,12 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def set_parameters(self, *args, **kwargs): def set_parameters(self, *args, **kwargs):
""" """
Set parameters for this task. This allows setting a complete set of key/value parameters, but does not support Set the parameters for a Task. This method sets a complete group of key-value parameter pairs, but does not
parameter descriptions (as the input is a dictionary or key/value pairs. support parameter descriptions (the input is a dictionary of key-value pairs).
:param args: Positional arguments (one or more dictionary or (key, value) iterable). These will be merged into :param args: Positional arguments, which are one or more dictionary or (key, value) iterable. They are
a single key/value dictionary. merged into a single key-value pair dictionary.
:param kwargs: Key/value pairs, merged into the parameters dictionary created from `args`. :param kwargs: Key-value pairs, merged into the parameters dictionary created from ``args``.
""" """
if not all(isinstance(x, (dict, Iterable)) for x in args): if not all(isinstance(x, (dict, Iterable)) for x in args):
raise ValueError('only dict or iterable are supported as positional arguments') raise ValueError('only dict or iterable are supported as positional arguments')
@ -641,11 +659,14 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def set_parameter(self, name, value, description=None): def set_parameter(self, name, value, description=None):
""" """
Set a single task parameter. This overrides any previous value for this parameter. Set a single Task parameter. This overrides any previous value for this parameter.
:param name: Parameter name :param name: The parameter name.
:param value: Parameter value :param value: The parameter value.
:param description: Parameter description (unused for now) :param description: The parameter description.
.. note::
The ``description`` is not yet in use.
""" """
self.set_parameters({name: value}, __update=True) self.set_parameters({name: value}, __update=True)
@ -662,14 +683,12 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def update_parameters(self, *args, **kwargs): def update_parameters(self, *args, **kwargs):
""" """
Update parameters for this task. Update the parameters for a Task. This method updates a complete group of key-value parameter pairs, but does
not support parameter descriptions (the input is a dictionary of key-value pairs).
This allows updating a complete set of key/value parameters,but does not support :param args: Positional arguments, which are one or more dictionary or (key, value) iterable. They are
parameter descriptions (as the input is a dictionary or key/value pairs. merged into a single key-value pair dictionary.
:param kwargs: Key-value pairs, merged into the parameters dictionary created from ``args``.
:param args: Positional arguments (one or more dictionary or (key, value) iterable). These will be merged into
a single key/value dictionary.
:param kwargs: Key/value pairs, merged into the parameters dictionary created from `args`.
""" """
self.set_parameters(__update=True, *args, **kwargs) self.set_parameters(__update=True, *args, **kwargs)
@ -700,7 +719,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
""" """
Set the base docker image for this experiment Set the base docker image for this experiment
If provided, this value will be used by trains-agent to execute this experiment If provided, this value will be used by trains-agent to execute this experiment
inside the provided docker image. inside the provided docker image.
""" """
with self._edit_lock: with self._edit_lock:
self.reload() self.reload()
@ -709,7 +728,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self._edit(execution=execution) self._edit(execution=execution)
def get_base_docker(self): def get_base_docker(self):
"""Get the base docker command (image) set for this experiment""" """Get the base Docker command (image) that is set for this experiment."""
return self._get_task_property('execution.docker_cmd', raise_on_error=False, log_on_error=False) return self._get_task_property('execution.docker_cmd', raise_on_error=False, log_on_error=False)
def set_artifacts(self, artifacts_list=None): def set_artifacts(self, artifacts_list=None):
@ -741,7 +760,8 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def get_labels_enumeration(self): def get_labels_enumeration(self):
""" """
Return a dictionary of labels (text) to ids (integers) {str(label): integer(id)} Get the label enumeration dictionary label enumeration dictionary of string (label) to integer (value) pairs.
:return: dict :return: dict
""" """
if not self.data or not self.data.execution: if not self.data or not self.data.execution:
@ -750,7 +770,8 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def get_model_design(self): def get_model_design(self):
""" """
Returns the model configuration as blob of text Get the model configuration as blob of text.
:return: :return:
""" """
design = self._get_task_property("execution.model_desc", default={}, raise_on_error=False, log_on_error=False) design = self._get_task_property("execution.model_desc", default={}, raise_on_error=False, log_on_error=False)
@ -808,9 +829,9 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def set_name(self, name): def set_name(self, name):
""" """
Set a comment text to the task. Set the Task name.
:param name: The name of the task :param name: The name of the Task.
:type name: str :type name: str
""" """
self._set_task_property("name", str(name)) self._set_task_property("name", str(name))
@ -818,9 +839,9 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def set_comment(self, comment): def set_comment(self, comment):
""" """
Set a comment text to the task. Set a comment / description for the Task.
:param comment: The comment of the task :param comment: The comment / description for the Task.
:type comment: str :type comment: str
""" """
self._set_task_property("comment", str(comment)) self._set_task_property("comment", str(comment))
@ -828,7 +849,16 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def set_initial_iteration(self, offset=0): def set_initial_iteration(self, offset=0):
""" """
Set initial iteration, instead of zero. Useful when continuing training from previous checkpoints Set the initial iteration offset. The default value is ``0``. This method is useful when continuing training
from previous checkpoints.
For example, to start on iteration 100000, including scalars and plots:
..code-block:: py
task.set_initial_iteration(100000)
Task.set_initial_iteration(100000)
:param int offset: Initial iteration (at starting point) :param int offset: Initial iteration (at starting point)
:return: newly set initial offset :return: newly set initial offset
@ -843,10 +873,12 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
def get_initial_iteration(self): def get_initial_iteration(self):
""" """
Return the initial iteration offset, default is 0. Get the initial iteration offset. The default value is ``0``. This method is useful when continuing training
Useful when continuing training from previous checkpoints. from previous checkpoints.
:return int: initial iteration offset :return: The initial iteration offset.
:rtype: int
""" """
return self._initial_iteration_offset return self._initial_iteration_offset
@ -1016,16 +1048,23 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
@classmethod @classmethod
def get_all(cls, session=None, log=None, **kwargs): def get_all(cls, session=None, log=None, **kwargs):
""" """
List all tasks based on specific projection List all the Tasks based on specific projection.
:param session: Session object used for sending requests to the API :param session: The session object used for sending requests to the API.
:type session: Session :type session: Session
:param log: Log object :param log: The Log object.
:type log: logging.Logger :type log: logging.Logger
:param kwargs: Keyword args passed to the GetAllRequest (see .backend_api.services.tasks.GetAllRequest) :param kwargs: Keyword args passed to the GetAllRequest (see :class:`.backend_api.services.v2_5.tasks.GetAllRequest`)
Example: status='completed', 'search_text'='specific_word', 'user'='user_id', 'project'='project_id'
For example:
.. code-block:: bash
status='completed', 'search_text'='specific_word', 'user'='user_id', 'project'='project_id'
:type kwargs: dict :type kwargs: dict
:return: API response
:return: The API response.
""" """
session = session if session else cls._get_default_session() session = session if session else cls._get_default_session()
req = tasks.GetAllRequest(**kwargs) req = tasks.GetAllRequest(**kwargs)

535
trains/logger.py
View File

@ -29,11 +29,25 @@ warnings.filterwarnings('always', category=DeprecationWarning, module=__name__)
class Logger(object): class Logger(object):
""" """
Console log and metric statistics interface. The ``Logger`` class is the Trains console log and metric statistics interface, and contains methods for explicit
reporting.
Explicit reporting extends Trains automagical capturing of inputs and output. Explicit reporting
methods include scalar plots, line plots, histograms, confusion matrices, 2D and 3D scatter
diagrams, text logging, tables, and image uploading and reporting.
In the **Trains Web-App (UI)**, ``Logger`` output appears in the **RESULTS** tab, **LOG**, **SCALARS**,
**PLOTS**, and **DEBUG IMAGES** sub-tabs. When you compare experiments, ``Logger`` output appears in the
comparisons.
.. warning::
Do not construct Logger objects directly.
You must get a Logger object before calling any of the other ``Logger`` class methods by calling
:meth:`.Task.get_logger` or :meth:`Logger.current_logger`.
This is how we send graphs/plots/text to the system, later we can compare the performance of different tasks.
**Usage:** :func:`Logger.current_logger` or :func:`Task.get_logger`
""" """
SeriesInfo = SeriesInfo SeriesInfo = SeriesInfo
_tensorboard_logging_auto_group_scalars = False _tensorboard_logging_auto_group_scalars = False
@ -41,9 +55,9 @@ class Logger(object):
def __init__(self, private_task): def __init__(self, private_task):
""" """
**Do not construct Logger manually!** .. warning::
**Do not construct Logger manually!**
please use :func:`Logger.current_logger` Please use :meth:`Logger.get_current`
""" """
assert isinstance(private_task, _Task), \ assert isinstance(private_task, _Task), \
'Logger object cannot be instantiated externally, use Logger.current_logger()' 'Logger object cannot be instantiated externally, use Logger.current_logger()'
@ -61,9 +75,17 @@ class Logger(object):
def current_logger(cls): def current_logger(cls):
# type: () -> Logger # type: () -> Logger
""" """
Return a logger object for the current task. Can be called from anywhere in the code Get the Logger object for the main execution Task, the current running Task, if one exists. If no Logger object
exists, this method creates one and returns it. Therefore, you can call this method from anywhere
in the code.
:return: Singleton Logger object for the current running task .. code-block:: py
logger = Logger.current_logger()
:return: The Logger object (a singleton) for the current running Task.
:rtype: Logger object
""" """
from .task import Task from .task import Task
task = Task.current_task() task = Task.current_task()
@ -73,22 +95,45 @@ class Logger(object):
def report_text(self, msg, level=logging.INFO, print_console=True, *args, **_): def report_text(self, msg, level=logging.INFO, print_console=True, *args, **_):
""" """
print text to log and optionally also prints to console For explicit reporting, print text to the log. Optionally, print a log level and print to the console.
:param str msg: text to print to the console (always send to the backend and displayed in console) For example:
:param int level: logging level, default: logging.INFO
:param bool print_console: If True we also print 'msg' to console .. code-block:: py
logger.report_text('log some text', level=logging.DEBUG, print_console=False)
You can view the reported text in the **Trains Web-App (UI)**, **RESULTS** tab, **LOG** sub-tab.
:param str msg: The text to log.
:param int level: The log level from the Python ``logging`` package. The default value is ``logging.INFO``.
:param bool print_console: In addition to the log, print to the console?
The values are:
- ``True`` - Print to the console. (Default)
- ``False`` - Do not print to the console.
""" """
return self._console(msg, level, not print_console, *args, **_) return self._console(msg, level, not print_console, *args, **_)
def report_scalar(self, title, series, value, iteration): def report_scalar(self, title, series, value, iteration):
""" """
Report a scalar value For explicit reporting, plot a scalar series.
:param str title: Title (AKA metric) For example, plot a scalar series:
:param str series: Series (AKA variant)
:param float value: Reported value .. code-block:: py
:param int iteration: Iteration number
scalar_series = [random.randint(0,10) for i in range(10)]
logger.report_scalar(title='scalar metrics','series', value=scalar_series[iteration], iteration=0)
You can view the scalar plots in the **Trains Web-App (UI)**, **RESULTS** tab, **SCALARS** sub-tab.
:param str title: The title of the plot. Plot more than one scalar series on the same plot by using
the same ``title`` for each call to this method.
:param str series: The title of the series.
:param float value: The value to plot per iteration.
:param int iteration: The iteration number. Iterations are on the x-axis.
""" """
# if task was not started, we have to start it # if task was not started, we have to start it
@ -99,16 +144,29 @@ class Logger(object):
def report_vector(self, title, series, values, iteration, labels=None, xlabels=None, def report_vector(self, title, series, values, iteration, labels=None, xlabels=None,
xaxis=None, yaxis=None): xaxis=None, yaxis=None):
""" """
Report a histogram plot For explicit reporting, plot a vector as (stacked) histogram.
:param str title: Title (AKA metric) For example:
:param str series: Series (AKA variant)
:param list(float) values: Reported values (or numpy array) .. code-block:: py
:param int iteration: Iteration number
:param list(str) labels: optional, labels for each bar group. vector_series = np.random.randint(10, size=10).reshape(2,5)
:param list(str) xlabels: optional label per entry in the vector (bucket in the histogram) logger.report_vector(title='vector example', series='vector series', values=vector_series, iteration=0,
:param str xaxis: optional x-axis title labels=['A','B'], xaxis='X axis label', yaxis='Y axis label')
:param str yaxis: optional y-axis title
You can view the vectors plots in the **Trains Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.
:param str title: The title of the plot.
:param str series: The title of the series.
:param list(float) values: The series values. A list of floats, or an N-dimensional Numpy array containing
data for each histogram bar.
:type values: list(float), numpy.ndarray
:param int iteration: The iteration number. Each ``iteration`` creates another plot.
:param list(str) labels: Labels for each bar group, creating a plot legend labeling each series. (Optional)
:param list(str) xlabels: Labels per entry in each bucket in the histogram (vector), creating a set of labels
for each histogram bar on the x-axis. (Optional)
:param str xaxis: The x-axis title. (Optional)
:param str yaxis: The y-axis title. (Optional)
""" """
self._touch_title_series(title, series) self._touch_title_series(title, series)
return self.report_histogram(title, series, values, iteration, labels=labels, xlabels=xlabels, return self.report_histogram(title, series, values, iteration, labels=labels, xlabels=xlabels,
@ -117,16 +175,29 @@ class Logger(object):
def report_histogram(self, title, series, values, iteration, labels=None, xlabels=None, def report_histogram(self, title, series, values, iteration, labels=None, xlabels=None,
xaxis=None, yaxis=None): xaxis=None, yaxis=None):
""" """
Report a histogram plot For explicit reporting, plot a (stacked) histogram.
:param str title: Title (AKA metric) For example:
:param str series: Series (AKA variant)
:param list(float) values: Reported values (or numpy array) .. code-block:: py
:param int iteration: Iteration number
:param list(str) labels: optional, labels for each bar group. vector_series = np.random.randint(10, size=10).reshape(2,5)
:param list(str) xlabels: optional label per entry in the vector (bucket in the histogram) logger.report_vector(title='histogram example', series='histogram series', values=vector_series, iteration=0,
:param str xaxis: optional x-axis title labels=['A','B'], xaxis='X axis label', yaxis='Y axis label')
:param str yaxis: optional y-axis title
You can view the reported histograms in the **Trains Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.
:param str title: The title of the plot.
:param str series: The title of the series.
:param list(float) values: The series values. A list of floats, or an N-dimensional Numpy array containing
data for each histogram bar.
:type values: list(float), numpy.ndarray
:param int iteration: The iteration number. Each ``iteration`` creates another plot.
:param list(str) labels: Labels for each bar group, creating a plot legend labeling each series. (Optional)
:param list(str) xlabels: Labels per entry in each bucket in the histogram (vector), creating a set of labels
for each histogram bar on the x-axis. (Optional)
:param str xaxis: The x-axis title. (Optional)
:param str yaxis: The y-axis title. (Optional)
""" """
if not isinstance(values, np.ndarray): if not isinstance(values, np.ndarray):
@ -148,24 +219,36 @@ class Logger(object):
def report_table(self, title, series, iteration, table_plot=None, csv=None, url=None): def report_table(self, title, series, iteration, table_plot=None, csv=None, url=None):
""" """
Report a table plot. For explicit report, report a table plot.
:param title: Title (AKA metric) One and only one of the following parameters must be provided.
:type title: str
:param series: Series (AKA variant) - ``table_plot`` - Pandas DataFrame
:type series: str - ``csv`` - CSV file
:param iteration: Iteration number - ``url`` - URL to CSV file
:type iteration: int
For example:
.. code-block:: py
df = pd.DataFrame({'num_legs': [2, 4, 8, 0],
'num_wings': [2, 0, 0, 0],
'num_specimen_seen': [10, 2, 1, 8]},
index=['falcon', 'dog', 'spider', 'fish'])
logger.report_table(title='table example',series='pandas DataFrame',iteration=0,table_plot=df)
You can view the reported tables in the **Trains Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.
:param str title: The title of the table.
:param str series: The title of the series.
:param int iteration: The iteration number.
:param table_plot: The output table plot object :param table_plot: The output table plot object
:type table_plot: pandas.DataFrame :type table_plot: pandas.DataFrame
:param csv: path to local csv file :param csv: path to local csv file
:type csv: str :type csv: str
:param url: A URL to the location of csv file. :param url: A URL to the location of csv file.
:type url: str :type url: str
.. note::
:paramref:`~.Logger.report_table.table_plot`, :paramref:`~.Logger.report_table.csv`
and :paramref:`~.Logger.report_table.url' are mutually exclusive, and at least one must be provided.
""" """
mutually_exclusive( mutually_exclusive(
UsageError, _check_none=True, UsageError, _check_none=True,
@ -192,16 +275,30 @@ class Logger(object):
def report_line_plot(self, title, series, iteration, xaxis, yaxis, mode='lines', def report_line_plot(self, title, series, iteration, xaxis, yaxis, mode='lines',
reverse_xaxis=False, comment=None): reverse_xaxis=False, comment=None):
""" """
Report a (possibly multiple) line plot. For explicit reporting, plot one or more series as lines.
:param str title: Title (AKA metric) :param str title: The title of the plot.
:param list(LineSeriesInfo) series: All the series' data, one for each line in the plot. :param list(LineSeriesInfo) series: All the series data, one list element for each line
:param int iteration: Iteration number in the plot.
:param str xaxis: optional x-axis title :param int iteration: The iteration number.
:param str yaxis: optional y-axis title :param str xaxis: The x-axis title. (Optional)
:param str mode: scatter plot with 'lines'/'markers'/'lines+markers' :param str yaxis: The y-axis title. (Optional)
:param bool reverse_xaxis: If true X axis will be displayed from high to low (reversed) :param str mode: The type of line plot.
:param str comment: comment underneath the title
The values are:
- ``lines`` (default)
- ``markers``
- ``lines+markers``
:param bool reverse_xaxis: Reverse the x-axis?
The values are:
- ``True`` - The x-axis is high to low (reversed).
- ``False`` - The x-axis is low to high (not reversed). (Default)
:param str comment: A comment displayed with the plot, underneath the title.
""" """
series = [self.SeriesInfo(**s) if isinstance(s, dict) else s for s in series] series = [self.SeriesInfo(**s) if isinstance(s, dict) else s for s in series]
@ -223,17 +320,47 @@ class Logger(object):
def report_scatter2d(self, title, series, scatter, iteration, xaxis=None, yaxis=None, labels=None, def report_scatter2d(self, title, series, scatter, iteration, xaxis=None, yaxis=None, labels=None,
mode='lines', comment=None): mode='lines', comment=None):
""" """
Report a 2d scatter graph (with lines) For explicit reporting, report a 2d scatter plot.
:param str title: Title (AKA metric) For example:
:param str series: Series (AKA variant)
:param np.ndarray scatter: A scattered data: list of (pairs of x,y) (or numpy array) .. code-block:: py
:param int iteration: Iteration number
:param str xaxis: optional x-axis title scatter2d = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
:param str yaxis: optional y-axis title logger.report_scatter2d(title='example_scatter', series='series', iteration=0, scatter=scatter2d,
:param list(str) labels: label (text) per point in the scatter (in the same order) xaxis="title x', yaxis="title y")
:param str mode: scatter plot with 'lines'/'markers'/'lines+markers'
:param str comment: comment underneath the title Plot multiple 2D scatter series on the same plot by passing the same ``title`` and ``iteration`` values
to this method:
.. code-block:: py
scatter2d_1 = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
logger.report_scatter2d(title='example_scatter', series='series_1', iteration=1, scatter=scatter2d_1,
xaxis="title x', yaxis="title y")
scatter2d_2 = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
logger.report_scatter2d("example_scatter', "series_2', iteration=1, scatter=scatter2d_2,
xaxis="title x', yaxis="title y")
:param str title: The title of the plot.
:param str series: The title of the series.
:param scatter: The scatter data.
:type numpy.ndarray, list of (pairs of x,y) scatter:
:param int iteration: The iteration number. To set an initial iteration, for example to continue a previously
:param str xaxis: The x-axis title. (Optional)
:param str yaxis: The y-axis title. (Optional)
:param list(str) labels: Labels per point in the data assigned to the ``scatter`` parameter. The labels must be
in the same order as the data.
:param str mode: The type of scatter plot.
The values are:
- ``lines``
- ``markers``
- ``lines+markers``
:param str comment: A comment displayed with the plot, underneath the title.
""" """
if not isinstance(scatter, np.ndarray): if not isinstance(scatter, np.ndarray):
@ -259,20 +386,42 @@ class Logger(object):
def report_scatter3d(self, title, series, scatter, iteration, xaxis=None, yaxis=None, zaxis=None, def report_scatter3d(self, title, series, scatter, iteration, xaxis=None, yaxis=None, zaxis=None,
labels=None, mode='markers', fill=False, comment=None): labels=None, mode='markers', fill=False, comment=None):
""" """
Report a 3d scatter graph (with markers) For explicit reporting, plot a 3d scatter graph (with markers).
:param str title: Title (AKA metric) :param str title: The title of the plot.
:param str series: Series (AKA variant) :param str series: The title of the series.
:param Union[np.ndarray, list] scatter: A scattered data: list of (pairs of x,y,z) (or numpy array) :param Union[numpy.ndarray, list] scatter: The scatter data.
or list of series [[(x1,y1,z1)...]] :type scatter: list of (pairs of x,y,z), list of series [[(x1,y1,z1)...]], or numpy.ndarray
:param int iteration: Iteration number :param int iteration: The iteration number.
:param str xaxis: optional x-axis title :param str xaxis: The x-axis title. (Optional)
:param str yaxis: optional y-axis title :param str yaxis: The y-axis title. (Optional)
:param str zaxis: optional z-axis title :param str zaxis: The z-axis title. (Optional)
:param list(str) labels: label (text) per point in the scatter (in the same order) :param list(str) labels: Labels per point in the data assigned to the ``scatter`` parameter. The labels must be
:param str mode: scatter plot with 'lines'/'markers'/'lines+markers' in the same order as the data.
:param bool fill: fill area under the curve :param str mode: The type of scatter plot.
:param str comment: comment underneath the title
The values are:
- ``lines``
- ``markers``
- ``lines+markers``
For example:
.. code-block:: py
scatter3d = np.random.randint(10, size=(10, 3))
logger.report_scatter3d(title='example_scatter_3d', series='series_xyz', iteration=1, scatter=scatter3d,
xaxis='title x', yaxis="title y', zaxis="title z")
:param bool fill: Fill the area under the curve?
The values are:
- ``True`` - Fill
- ``False`` - Do not fill (Default)
:param str comment: A comment displayed with the plot, underneath the title.
""" """
# check if multiple series # check if multiple series
multi_series = ( multi_series = (
@ -317,17 +466,25 @@ class Logger(object):
def report_confusion_matrix(self, title, series, matrix, iteration, xaxis=None, yaxis=None, def report_confusion_matrix(self, title, series, matrix, iteration, xaxis=None, yaxis=None,
xlabels=None, ylabels=None, comment=None): xlabels=None, ylabels=None, comment=None):
""" """
Report a heat-map matrix For explicit reporting, plot a heat-map matrix.
:param str title: Title (AKA metric) For example:
:param str series: Series (AKA variant)
:param np.ndarray matrix: A heat-map matrix (example: confusion matrix) .. code-block:: py
:param int iteration: Iteration number
:param str xaxis: optional x-axis title confusion = np.random.randint(10, size=(10, 10))
:param str yaxis: optional y-axis title logger.report_confusion_matrix("example confusion matrix', "ignored', iteration=1, matrix=confusion,
:param list(str) xlabels: optional label per column of the matrix xaxis='title X', yaxis='title Y")
:param list(str) ylabels: optional label per row of the matrix
:param str comment: comment underneath the title :param str title: The title of the plot.
:param str series: The title of the series.
:param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
:param int iteration: The iteration number.
:param str xaxis: The x-axis title. (Optional)
:param str yaxis: The y-axis title. (Optional)
:param list(str) xlabels: Labels for each column of the matrix. (Optional)
:param list(str) ylabels: Labels for each row of the matrix. (Optional)
:param str comment: A comment displayed with the plot, underneath the title.
""" """
if not isinstance(matrix, np.ndarray): if not isinstance(matrix, np.ndarray):
@ -350,17 +507,19 @@ class Logger(object):
def report_matrix(self, title, series, matrix, iteration, xaxis=None, yaxis=None, xlabels=None, ylabels=None): def report_matrix(self, title, series, matrix, iteration, xaxis=None, yaxis=None, xlabels=None, ylabels=None):
""" """
Same as report_confusion_matrix For explicit reporting, plot a confusion matrix.
Report a heat-map matrix
:param str title: Title (AKA metric) .. note::
:param str series: Series (AKA variant) This method is the same as :meth:`Logger.report_confusion_matrix`.
:param np.ndarray matrix: A heat-map matrix (example: confusion matrix)
:param int iteration: Iteration number :param str title: The title of the plot.
:param str xaxis: optional x-axis title :param str series: The title of the series.
:param str yaxis: optional y-axis title :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
:param list(str) xlabels: optional label per column of the matrix :param int iteration: The iteration number.
:param list(str) ylabels: optional label per row of the matrix :param str xaxis: The x-axis title. (Optional)
:param str yaxis: The y-axis title. (Optional)
:param list(str) xlabels: Labels for each column of the matrix. (Optional)
:param list(str) ylabels: Labels for each row of the matrix. (Optional)
""" """
self._touch_title_series(title, series) self._touch_title_series(title, series)
return self.report_confusion_matrix(title, series, matrix, iteration, return self.report_confusion_matrix(title, series, matrix, iteration,
@ -369,19 +528,29 @@ class Logger(object):
def report_surface(self, title, series, matrix, iteration, xaxis=None, yaxis=None, zaxis=None, def report_surface(self, title, series, matrix, iteration, xaxis=None, yaxis=None, zaxis=None,
xlabels=None, ylabels=None, camera=None, comment=None): xlabels=None, ylabels=None, camera=None, comment=None):
""" """
Report a 3d surface (same data as heat-map matrix, only presented differently) For explicit reporting, report a 3d surface plot.
:param str title: Title (AKA metric) .. note::
:param str series: Series (AKA variant) This method plots the same data as :meth:`Logger.report_confusion_matrix`, but presents the
:param np.ndarray matrix: A heat-map matrix (example: confusion matrix) data as a surface diagram not a confusion matrix.
:param int iteration: Iteration number
:param str xaxis: optional x-axis title .. code-block:: py
:param str yaxis: optional y-axis title
:param str zaxis: optional z-axis title surface_matrix = np.random.randint(10, size=(10, 10))
:param list(str) xlabels: optional label per column of the matrix logger.report_surface("example surface', "series', iteration=0, matrix=surface_matrix,
:param list(str) ylabels: optional label per row of the matrix xaxis='title X', yaxis='title Y', zaxis="title Z")
:param list(float) camera: X,Y,Z camera position. def: (1,1,1)
:param str comment: comment underneath the title :param str title: The title of the plot.
:param str series: The title of the series.
:param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
:param int iteration: The iteration number.
:param str xaxis: The x-axis title. (Optional)
:param str yaxis: The y-axis title. (Optional)
:param str zaxis: The z-axis title. (Optional)
:param list(str) xlabels: Labels for each column of the matrix. (Optional)
:param list(str) ylabels: Labels for each row of the matrix. (Optional)
:param list(float) camera: X,Y,Z coordinates indicating the camera position. The default value is ``(1,1,1)``.
:param str comment: A comment displayed with the plot, underneath the title.
""" """
if not isinstance(matrix, np.ndarray): if not isinstance(matrix, np.ndarray):
@ -407,27 +576,50 @@ class Logger(object):
def report_image(self, title, series, iteration, local_path=None, image=None, matrix=None, max_image_history=None, def report_image(self, title, series, iteration, local_path=None, image=None, matrix=None, max_image_history=None,
delete_after_upload=False, url=None): delete_after_upload=False, url=None):
""" """
Report an image and upload its contents. For explicit reporting, report an image and upload its contents.
Image is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename) This method uploads the image to a preconfigured bucket (see :meth:`Logger.setup_upload`) with a key (filename)
describing the task ID, title, series and iteration. describing the task ID, title, series and iteration.
.. note:: For example:
:paramref:`~.Logger.report_image.local_path`, :paramref:`~.Logger.report_image.url`, :paramref:`~.Logger.report_image.image` and :paramref:`~.Logger.report_image.matrix`
are mutually exclusive, and at least one must be provided.
:param str title: Title (AKA metric) .. code-block:: py
:param str series: Series (AKA variant)
:param int iteration: Iteration number matrix = np.eye(256, 256, dtype=np.uint8)*255
matrix = np.concatenate((np.atleast_3d(matrix), np.zeros((256, 256, 2), dtype=np.uint8)), axis=2)
logger.report_image("test case', "image color red', iteration=1, image=m)
image_open = Image.open(os.path.join("<image_path>', "<image_filename>"))
logger.report_image("test case', "image PIL', iteration=1, image=image_open)
One and only one of the following parameters must be provided.
- :paramref:`~.Logger.report_image.local_path`
- :paramref:`~.Logger.report_image.url`
- :paramref:`~.Logger.report_image.image`
- :paramref:`~.Logger.report_image.matrix`
:param str title: The title of the image.
:param str series: The title of the series of this image.
:param int iteration: The iteration number.
:param str local_path: A path to an image file. :param str local_path: A path to an image file.
:param str url: A URL to the location of a pre-uploaded image. :param str url: A URL for the location of a pre-uploaded image.
:param np.ndarray or PIL.Image.Image image: Could be a PIL.Image.Image object or a 3D numpy.ndarray :param image: Image data (RGB).
object containing image data (RGB). :type image: numpy.ndarray, PIL.Image.Image
:param np.ndarray matrix: A 3D numpy.ndarray object containing image data (RGB). :param numpy.ndarray matrix: Image data (RGB).
This is deprecated, use image variable instead.
:param int max_image_history: maximum number of image to store per metric/variant combination .. note::
use negative value for unlimited. default is set in global configuration (default=5) The ``matrix`` paramater is deprecated. Use the ``image`` parameters.
:param bool delete_after_upload: if True, one the file was uploaded the local copy will be deleted :type matrix: 3D numpy.ndarray
:param int max_image_history: The maximum number of images to store per metric/variant combination.
For an unlimited number, use a negative value. The default value is set in global configuration
(default=``5``).
:param bool delete_after_upload: After the upload, delete the local copy of the image?
The values are:
- ``True`` - Delete after upload.
- ``False`` - Do not delete after upload. (Default)
""" """
mutually_exclusive( mutually_exclusive(
UsageError, _check_none=True, UsageError, _check_none=True,
@ -544,15 +736,22 @@ class Logger(object):
def set_default_upload_destination(self, uri): def set_default_upload_destination(self, uri):
""" """
Set the uri to upload all the debug images to. Set the destination storage URI (for example, S3, Google Cloud Storage, a file path) for uploading debug images.
Images are uploaded separately to the destination storage (e.g. s3,gc,file) and then The images are uploaded separately. A link to each image is reported.
a link to the uploaded image is sent in the report
Notice: credentials for the upload destination will be pooled from the .. note::
global configuration file (i.e. ~/trains.conf) Credentials for the destination storage are specified in the Trains configuration file,
``~/trains.conf``.
:param str uri: example: 's3://bucket/directory/' or 'file:///tmp/debug/' :param str uri: example: 's3://bucket/directory/' or 'file:///tmp/debug/'
:return: True if destination scheme is supported (i.e. s3:// file:// gc:// etc...)
:return: bool
The values are:
- ``True`` - The destination scheme is supported (for example, ``s3://``, ``file://``, or ``gc://``).
- ``False`` - The destination scheme is not supported.
""" """
# Create the storage helper # Create the storage helper
@ -565,14 +764,14 @@ class Logger(object):
def get_default_upload_destination(self): def get_default_upload_destination(self):
""" """
Get the uri to upload all the debug images to. Get the destination storage URI (for example, S3, Google Cloud Storage, a file path) for uploading debug images
(see :meth:`Logger.set_default_upload_destination`).
Images are uploaded separately to the destination storage (e.g. s3,gc,file) and then :return: The default upload destination URI.
a link to the uploaded image is sent in the report
Notice: credentials for the upload destination will be pooled from the
global configuration file (i.e. ~/trains.conf)
:return: Uri (str) example: 's3://bucket/directory/' or 'file:///tmp/debug/' etc... For example, ``s3://bucket/directory/`` or ``file:///tmp/debug/``.
:rtype: str
""" """
return self._default_upload_destination or self._task._get_default_report_storage_uri() return self._default_upload_destination or self._task._get_default_report_storage_uri()
@ -580,7 +779,12 @@ class Logger(object):
""" """
Flush cached reports and console outputs to backend. Flush cached reports and console outputs to backend.
:return: True if successful :return: bool
The values are:
- ``True`` - Successfully flushed the cache.
- ``False`` - Failed.
""" """
self._flush_stdout_handler() self._flush_stdout_handler()
if self._task: if self._task:
@ -589,7 +793,11 @@ class Logger(object):
def get_flush_period(self): def get_flush_period(self):
""" """
:return: logger flush period in seconds Get the Logger flush period.
:return: The logger flush period in seconds.
:rtype: int
""" """
if self._flusher: if self._flusher:
return self._flusher.period return self._flusher.period
@ -597,10 +805,10 @@ class Logger(object):
def set_flush_period(self, period): def set_flush_period(self, period):
""" """
Set the period of the logger flush. Set the logger flush period.
:param float period: The period to flush the logger in seconds. If None or 0, :param float period: The period to flush the logger in seconds. To set no periodic flush,
There will be no periodic flush. specify ``None`` or ``0``.
""" """
if self._task.is_main_task() and DevWorker.report_stdout and DevWorker.report_period and \ if self._task.is_main_task() and DevWorker.report_stdout and DevWorker.report_period and \
not running_remotely() and period is not None: not running_remotely() and period is not None:
@ -619,7 +827,8 @@ class Logger(object):
def report_image_and_upload(self, title, series, iteration, path=None, matrix=None, max_image_history=None, def report_image_and_upload(self, title, series, iteration, path=None, matrix=None, max_image_history=None,
delete_after_upload=False): delete_after_upload=False):
""" """
Deprecated: Backwards compatibility, please use report_image instead .. deprecated:: 0.13.0
Use :meth:`Logger.report_image` instead
""" """
self.report_image(title=title, series=series, iteration=iteration, local_path=path, image=matrix, self.report_image(title=title, series=series, iteration=iteration, local_path=path, image=matrix,
max_image_history=max_image_history, delete_after_upload=delete_after_upload) max_image_history=max_image_history, delete_after_upload=delete_after_upload)
@ -627,17 +836,32 @@ class Logger(object):
@classmethod @classmethod
def tensorboard_auto_group_scalars(cls, group_scalars=False): def tensorboard_auto_group_scalars(cls, group_scalars=False):
""" """
If `group_scalars` set to True, we preserve backward compatible Tensorboard auto-magic behaviour, Group together TensorBoard scalars that do not have a title, or assign a title/series with the same tag.
i.e. Scalars without specific title will be grouped under the "Scalars" graph.
Default is False: Tensorboard scalars without title will have title/series with the same tag :param group_scalars: Group TensorBoard scalars without a title?
The values are:
- ``True`` - Scalars without specific titles are grouped together in the "Scalars" plot, preserving
backward compatibility with Trains automagical behavior.
- ``False`` - TensorBoard scalars without titles get a title/series with the same tag. (Default)
:type group_scalars: bool
""" """
cls._tensorboard_logging_auto_group_scalars = group_scalars cls._tensorboard_logging_auto_group_scalars = group_scalars
@classmethod @classmethod
def tensorboard_single_series_per_graph(cls, single_series=False): def tensorboard_single_series_per_graph(cls, single_series=False):
""" """
If `single_series` set to True, we generate a separate graph (plot) for each Tensorboard scalar series Group TensorBoard scalar series together or in separate plots.
Default is False: Tensorboard scalar series will be grouped according to their title
:param single_series: Group TensorBoard scalar series together?
The values are:
- ``True`` - Generate a separate plot for each TensorBoard scalar series.
- ``False`` - Group the TensorBoard scalar series together in the same plot. (Default)
:type single_series: bool
""" """
cls._tensorboard_logging_single_series_per_graphs = single_series cls._tensorboard_logging_single_series_per_graphs = single_series
@ -649,9 +873,14 @@ class Logger(object):
""" """
print text to log (same as print to console, and also prints to console) print text to log (same as print to console, and also prints to console)
:param msg: text to print to the console (always send to the backend and displayed in console) :param str msg: text to print to the console (always send to the backend and displayed in console)
:param level: logging level, default: logging.INFO :param level: logging level, default: logging.INFO
:param omit_console: If True we only send 'msg' to log (no console print) :type level: Logging Level
:param bool omit_console: Omit the console output, and only send the ``msg`` value to the log?
- ``True`` - Omit the console output.
- ``False`` - Print the console output. (Default)
""" """
try: try:
level = int(level) level = int(level)
@ -700,7 +929,7 @@ class Logger(object):
""" """
Report an image, upload its contents, and present in plots section using plotly Report an image, upload its contents, and present in plots section using plotly
Image is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename) Image is uploaded to a preconfigured bucket (see :meth:`Logger.setup_upload`) with a key (filename)
describing the task ID, title, series and iteration. describing the task ID, title, series and iteration.
:param title: Title (AKA metric) :param title: Title (AKA metric)
@ -747,7 +976,7 @@ class Logger(object):
""" """
Upload a file and report it as link in the debug images section. Upload a file and report it as link in the debug images section.
File is uploaded to a preconfigured storage (see setup_upload()) with a key (filename) File is uploaded to a preconfigured storage (see :meth:`Logger.setup_upload`) with a key (filename)
describing the task ID, title, series and iteration. describing the task ID, title, series and iteration.
:param title: Title (AKA metric) :param title: Title (AKA metric)

481
trains/model.py
View File

@ -99,91 +99,108 @@ class BaseModel(object):
@property @property
def id(self): def id(self):
""" """
return the id of the model (string) The Id (system UUID) of the model.
:return: model id (string) :return: The model id.
:rtype: str
""" """
return self._get_model_data().id return self._get_model_data().id
@property @property
def name(self): def name(self):
""" """
return the name of the model (string) The name of the model.
:return: model name (string) :return: The model name.
:rtype: str
""" """
return self._get_model_data().name return self._get_model_data().name
@name.setter @name.setter
def name(self, value): def name(self, value):
""" """
Update the model name Set the model name.
:param value: model name (string) :param str value: The model name.
""" """
self._get_base_model().update(name=value) self._get_base_model().update(name=value)
@property @property
def comment(self): def comment(self):
""" """
return comment/description of the model (string) The comment for the model. Also, use for a model description.
:return: model description (string) :return: The model comment / description.
:rtype: str
""" """
return self._get_model_data().comment return self._get_model_data().comment
@comment.setter @comment.setter
def comment(self, value): def comment(self, value):
""" """
Update the model comment/description of the model (string) Set comment for the model. Also, use for a model description.
:param value: model comment/description (string) :param str value: The model comment/description.
""" """
self._get_base_model().update(comment=value) self._get_base_model().update(comment=value)
@property @property
def tags(self): def tags(self):
""" """
Return the list of tags the model has A list of tags describing the model.
:return: list of strings (tags) :return: The list of tags.
:rtype: list(str)
""" """
return self._get_model_data().tags return self._get_model_data().tags
@tags.setter @tags.setter
def tags(self, value): def tags(self, value):
""" """
Update the model list of tags (list of strings) Set the list of tags describing the model.
:param value: list of strings as tags :param value: The tags.
:type value: list(str)
""" """
self._get_base_model().update(tags=value) self._get_base_model().update(tags=value)
@property @property
def config_text(self): def config_text(self):
""" """
returns a string representing the model configuration (from prototxt to ini file or python code to evaluate) The configuration as a string. For example, prototxt, an ini file, or Python code to evaluate.
:return: string :return: The configuration.
:rtype: str
""" """
return _Model._unwrap_design(self._get_model_data().design) return _Model._unwrap_design(self._get_model_data().design)
@property @property
def config_dict(self): def config_dict(self):
""" """
returns a configuration dictionary parsed from the design text, The configuration as a dictionary, parsed from the design text. This usually represents the model configuration.
usually representing the model configuration (from prototxt to ini file or python code to evaluate) For example, prototxt, an ini file, or Python code to evaluate.
:return: Dictionary :return: The configuration.
:rtype: dict
""" """
return self._text_to_config_dict(self.config_text) return self._text_to_config_dict(self.config_text)
@property @property
def labels(self): def labels(self):
""" """
Return the labels enumerator {str(label): integer(id)} as saved in the model object The label enumeration of string (label) to integer (value) pairs.
:return: labels_dict, dictionary with labels (text) keys and values as integers
:return: A dictionary containing labels enumeration, where the keys are labels and the values as integers.
:rtype: dict
""" """
return self._get_model_data().labels return self._get_model_data().labels
@ -221,20 +238,28 @@ class BaseModel(object):
def get_weights(self): def get_weights(self):
""" """
Download the base model and returns a string of locally stored filename Download the base model and return the locally stored filename.
:return: string to locally stored file :return: The locally stored file.
:rtype: str
""" """
# download model (synchronously) and return local file # download model (synchronously) and return local file
return self._get_base_model().download_model_weights() return self._get_base_model().download_model_weights()
def get_weights_package(self, return_path=False): def get_weights_package(self, return_path=False):
""" """
Download the base model package, extract the files and return list of locally stored filenames Download the base model package into a temporary directory (extract the files), or return a list of the
locally stored filenames.
:param return_path: if True the model weights are downloaded into a :param bool return_path: Return the model weights or a list of filenames? (Optional)
temporary directory and the directory path is returned, instead of list of files
:return: string to locally stored file - ``True`` - Download the model weights into a temporary directory, and return the temporary directory path.
- ``False`` - Return a list of the locally stored filenames. (Default)
:return: The model weights, or a list of the locally stored filenames.
:rtype: package or path
""" """
# check if model was packaged # check if model was packaged
if self._package_tag not in self._get_model_data().tags: if self._package_tag not in self._get_model_data().tags:
@ -266,9 +291,8 @@ class BaseModel(object):
def publish(self): def publish(self):
""" """
Set the model to 'published' and set it for public use. Set the model to the status ``published`` and for public use. If the model's status is already ``published``,
then this method is a no-op.
If the model is already published, this method is a no-op.
""" """
if not self.published: if not self.published:
@ -412,31 +436,72 @@ class InputModel(Model):
framework=None, framework=None,
): ):
""" """
Create a model from pre-existing model file (link must be valid), and model configuration. Create an InputModel object from a pre-trained model by specifying the URL of an initial weight files.
Optionally, input a configuration, label enumeration, name for the model, tags describing the model,
comment as a description of the model, indicate whether the model is a package, specify the model's
framework, and indicate whether to immediately set the model's status to ``Published``.
The model is read-only.
If the url to the weights file already exists, the import process will stop with a warning The **Trains Server** (backend) may already store the model's URL. If the input model's URL is not
and automatically it will try to import the model that was found. stored, meaning the model is new, then it is imported and Trains stores its metadata.
The Model will be read-only and can be used to pre initialize a network If the URL is already stored, the import process stops, Trains issues a warning message, and Trains
We can connect the model to a task as input model, then when running remotely override it with the UI. reuses the model.
Load model based on id, returned object is read-only and can be connected to a task
That is, we can override the input model when running remotely
:param weights_url: valid url for the weights file (string). In your Python experiment script, after importing the model, you can connect it to the main execution
examples: "https://domain.com/file.bin" or "s3://bucket/file.bin" or "file:///home/user/file.bin". Task as an input model using :meth:`InputModel.connect` or :meth:`.Task.connect`. That initializes the
NOTE: if a model with the exact same URL exists, it will be used, and all other arguments will be ignored. network.
:param config_text: model configuration (unconstrained text string). usually the content of
configuration file. If `config_text` is not None, `config_dict` must not be provided. .. note::
:param config_dict: model configuration parameters (dict). Using the **Trains Web-App** (user interface), you can reuse imported models and switch models in
If `config_dict` is not None, `config_text` must not be provided. experiments.
:param label_enumeration: dictionary of string to integer, enumerating the model output to labels
example: {'background': 0 , 'person': 1} :param str weights_url: A valid URL for the initial weights file. If the **Trains Web-App** (backend)
:param name: optional, name for the newly imported model already stores the metadata of a model with the same URL, that existing model is returned
:param tags: optional, list of strings as tags and Trains ignores all other parameters.
:param comment: optional, string description for the model
:param is_package: Boolean. Indicates that the imported weights file is a package. For example:
If True, and a new model was created, a package tag will be added.
:param create_as_published: Boolean. If True, and a new model is created, it will be published. - ``https://domain.com/file.bin``
:param framework: optional, string name of the framework of the model or Framework - ``s3://bucket/file.bin``
- ``file:///home/user/file.bin``
:param str config_text: The configuration as a string. This is usually the content of a configuration
dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
:type config_text: unconstrained text string
:param dict config_dict: The configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
but not both.
:param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs. (Optional)
For example:
.. code-block:: javascript
{
'background': 0,
'person': 1
}
:param str name: The name of the newly imported model. (Optional)
:param tags: The list of tags which describe the model. (Optional)
:type tags: list(str)
:param str comment: A comment / description for the model. (Optional)
:type comment str:
:param is_package: Is the imported weights file is a package? (Optional)
- ``True`` - Is a package. Add a package tag to the model.
- ``False`` - Is not a package. Do not add a package tag. (Default)
:type is_package: bool
:param bool create_as_published: Set the model's status to Published? (Optional)
- ``True`` - Set the status to Published.
- ``False`` - Do not set the status to Published. The status will be Draft. (Default)
:param str framework: The framework of the model. (Optional)
:type framework: str or Framework object
:return: The imported model or existing model (see above).
:rtype: A model object.
""" """
config_text = cls._resolve_config(config_text=config_text, config_dict=config_dict) config_text = cls._resolve_config(config_text=config_text, config_dict=config_dict)
weights_url = StorageHelper.conform_url(weights_url) weights_url = StorageHelper.conform_url(weights_url)
@ -567,15 +632,24 @@ class InputModel(Model):
label_enumeration=None, label_enumeration=None,
): ):
""" """
Create an empty model, so that later we can execute the task in remote and Create an empty model object. Later, you can assign a model to the empty model object.
replace the empty model with pre-trained model file
:param config_text: model configuration (unconstrained text string). usually the content of a config_dict file. :param config_text: The model configuration as a string. This is usually the content of a configuration
If `config_text` is not None, `config_dict` must not be provided. dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
:param config_dict: model configuration parameters (dict). :type config_text: unconstrained text string
If `config_dict` is not None, `config_text` must not be provided. :param dict config_dict: The model configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
:param label_enumeration: dictionary of string to integer, enumerating the model output to labels but not both.
example: {'background': 0 , 'person': 1} :param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs.
(Optional)
For example:
.. code-block:: javascript
{
'background': 0,
'person': 1
}
""" """
design = cls._resolve_config(config_text=config_text, config_dict=config_dict) design = cls._resolve_config(config_text=config_text, config_dict=config_dict)
@ -591,11 +665,8 @@ class InputModel(Model):
def __init__(self, model_id): def __init__(self, model_id):
""" """
Load model based on id, returned object is read-only and can be connected to a task :param str model_id: The Trains Id (system UUID) of the input model whose metadata the **Trains Server**
(backend) stores.
Notice, we can override the input model when running remotely
:param model_id: id (string)
""" """
super(InputModel, self).__init__(model_id) super(InputModel, self).__init__(model_id)
@ -605,16 +676,22 @@ class InputModel(Model):
def connect(self, task): def connect(self, task):
""" """
Connect current model with a specific task, only supported for preexisting models, Connect the current model to a Task object, if the model is preexisting. Preexisting models include:
i.e. not supported on objects created with create_and_connect() - Imported models (InputModel objects created using the :meth:`Logger.import_model` method).
When running in debug mode (i.e. locally), the task is updated with the model object - Models whose metadata is already in the Trains platform, meaning the InputModel object is instantiated
(i.e. task input model is the load_model_id) from the ``InputModel`` class specifying the the model's Trains Id as an argument.
When running remotely (i.e. from a daemon) the model is being updated from the task - Models whose origin is not Trains that are used to create an InputModel object. For example,
Notice! when running remotely the load_model_id is ignored and loaded from the task object models created using TensorFlow models.
regardless of the code
:param task: Task object When the experiment is executed remotely in a worker, the input model already specified in the experiment is
used.
.. note::
The **Trains Web-App** allows you to switch one input model for another and then enqueue the experiment
to execute in a worker.
:param object task: A Task object.
""" """
self._set_task(task) self._set_task(task)
@ -640,13 +717,23 @@ class InputModel(Model):
class OutputModel(BaseModel): class OutputModel(BaseModel):
""" """
Create an output model for a task to store the training results in. Create an output model for a Task (experiment) to store the training results.
By definition the Model is always connected to a task, and is automatically registered as its output model. The OutputModel object is always connected to a Task object, because it is instantiated with a Task object
The common use case is reusing the model object, and overriding the weights every stored snapshot. as an argument. It is, therefore, automatically registered as the Task's (experiment's) output model.
A user can create multiple output models for a task, think a snapshot after a validation test has a new high-score.
The Model will be read-write and if config/label-enumeration are None, The OutputModel object is read-write.
their values will be initialized from the task input model.
A common use case is to reuse the OutputModel object, and override the weights after storing a model snapshot.
Another use case is to create multiple OutputModel objects for a Task (experiment), and after a new high score
is found, store a model snapshot.
If the model configuration and / or the model's label enumeration
are ``None``, then the output model is initialized with the values from the Task object's input model.
.. note::
When executing a Task (experiment) remotely in a worker, you can modify the model configuration and / or model's
label enumeration using the **Trains Web-App**.
""" """
@property @property
@ -658,49 +745,78 @@ class OutputModel(BaseModel):
@property @property
def config_text(self): def config_text(self):
""" """
returns a string representing the model configuration (from prototxt to ini file or python code to evaluate) Get the configuration as a string. For example, prototxt, an ini file, or Python code to evaluate.
:return: string :return: The configuration.
:rtype: str
""" """
return _Model._unwrap_design(self._get_model_data().design) return _Model._unwrap_design(self._get_model_data().design)
@config_text.setter @config_text.setter
def config_text(self, value): def config_text(self, value):
""" """
Update the model configuration, store a blob of text for custom usage Set the configuration. Store a blob of text for custom usage.
""" """
self.update_design(config_text=value) self.update_design(config_text=value)
@property @property
def config_dict(self): def config_dict(self):
""" """
returns a configuration dictionary parsed from the config_text text, Get the configuration as a dictionary parsed from the ``config_text`` text. This usually represents the model
usually representing the model configuration (from prototxt to ini file or python code to evaluate) configuration. For example, from prototxt to ini file or python code to evaluate.
:return: Dictionary :return: The configuration.
:rtype: dict
""" """
return self._text_to_config_dict(self.config_text) return self._text_to_config_dict(self.config_text)
@config_dict.setter @config_dict.setter
def config_dict(self, value): def config_dict(self, value):
""" """
Update the model configuration: model configuration parameters (dict). Set the configuration. Saved in the model object.
:param dict value: The configuration parameters.
""" """
self.update_design(config_dict=value) self.update_design(config_dict=value)
@property @property
def labels(self): def labels(self):
""" """
Return the labels enumerator {str(label): integer(id)} as saved in the model object Get the label enumeration as a dictionary of string (label) to integer (value) pairs.
:return: labels_dict, dictionary with labels (text) keys and values as integers For example:
.. code-block:: javascript
{
'background': 0,
'person': 1
}
:return: The label enumeration.
:rtype: dict
""" """
return self._get_model_data().labels return self._get_model_data().labels
@labels.setter @labels.setter
def labels(self, value): def labels(self, value):
""" """
update the labels enumerator {str(label): integer(id)} as saved in the model object Set the label enumeration.
:param dict value: The label enumeration dictionary of string (label) to integer (value) pairs.
For example:
.. code-block:: javascript
{
'background': 0,
'person': 1
}
""" """
self.update_labels(labels=value) self.update_labels(labels=value)
@ -726,19 +842,30 @@ class OutputModel(BaseModel):
We do not allow for Model creation without a task, so we always keep track on how we created the models We do not allow for Model creation without a task, so we always keep track on how we created the models
In remote execution, Model parameters can be overridden by the Task (such as model configuration & label enumerator) In remote execution, Model parameters can be overridden by the Task (such as model configuration & label enumerator)
:param task: Task object :param task: The Task object with which the OutputModel object is associated.
:type task: Task :type task: Task
:param config_text: model configuration (unconstrained text string). usually the content of a config_dict file. :param config_text: The configuration as a string. This is usually the content of a configuration
If `config_text` is not None, `config_dict` must not be provided. dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
:param config_dict: model configuration parameters (dict). :type config_text: unconstrained text string
If `config_dict` is not None, `config_text` must not be provided. :param dict config_dict: The configuration as a dictionary.
:param label_enumeration: dictionary of string to integer, enumerating the model output to labels Specify ``config_dict`` or ``config_text``, but not both.
example: {'background': 0 , 'person': 1} :param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs.
:type label_enumeration: dict[str: int] or None (Optional)
:param name: optional, name for the newly created model
:param tags: optional, list of strings as tags For example:
:param comment: optional, string description for the model
:param framework: optional, string name of the framework of the model or Framework .. code-block:: javascript
{
'background': 0,
'person': 1
}
:param str name: The name for the newly created model. (Optional)
:param list(str) tags: A list of strings which are tags for the model. (Optional)
:param str comment: A comment / description for the model. (Optional)
:param framework: The framework of the model or a Framework object. (Optional)
:type framework: str or Framework object
:param base_model_id: optional, model id to be reused :param base_model_id: optional, model id to be reused
""" """
super(OutputModel, self).__init__(task=task) super(OutputModel, self).__init__(task=task)
@ -782,16 +909,13 @@ class OutputModel(BaseModel):
def connect(self, task): def connect(self, task):
""" """
Connect current model with a specific task, only supported for preexisting models, Connect the current model to a Task object, if the model is a preexisting model. Preexisting models include:
i.e. not supported on objects created with create_and_connect() - Imported models.
When running in debug mode (i.e. locally), the task is updated with the model object - Models whose metadata the **Trains Server** (backend) is already storing.
(i.e. task input model is the load_model_id) - Models from another source, such as frameworks like TensorFlow.
When running remotely (i.e. from a daemon) the model is being updated from the task
Notice! when running remotely the load_model_id is ignored and loaded from the task object
regardless of the code
:param task: Task object :param object task: A Task object.
""" """
if self._task != task: if self._task != task:
raise ValueError('Can only connect preexisting model to task, but this is a fresh model') raise ValueError('Can only connect preexisting model to task, but this is a fresh model')
@ -824,15 +948,28 @@ class OutputModel(BaseModel):
def set_upload_destination(self, uri): def set_upload_destination(self, uri):
""" """
Set the uri to upload all the model weight files to. Set the URI of the storage destination for uploaded model weight files. Supported storage destinations include
S3, Google Cloud Storage), and file locations.
Files are uploaded separately to the destination storage (e.g. s3,gc,file) and then Using this method, files uploads are separate and then a link to each is stored in the model object.
a link to the uploaded model is stored in the model object
Notice: credentials for the upload destination will be pooled from the
global configuration file (i.e. ~/trains.conf)
:param uri: upload destination (string). example: 's3://bucket/directory/' or 'file:///tmp/debug/' .. note::
:return: True if destination scheme is supported (i.e. s3:// file:// gc:// etc...) For storage requiring credentials, the credentials are stored in the Trains configuration file,
``~/trains.conf``.
:param str uri: The URI of the upload storage destination.
For example:
- ``s3://bucket/directory/``
- ``file:///tmp/debug/``
:return: The status of whether the storage destination schema is supported.
- ``True`` - The storage destination scheme is supported.
- ``False`` - The storage destination scheme is not supported.
:rtype: bool
""" """
if not uri: if not uri:
return return
@ -857,15 +994,31 @@ class OutputModel(BaseModel):
""" """
Update the model weights from a locally stored model filename. Update the model weights from a locally stored model filename.
Uploading the model is a background process, the call returns immediately. .. note::
Uploading the model is a background process. A call to this method returns immediately.
:param weights_filename: locally stored filename to be uploaded as is :param str weights_filename: The name of the locally stored weights file to upload. Specify ``weights_filename``
:param upload_uri: destination uri for model weights upload (default: previously used uri) or ``register_uri``, but not both.
:param target_filename: the newly created filename in the destination uri location (default: weights_filename) :param str upload_uri: The URI of the storage destination for model weights upload. The default value
:param auto_delete_file: delete temporary file after uploading is the previously used URI. (Optional)
:param register_uri: register an already uploaded weights file (uri must be valid) :param str target_filename: The newly created filename in the storage destination location. The default value
:param update_comment: if True, model comment will be updated with local weights file location (provenance) is the ``weights_filename`` value. (Optional)
:return: uploaded uri :param bool auto_delete_file: Delete the temporary file after uploading? (Optional)
- ``True`` - Delete (Default)
- ``False`` - Do not delete
:param str register_uri: The URI of an already uploaded weights file. The URI must be valid. Specify
``register_uri`` or ``weights_filename``, but not both.
:param bool update_comment: Update the model comment with the local weights file name (to maintain
provenance)? (Optional)
- ``True`` - Update model comment (Default)
- ``False`` - Do not update
:return: The uploaded URI.
:rtype: str
""" """
def delete_previous_weights_file(filename=weights_filename): def delete_previous_weights_file(filename=weights_filename):
@ -961,18 +1114,29 @@ class OutputModel(BaseModel):
def update_weights_package(self, weights_filenames=None, weights_path=None, upload_uri=None, def update_weights_package(self, weights_filenames=None, weights_path=None, upload_uri=None,
target_filename=None, auto_delete_file=True, iteration=None): target_filename=None, auto_delete_file=True, iteration=None):
""" """
Update the model weights from a locally stored model files (or directory containing multiple files). Update the model weights from locally stored model files, or from directory containing multiple files.
Uploading the model is a background process, the call returns immediately. .. note::
Uploading the model weights is a background process. A call to this method returns immediately.
:param weights_filenames: list of locally stored filenames (list of strings) :param weights_filenames: The file names of the locally stored model files. Specify ``weights_filenames``
:type weights_filenames: list or ``weights_path``, but not both.
:param weights_path: directory path to package (all the files in the directory will be uploaded) :type weights_filenames: list(str)
:param weights_path: The directory path to a package. All the files in the directory will be uploaded.
Specify ``weights_path`` or ``weights_filenames``, but not both.
:type weights_path: str :type weights_path: str
:param upload_uri: destination uri for model weights upload (default: previously used uri) :param str upload_uri: The URI of the storage destination for the model weights upload. The default
:param target_filename: the newly created filename in the destination uri location (default: weights_filename) is the previously used URI. (Optional)
:param auto_delete_file: delete temporary file after uploading :param str target_filename: The newly created filename in the storage destination URI location. The default
:return: uploaded uri for the weights package is the value specified in the ``weights_filename`` parameter. (Optional)
:param bool auto_delete_file: Delete temporary file after uploading? (Optional)
- ``True`` - Delete (Default)
- ``False`` - Do not delete
:return: The uploaded URI for the weights package.
:rtype: str
""" """
# create list of files # create list of files
if (not weights_filenames and not weights_path) or (weights_filenames and weights_path): if (not weights_filenames and not weights_path) or (weights_filenames and weights_path):
@ -1022,23 +1186,31 @@ class OutputModel(BaseModel):
def update_design(self, config_text=None, config_dict=None): def update_design(self, config_text=None, config_dict=None):
""" """
Update the model configuration, basically store a blob of text for custom usage Update the model configuration. Store a blob of text for custom usage.
Notice: this is done in a lazily, only when updating weights we force the update of configuration in the backend .. note::
This method's behavior is lazy. The design update is only forced when the weights
are updated.
:param config_text: model configuration (unconstrained text string). usually the content of a config_dict file. :param config_text: The configuration as a string. This is usually the content of a configuration
If `config_text` is not None, `config_dict` must not be provided. dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
:param config_dict: model configuration parameters (dict). :type config_text: unconstrained text string
If `config_dict` is not None, `config_text` must not be provided. :param dict config_dict: The configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
:return: True if update was successful but not both.
:return: The status of the update.
- ``True`` - Update successful.
- ``False`` - Update not successful.
:rtype: bool
""" """
if not self._validate_update(): if not self._validate_update():
return return
config_text = self._resolve_config(config_text=config_text, config_dict=config_dict) config_text = self._resolve_config(config_text=config_text, config_dict=config_dict)
if self._task and not self._task._get_model_config_text(): if self._task and not self._task.get_model_config_text():
self._task._set_model_config(config_text=config_text) self._task.set_model_config(config_text=config_text)
if self.id: if self.id:
# update the model object (this will happen if we resumed a training task) # update the model object (this will happen if we resumed a training task)
@ -1052,10 +1224,19 @@ class OutputModel(BaseModel):
def update_labels(self, labels): def update_labels(self, labels):
""" """
Update the model label enumeration {str(label): integer(id)} Update the label enumeration.
:param dict labels: The label enumeration dictionary of string (label) to integer (value) pairs.
For example:
.. code-block:: javascript
{
'background': 0,
'person': 1
}
:param labels: dictionary with labels (text) keys and values as integers
example: {'background': 0 , 'person': 1}
:return: :return:
""" """
validate_dict(labels, key_types=six.string_types, value_types=six.integer_types, desc='label enumeration') validate_dict(labels, key_types=six.string_types, value_types=six.integer_types, desc='label enumeration')
@ -1079,11 +1260,11 @@ class OutputModel(BaseModel):
@classmethod @classmethod
def wait_for_uploads(cls, timeout=None, max_num_uploads=None): def wait_for_uploads(cls, timeout=None, max_num_uploads=None):
""" """
Wait for any pending/in-progress model uploads. If no uploads are pending or in-progress, returns immediately. Wait for any pending or in-progress model uploads to complete. If no uploads are pending or in-progress,
then the ``wait_for_uploads`` returns immediately.
:param timeout: If not None, a floating point number specifying a timeout in seconds after which this call will :param float timeout: The timeout interval to wait for uploads (seconds). (Optional).
return. :param int max_num_uploads: The maximum number of uploads to wait for. (Optional).
:param max_num_uploads: Max number of uploads to wait for.
""" """
_Model.wait_for_results(timeout=timeout, max_num_uploads=max_num_uploads) _Model.wait_for_results(timeout=timeout, max_num_uploads=max_num_uploads)

736
trains/task.py
View File

File diff suppressed because it is too large Load Diff