From 0b69a81c8ea63fa0d62441eb658777b46001c51a Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 24 Mar 2022 21:52:18 +0200 Subject: [PATCH] Add support for setting reported values for `NaN` and `Inf` (#604) --- clearml/backend_interface/metrics/events.py | 39 ++++++++++++++++----- clearml/logger.py | 28 ++++++++++++++- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/clearml/backend_interface/metrics/events.py b/clearml/backend_interface/metrics/events.py index 31ce33eb..12a64269 100644 --- a/clearml/backend_interface/metrics/events.py +++ b/clearml/backend_interface/metrics/events.py @@ -13,6 +13,7 @@ from six.moves.urllib.parse import urlparse, urlunparse from ...backend_api.services import events from ...config import deferred_config +from ...debugging.log import LoggerRoot from ...storage.util import quote_url from ...utilities.attrs import attrs from ...utilities.process.mp import SingletonLock @@ -25,8 +26,13 @@ class MetricsEventAdapter(object): metrics manager when batching and writing events. """ - _default_nan_value = 0. - """ Default value used when a np.nan value is encountered """ + default_nan_value = 0. + default_inf_value = 0. + """ Default value used when a np.nan or np.inf value is encountered """ + report_nan_warning_period = 1000 + report_inf_warning_period = 1000 + _report_nan_warning_iteration = float('inf') + _report_inf_warning_iteration = float('inf') @attrs(cmp=False, slots=True) class FileEntry(object): @@ -116,9 +122,27 @@ class MetricsEventAdapter(object): return res @classmethod - def _convert_np_nan(cls, val): - if np.isnan(val) or np.isinf(val): - return cls._default_nan_value + def _convert_np_nan_inf(cls, val): + if np.isnan(val): + cls._report_nan_warning_iteration += 1 + if cls._report_nan_warning_iteration >= cls.report_nan_warning_period: + LoggerRoot.get_base_logger().info( + "NaN value encountered. Reporting it as '{}'. Use clearml.Logger.set_reporting_nan_value to assign another value".format( + cls.default_nan_value + ) + ) + cls._report_nan_warning_iteration = 0 + return cls.default_nan_value + if np.isinf(val): + cls._report_inf_warning_iteration += 1 + if cls._report_inf_warning_iteration >= cls.report_inf_warning_period: + LoggerRoot.get_base_logger().info( + "inf value encountered. Reporting it as '{}'. Use clearml.Logger.set_reporting_inf_value to assign another value".format( + cls.default_inf_value + ) + ) + cls._report_inf_warning_iteration = 0 + return cls.default_inf_value return val @@ -126,7 +150,7 @@ class ScalarEvent(MetricsEventAdapter): """ Scalar event adapter """ def __init__(self, metric, variant, value, iter, **kwargs): - self._value = self._convert_np_nan(value) + self._value = self._convert_np_nan_inf(value) super(ScalarEvent, self).__init__(metric=metric, variant=variant, iter=iter, **kwargs) def get_api_event(self): @@ -157,7 +181,7 @@ class VectorEvent(MetricsEventAdapter): """ Vector event adapter """ def __init__(self, metric, variant, values, iter, **kwargs): - self._values = [self._convert_np_nan(v) for v in values] + self._values = [self._convert_np_nan_inf(v) for v in values] super(VectorEvent, self).__init__(metric=metric, variant=variant, iter=iter, **kwargs) def get_api_event(self): @@ -346,7 +370,6 @@ class UploadEvent(MetricsEventAdapter): output = None if output is None: - from ...debugging.log import LoggerRoot LoggerRoot.get_base_logger().warning( 'Skipping upload, could not find object file \'{}\''.format(self._local_image_path)) return None diff --git a/clearml/logger.py b/clearml/logger.py index 375bf4f3..d44da1da 100644 --- a/clearml/logger.py +++ b/clearml/logger.py @@ -19,7 +19,7 @@ from .backend_interface.logger import StdStreamPatch from .backend_interface.task import Task as _Task from .backend_interface.task.log import TaskHandler from .backend_interface.util import mutually_exclusive -from .backend_interface.metrics.events import UploadEvent +from .backend_interface.metrics.events import UploadEvent, MetricsEventAdapter from .config import running_remotely, get_cache_dir, config, DEBUG_SIMULATE_REMOTE_TASK, deferred_config from .errors import UsageError from .storage.helper import StorageHelper @@ -1233,6 +1233,32 @@ class Logger(object): from clearml.backend_interface.metrics import Reporter Reporter.matplotlib_force_report_non_interactive(force=force) + @classmethod + def set_reporting_nan_value(cls, value, warn_period=1000): + # type: (float, int) -> None + """ + When a NaN value is encountered, it is reported as a floating value (by default 0) and the user is warned. + This function is used to change the value NaN is converted to and the warning period. + + :param value: The value NaN is converted to + :param warn_period: Number of times NaN is encountered and converted until the next warning + """ + MetricsEventAdapter.default_nan_value = value + MetricsEventAdapter.report_nan_warning_period = warn_period + + @classmethod + def set_reporting_inf_value(cls, value, warn_period=1000): + # type: (float, int) -> None + """ + When an inf value is encountered, it is reported as a floating value (by default 0) and the user is warned. + This function is used to change the value inf is converted to and the warning period. + + :param value: The value inf is converted to + :param warn_period: Number of times inf is encountered and converted until the next warning + """ + MetricsEventAdapter.default_inf_value = value + MetricsEventAdapter.report_inf_warning_period = warn_period + @classmethod def _remove_std_logger(cls): # noinspection PyBroadException