diff --git a/clearml/backend_interface/task/args.py b/clearml/backend_interface/task/args.py index 7caf463f..c2089930 100644 --- a/clearml/backend_interface/task/args.py +++ b/clearml/backend_interface/task/args.py @@ -449,7 +449,7 @@ class _Arguments(object): if descriptions: descriptions = dict((prefix+k, v) for k, v in descriptions.items()) if param_types: - param_types = dict((prefix+k, v) for k, v in param_types.items()) + param_types = dict((prefix + k, v) for k, v in param_types.items()) # this will only set the specific section self._task.update_parameters( dictionary, diff --git a/clearml/backend_interface/task/task.py b/clearml/backend_interface/task/task.py index e62b2085..b6ab1c82 100644 --- a/clearml/backend_interface/task/task.py +++ b/clearml/backend_interface/task/task.py @@ -28,7 +28,7 @@ import six from six.moves.urllib.parse import quote from ...utilities.locks import RLock as FileRLock -from ...utilities.proxy_object import verify_basic_type +from ...utilities.proxy_object import verify_basic_type, cast_basic_type, get_basic_type from ...binding.artifacts import Artifacts from ...backend_interface.task.development.worker import DevWorker from ...backend_interface.session import SendError @@ -927,8 +927,8 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): self._edit(execution=self.data.execution) - def get_parameters(self, backwards_compatibility=True): - # type: (bool) -> (Optional[dict]) + def get_parameters(self, backwards_compatibility=True, cast=False): + # type: (bool, bool) -> (Optional[dict]) """ Get the parameters for a Task. This method returns a complete group of key-value parameter pairs, but does not support parameter descriptions (the result is a dictionary of key-value pairs). @@ -938,6 +938,8 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): :param backwards_compatibility: If True (default) parameters without section name (API version < 2.9, clearml-server < 0.16) will be at dict root level. If False, parameters without section name, will be nested under "Args/" key. + :param cast: If True, cast the parameter to the original type. Default False, + values are returned in their string representation :return: dict of the task parameters, all flattened to key/value. Different sections with key prefix "section/" @@ -951,14 +953,16 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): if not backwards_compatibility: for section in hyperparams: for key, section_param in hyperparams[section].items(): - parameters['{}/{}'.format(section, key)] = section_param.value + parameters['{}/{}'.format(section, key)] = \ + cast_basic_type(section_param.value, section_param.type) if cast else section_param.value else: for section in hyperparams: for key, section_param in hyperparams[section].items(): + v = cast_basic_type(section_param.value, section_param.type) if cast else section_param.value if section_param.type == 'legacy' and section in (self._legacy_parameters_section_name, ): - parameters['{}'.format(key)] = section_param.value + parameters['{}'.format(key)] = v else: - parameters['{}/{}'.format(section, key)] = section_param.value + parameters['{}/{}'.format(section, key)] = v return parameters @@ -1086,7 +1090,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): section = hyperparams.get(section_name, dict()) org_param = org_hyperparams.get(section_name, dict()).get(key, None) param_type = params_types[org_k] if org_k in params_types else ( - org_param.type if org_param is not None else type(v) if v is not None else None + org_param.type if org_param is not None else get_basic_type(v) if v is not None else None ) if param_type and not isinstance(param_type, str): param_type = param_type.__name__ if hasattr(param_type, '__name__') else str(param_type) diff --git a/clearml/task.py b/clearml/task.py index 8e1b5c8c..f4a5c044 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -1854,16 +1854,19 @@ class Task(_Task): j['variant'], {'last': j['value'], 'min': j['min_value'], 'max': j['max_value']}) return scalar_metrics - def get_parameters_as_dict(self): - # type: () -> Dict + def get_parameters_as_dict(self, cast=False): + # type: (bool) -> Dict """ Get the Task parameters as a raw nested dictionary. .. note:: - The values are not parsed. They are returned as is. + If `cast` is False (default) The values are not parsed. They are returned as is. + + :param cast: If True, cast the parameter to the original type. Default False, + values are returned in their string representation """ - return naive_nested_from_flat_dictionary(self.get_parameters()) + return naive_nested_from_flat_dictionary(self.get_parameters(cast=cast)) def set_parameters_as_dict(self, dictionary): # type: (Dict) -> None diff --git a/clearml/utilities/proxy_object.py b/clearml/utilities/proxy_object.py index a64e9100..dcdb7e91 100644 --- a/clearml/utilities/proxy_object.py +++ b/clearml/utilities/proxy_object.py @@ -1,7 +1,9 @@ import itertools +import json from copy import copy import six +import yaml class ProxyDictPostWrite(dict): @@ -91,6 +93,59 @@ def verify_basic_type(a_dict_list, basic_types=None): all(verify_basic_type(v) for v in a_dict_list.values()) +def cast_basic_type(value, type_str): + if not type_str: + return value + + basic_types = {str(getattr(v, '__name__', v)): v for v in (float, int, bool, str, list, tuple, dict)} + + parts = type_str.split('/') + # nested = len(parts) > 1 + + if parts[0] in ('list', 'tuple'): + v = '[' + value.lstrip('[(').rstrip('])') + ']' + v = yaml.load(v, Loader=yaml.SafeLoader) + return basic_types.get(parts[0])(v) + elif parts[0] in ('dict', ): + try: + return json.loads(value) + except Exception: + pass + return value + + t = basic_types.get(str(type_str).lower().strip(), False) + if t is not False: + # noinspection PyBroadException + try: + return t(value) + except Exception: + return value + + return value + + +def get_basic_type(value): + basic_types = (float, int, bool, six.string_types, list, tuple, dict) + + if isinstance(value, (list, tuple)) and value: + tv = type(value) + t = type(value[0]) + if all(t == type(v) for v in value): + return '{}/{}'.format(str(getattr(tv, '__name__', tv)), str(getattr(t, '__name__', t))) + elif isinstance(value, dict) and value: + t = type(list(value.values())[0]) + if all(t == type(v) for v in value.values()): + return 'dict/{}'.format(str(getattr(t, '__name__', t))) + + # it might be an empty list/dict/tuple + t = type(value) + if isinstance(value, basic_types): + return str(getattr(t, '__name__', t)) + + # we are storing it, even though we will not be able to restore it + return str(getattr(t, '__name__', t)) + + def flatten_dictionary(a_dict, prefix='', sep='/'): flat_dict = {} basic_types = (float, int, bool, six.string_types, )