diff --git a/clearml/backend_interface/task/task.py b/clearml/backend_interface/task/task.py index 6e057098..9690afc4 100644 --- a/clearml/backend_interface/task/task.py +++ b/clearml/backend_interface/task/task.py @@ -937,18 +937,26 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): return "" str_value = str(value) - if isinstance(value, (tuple, list, dict)) and 'None' in re.split(r'[ ,\[\]{}()]', str_value): - # If we have None in the string we have to use json to replace it with null, - # otherwise we end up with None as string when running remotely - try: - str_json = json.dumps(value) - # verify we actually have a null in the string, otherwise prefer the str cast - # This is because we prefer to have \' as in str and not \" used in json - if 'null' in re.split(r'[ ,\[\]{}()]', str_json): + if isinstance(value, (tuple, list, dict)): + if 'None' in re.split(r'[ ,\[\]{}()]', str_value): + # If we have None in the string we have to use json to replace it with null, + # otherwise we end up with None as string when running remotely + try: + str_json = json.dumps(value) + # verify we actually have a null in the string, otherwise prefer the str cast + # This is because we prefer to have \' as in str and not \" used in json + if 'null' in re.split(r'[ ,\[\]{}()]', str_json): + return str_json + except TypeError: + # if we somehow failed to json serialize, revert to previous std casting + pass + elif any('\\' in str(v) for v in value): + try: + str_json = json.dumps(value) return str_json - except TypeError: - # if we somehow failed to json serialize, revert to previous std casting - pass + except TypeError: + pass + return str_value if not all(isinstance(x, (dict, Iterable)) for x in args): @@ -1878,7 +1886,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): self._edit(script=script) def _set_configuration(self, name, description=None, config_type=None, config_text=None, config_dict=None): - # type: (str, Optional[str], Optional[str], Optional[str], Optional[Mapping]) -> None + # type: (str, Optional[str], Optional[str], Optional[str], Optional[Union[Mapping, list]]) -> None """ Set Task configuration text/dict. Multiple configurations are supported. diff --git a/clearml/task.py b/clearml/task.py index 436ae38f..55ba3eb7 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -1121,7 +1121,7 @@ class Task(_Task): raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__) def connect_configuration(self, configuration, name=None, description=None): - # type: (Union[Mapping, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str] + # type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str] """ Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object. This method should be called before reading the configuration file. @@ -1136,7 +1136,7 @@ class Task(_Task): config_file = task.connect_configuration(config_file) my_params = json.load(open(config_file,'rt')) - A parameter dictionary: + A parameter dictionary/list: .. code-block:: py @@ -1145,7 +1145,7 @@ class Task(_Task): :param configuration: The configuration. This is usually the configuration used in the model training process. Specify one of the following: - - A dictionary - A dictionary containing the configuration. ClearML stores the configuration in + - A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable. - A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file. A local path must be relative path. When executing a Task remotely in a worker, the contents brought @@ -1160,7 +1160,7 @@ class Task(_Task): specified, then a path to a local configuration file is returned. Configuration object. """ pathlib_Path = None # noqa - if not isinstance(configuration, (dict, Path, six.string_types)): + if not isinstance(configuration, (dict, list, Path, six.string_types)): try: from pathlib import Path as pathlib_Path # noqa except ImportError: @@ -1178,7 +1178,7 @@ class Task(_Task): "please upgrade to the latest version") # parameter dictionary - if isinstance(configuration, dict): + if isinstance(configuration, (dict, list,)): def _update_config_dict(task, config_dict): if multi_config_support: # noinspection PyProtectedMember @@ -1194,7 +1194,8 @@ class Task(_Task): name=name, description=description, config_type='dictionary', config_dict=configuration) else: self._set_model_config(config_dict=configuration) - configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration) + if isinstance(configuration, dict): + configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration) else: # noinspection PyBroadException try: @@ -1214,9 +1215,14 @@ class Task(_Task): config_type='dictionary', config_dict=configuration) return configuration - configuration.clear() - configuration.update(remote_configuration) - configuration = ProxyDictPreWrite(False, False, **configuration) + if isinstance(configuration, dict): + configuration.clear() + configuration.update(remote_configuration) + configuration = ProxyDictPreWrite(False, False, **configuration) + elif isinstance(configuration, list): + configuration.clear() + configuration.extend(remote_configuration) + return configuration # it is a path to a local file diff --git a/clearml/utilities/config.py b/clearml/utilities/config.py index cb9c09c2..7d3f9423 100644 --- a/clearml/utilities/config.py +++ b/clearml/utilities/config.py @@ -56,8 +56,8 @@ def config_dict_to_text(config): # if already string return as is if isinstance(config, six.string_types): return config - if not isinstance(config, dict): - raise ValueError("Configuration only supports dictionary objects") + if not isinstance(config, (dict, list)): + raise ValueError("Configuration only supports dictionary/list objects") try: # noinspection PyBroadException try: @@ -79,7 +79,7 @@ def text_to_config_dict(text): raise ValueError("Configuration parsing only supports string") # noinspection PyBroadException try: - return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict()) + return hocon_unquote_key(ConfigFactory.parse_string(text)) except pyparsing.ParseBaseException as ex: pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column) six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None) diff --git a/clearml/utilities/dicts.py b/clearml/utilities/dicts.py index 2d5ad809..f8dc1592 100644 --- a/clearml/utilities/dicts.py +++ b/clearml/utilities/dicts.py @@ -127,11 +127,17 @@ def merge_dicts(dict1, dict2): return dict1 -def hocon_quote_key(a_dict): +def hocon_quote_key(a_obj): """ Recursively quote key with '.' to \"key\" """ - if not isinstance(a_dict, dict): - return a_dict + if isinstance(a_obj, list): + return [hocon_quote_key(a) for a in a_obj] + elif isinstance(a_obj, tuple): + return tuple(hocon_quote_key(a) for a in a_obj) + elif not isinstance(a_obj, dict): + return a_obj + # preserve dict type + a_dict = a_obj new_dict = type(a_dict)() for k, v in a_dict.items(): if isinstance(k, str) and '.' in k: @@ -141,10 +147,22 @@ def hocon_quote_key(a_dict): return new_dict -def hocon_unquote_key(a_dict): +def hocon_unquote_key(a_obj): """ Recursively unquote \"key\" with '.' to key """ - if not isinstance(a_dict, dict): - return a_dict + + if isinstance(a_obj, list): + return [hocon_unquote_key(a) for a in a_obj] + elif isinstance(a_obj, tuple): + return tuple(hocon_unquote_key(a) for a in a_obj) + elif not isinstance(a_obj, dict): + return a_obj + + a_dict = a_obj + + # ConfigTree to dict + if hasattr(a_dict, 'as_plain_ordered_dict'): + a_dict = a_dict.as_plain_ordered_dict() + # preserve dict type new_dict = type(a_dict)() for k, v in a_dict.items():