Add support for connect_configuration(list), not just dict

clearml/backend_interface/task/task.py

@@ -937,18 +937,26 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
                 return ""
 
             str_value = str(value)
-            if isinstance(value, (tuple, list, dict)) and 'None' in re.split(r'[ ,\[\]{}()]', str_value):
-                # If we have None in the string we have to use json to replace it with null,
-                # otherwise we end up with None as string when running remotely
-                try:
-                    str_json = json.dumps(value)
-                    # verify we actually have a null in the string, otherwise prefer the str cast
-                    # This is because we prefer to have \' as in str and not \" used in json
-                    if 'null' in re.split(r'[ ,\[\]{}()]', str_json):
+            if isinstance(value, (tuple, list, dict)):
+                if 'None' in re.split(r'[ ,\[\]{}()]', str_value):
+                    # If we have None in the string we have to use json to replace it with null,
+                    # otherwise we end up with None as string when running remotely
+                    try:
+                        str_json = json.dumps(value)
+                        # verify we actually have a null in the string, otherwise prefer the str cast
+                        # This is because we prefer to have \' as in str and not \" used in json
+                        if 'null' in re.split(r'[ ,\[\]{}()]', str_json):
+                            return str_json
+                    except TypeError:
+                        # if we somehow failed to json serialize, revert to previous std casting
+                        pass
+                elif any('\\' in str(v) for v in value):
+                    try:
+                        str_json = json.dumps(value)
                         return str_json
-                except TypeError:
-                    # if we somehow failed to json serialize, revert to previous std casting
-                    pass
+                    except TypeError:
+                        pass
+
             return str_value
 
         if not all(isinstance(x, (dict, Iterable)) for x in args):
@@ -1878,7 +1886,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
             self._edit(script=script)
 
     def _set_configuration(self, name, description=None, config_type=None, config_text=None, config_dict=None):
-        # type: (str, Optional[str], Optional[str], Optional[str], Optional[Mapping]) -> None
+        # type: (str, Optional[str], Optional[str], Optional[str], Optional[Union[Mapping, list]]) -> None
         """
         Set Task configuration text/dict. Multiple configurations are supported.
 

clearml/task.py

@@ -1121,7 +1121,7 @@ class Task(_Task):
         raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__)
 
     def connect_configuration(self, configuration, name=None, description=None):
-        # type: (Union[Mapping, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
+        # type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
         """
         Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object.
         This method should be called before reading the configuration file.
@@ -1136,7 +1136,7 @@ class Task(_Task):
            config_file = task.connect_configuration(config_file)
            my_params = json.load(open(config_file,'rt'))
 
-        A parameter dictionary:
+        A parameter dictionary/list:
 
         .. code-block:: py
 
@@ -1145,7 +1145,7 @@ class Task(_Task):
         :param configuration: The configuration. This is usually the configuration used in the model training process.
             Specify one of the following:
 
-            - A dictionary - A dictionary containing the configuration. ClearML stores the configuration in
+            - A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in
               the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable.
             - A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file.
               A local path must be relative path. When executing a Task remotely in a worker, the contents brought
@@ -1160,7 +1160,7 @@ class Task(_Task):
             specified, then a path to a local configuration file is returned. Configuration object.
         """
         pathlib_Path = None  # noqa
-        if not isinstance(configuration, (dict, Path, six.string_types)):
+        if not isinstance(configuration, (dict, list, Path, six.string_types)):
             try:
                 from pathlib import Path as pathlib_Path  # noqa
             except ImportError:
@@ -1178,7 +1178,7 @@ class Task(_Task):
                              "please upgrade to the latest version")
 
         # parameter dictionary
-        if isinstance(configuration, dict):
+        if isinstance(configuration, (dict, list,)):
             def _update_config_dict(task, config_dict):
                 if multi_config_support:
                     # noinspection PyProtectedMember
@@ -1194,7 +1194,8 @@ class Task(_Task):
                         name=name, description=description, config_type='dictionary', config_dict=configuration)
                 else:
                     self._set_model_config(config_dict=configuration)
-                configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
+                if isinstance(configuration, dict):
+                    configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
             else:
                 # noinspection PyBroadException
                 try:
@@ -1214,9 +1215,14 @@ class Task(_Task):
                             config_type='dictionary', config_dict=configuration)
                     return configuration
 
-                configuration.clear()
-                configuration.update(remote_configuration)
-                configuration = ProxyDictPreWrite(False, False, **configuration)
+                if isinstance(configuration, dict):
+                    configuration.clear()
+                    configuration.update(remote_configuration)
+                    configuration = ProxyDictPreWrite(False, False, **configuration)
+                elif isinstance(configuration, list):
+                    configuration.clear()
+                    configuration.extend(remote_configuration)
+
             return configuration
 
         # it is a path to a local file

clearml/utilities/config.py

@@ -56,8 +56,8 @@ def config_dict_to_text(config):
     # if already string return as is
     if isinstance(config, six.string_types):
         return config
-    if not isinstance(config, dict):
-        raise ValueError("Configuration only supports dictionary objects")
+    if not isinstance(config, (dict, list)):
+        raise ValueError("Configuration only supports dictionary/list objects")
     try:
         # noinspection PyBroadException
         try:
@@ -79,7 +79,7 @@ def text_to_config_dict(text):
         raise ValueError("Configuration parsing only supports string")
     # noinspection PyBroadException
     try:
-        return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict())
+        return hocon_unquote_key(ConfigFactory.parse_string(text))
     except pyparsing.ParseBaseException as ex:
         pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
         six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)

clearml/utilities/dicts.py

@@ -127,11 +127,17 @@ def merge_dicts(dict1, dict2):
     return dict1
 
 
-def hocon_quote_key(a_dict):
+def hocon_quote_key(a_obj):
     """ Recursively quote key with '.' to \"key\" """
-    if not isinstance(a_dict, dict):
-        return a_dict
+    if isinstance(a_obj, list):
+        return [hocon_quote_key(a) for a in a_obj]
+    elif isinstance(a_obj, tuple):
+        return tuple(hocon_quote_key(a) for a in a_obj)
+    elif not isinstance(a_obj, dict):
+        return a_obj
+
     # preserve dict type
+    a_dict = a_obj
     new_dict = type(a_dict)()
     for k, v in a_dict.items():
         if isinstance(k, str) and '.' in k:
@@ -141,10 +147,22 @@ def hocon_quote_key(a_dict):
     return new_dict
 
 
-def hocon_unquote_key(a_dict):
+def hocon_unquote_key(a_obj):
     """ Recursively unquote \"key\" with '.' to key """
-    if not isinstance(a_dict, dict):
-        return a_dict
+
+    if isinstance(a_obj, list):
+        return [hocon_unquote_key(a) for a in a_obj]
+    elif isinstance(a_obj, tuple):
+        return tuple(hocon_unquote_key(a) for a in a_obj)
+    elif not isinstance(a_obj, dict):
+        return a_obj
+
+    a_dict = a_obj
+
+    # ConfigTree to dict
+    if hasattr(a_dict, 'as_plain_ordered_dict'):
+        a_dict = a_dict.as_plain_ordered_dict()
+
     # preserve dict type
     new_dict = type(a_dict)()
     for k, v in a_dict.items():