Add support for connect_configuration(list), not just dict

This commit is contained in:
allegroai 2021-07-31 23:52:46 +03:00
parent 18ee6ed177
commit 774957797e
4 changed files with 62 additions and 30 deletions

View File

@ -937,18 +937,26 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
return ""
str_value = str(value)
if isinstance(value, (tuple, list, dict)) and 'None' in re.split(r'[ ,\[\]{}()]', str_value):
# If we have None in the string we have to use json to replace it with null,
# otherwise we end up with None as string when running remotely
try:
str_json = json.dumps(value)
# verify we actually have a null in the string, otherwise prefer the str cast
# This is because we prefer to have \' as in str and not \" used in json
if 'null' in re.split(r'[ ,\[\]{}()]', str_json):
if isinstance(value, (tuple, list, dict)):
if 'None' in re.split(r'[ ,\[\]{}()]', str_value):
# If we have None in the string we have to use json to replace it with null,
# otherwise we end up with None as string when running remotely
try:
str_json = json.dumps(value)
# verify we actually have a null in the string, otherwise prefer the str cast
# This is because we prefer to have \' as in str and not \" used in json
if 'null' in re.split(r'[ ,\[\]{}()]', str_json):
return str_json
except TypeError:
# if we somehow failed to json serialize, revert to previous std casting
pass
elif any('\\' in str(v) for v in value):
try:
str_json = json.dumps(value)
return str_json
except TypeError:
# if we somehow failed to json serialize, revert to previous std casting
pass
except TypeError:
pass
return str_value
if not all(isinstance(x, (dict, Iterable)) for x in args):
@ -1878,7 +1886,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self._edit(script=script)
def _set_configuration(self, name, description=None, config_type=None, config_text=None, config_dict=None):
# type: (str, Optional[str], Optional[str], Optional[str], Optional[Mapping]) -> None
# type: (str, Optional[str], Optional[str], Optional[str], Optional[Union[Mapping, list]]) -> None
"""
Set Task configuration text/dict. Multiple configurations are supported.

View File

@ -1121,7 +1121,7 @@ class Task(_Task):
raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__)
def connect_configuration(self, configuration, name=None, description=None):
# type: (Union[Mapping, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
# type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
"""
Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object.
This method should be called before reading the configuration file.
@ -1136,7 +1136,7 @@ class Task(_Task):
config_file = task.connect_configuration(config_file)
my_params = json.load(open(config_file,'rt'))
A parameter dictionary:
A parameter dictionary/list:
.. code-block:: py
@ -1145,7 +1145,7 @@ class Task(_Task):
:param configuration: The configuration. This is usually the configuration used in the model training process.
Specify one of the following:
- A dictionary - A dictionary containing the configuration. ClearML stores the configuration in
- A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in
the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable.
- A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file.
A local path must be relative path. When executing a Task remotely in a worker, the contents brought
@ -1160,7 +1160,7 @@ class Task(_Task):
specified, then a path to a local configuration file is returned. Configuration object.
"""
pathlib_Path = None # noqa
if not isinstance(configuration, (dict, Path, six.string_types)):
if not isinstance(configuration, (dict, list, Path, six.string_types)):
try:
from pathlib import Path as pathlib_Path # noqa
except ImportError:
@ -1178,7 +1178,7 @@ class Task(_Task):
"please upgrade to the latest version")
# parameter dictionary
if isinstance(configuration, dict):
if isinstance(configuration, (dict, list,)):
def _update_config_dict(task, config_dict):
if multi_config_support:
# noinspection PyProtectedMember
@ -1194,7 +1194,8 @@ class Task(_Task):
name=name, description=description, config_type='dictionary', config_dict=configuration)
else:
self._set_model_config(config_dict=configuration)
configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
if isinstance(configuration, dict):
configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
else:
# noinspection PyBroadException
try:
@ -1214,9 +1215,14 @@ class Task(_Task):
config_type='dictionary', config_dict=configuration)
return configuration
configuration.clear()
configuration.update(remote_configuration)
configuration = ProxyDictPreWrite(False, False, **configuration)
if isinstance(configuration, dict):
configuration.clear()
configuration.update(remote_configuration)
configuration = ProxyDictPreWrite(False, False, **configuration)
elif isinstance(configuration, list):
configuration.clear()
configuration.extend(remote_configuration)
return configuration
# it is a path to a local file

View File

@ -56,8 +56,8 @@ def config_dict_to_text(config):
# if already string return as is
if isinstance(config, six.string_types):
return config
if not isinstance(config, dict):
raise ValueError("Configuration only supports dictionary objects")
if not isinstance(config, (dict, list)):
raise ValueError("Configuration only supports dictionary/list objects")
try:
# noinspection PyBroadException
try:
@ -79,7 +79,7 @@ def text_to_config_dict(text):
raise ValueError("Configuration parsing only supports string")
# noinspection PyBroadException
try:
return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict())
return hocon_unquote_key(ConfigFactory.parse_string(text))
except pyparsing.ParseBaseException as ex:
pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)

View File

@ -127,11 +127,17 @@ def merge_dicts(dict1, dict2):
return dict1
def hocon_quote_key(a_dict):
def hocon_quote_key(a_obj):
""" Recursively quote key with '.' to \"key\" """
if not isinstance(a_dict, dict):
return a_dict
if isinstance(a_obj, list):
return [hocon_quote_key(a) for a in a_obj]
elif isinstance(a_obj, tuple):
return tuple(hocon_quote_key(a) for a in a_obj)
elif not isinstance(a_obj, dict):
return a_obj
# preserve dict type
a_dict = a_obj
new_dict = type(a_dict)()
for k, v in a_dict.items():
if isinstance(k, str) and '.' in k:
@ -141,10 +147,22 @@ def hocon_quote_key(a_dict):
return new_dict
def hocon_unquote_key(a_dict):
def hocon_unquote_key(a_obj):
""" Recursively unquote \"key\" with '.' to key """
if not isinstance(a_dict, dict):
return a_dict
if isinstance(a_obj, list):
return [hocon_unquote_key(a) for a in a_obj]
elif isinstance(a_obj, tuple):
return tuple(hocon_unquote_key(a) for a in a_obj)
elif not isinstance(a_obj, dict):
return a_obj
a_dict = a_obj
# ConfigTree to dict
if hasattr(a_dict, 'as_plain_ordered_dict'):
a_dict = a_dict.as_plain_ordered_dict()
# preserve dict type
new_dict = type(a_dict)()
for k, v in a_dict.items():