Add support for connect_configuration(list), not just dict

This commit is contained in:
allegroai 2021-07-31 23:52:46 +03:00
parent 18ee6ed177
commit 774957797e
4 changed files with 62 additions and 30 deletions

View File

@ -937,7 +937,8 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
return "" return ""
str_value = str(value) str_value = str(value)
if isinstance(value, (tuple, list, dict)) and 'None' in re.split(r'[ ,\[\]{}()]', str_value): if isinstance(value, (tuple, list, dict)):
if 'None' in re.split(r'[ ,\[\]{}()]', str_value):
# If we have None in the string we have to use json to replace it with null, # If we have None in the string we have to use json to replace it with null,
# otherwise we end up with None as string when running remotely # otherwise we end up with None as string when running remotely
try: try:
@ -949,6 +950,13 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
except TypeError: except TypeError:
# if we somehow failed to json serialize, revert to previous std casting # if we somehow failed to json serialize, revert to previous std casting
pass pass
elif any('\\' in str(v) for v in value):
try:
str_json = json.dumps(value)
return str_json
except TypeError:
pass
return str_value return str_value
if not all(isinstance(x, (dict, Iterable)) for x in args): if not all(isinstance(x, (dict, Iterable)) for x in args):
@ -1878,7 +1886,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
self._edit(script=script) self._edit(script=script)
def _set_configuration(self, name, description=None, config_type=None, config_text=None, config_dict=None): def _set_configuration(self, name, description=None, config_type=None, config_text=None, config_dict=None):
# type: (str, Optional[str], Optional[str], Optional[str], Optional[Mapping]) -> None # type: (str, Optional[str], Optional[str], Optional[str], Optional[Union[Mapping, list]]) -> None
""" """
Set Task configuration text/dict. Multiple configurations are supported. Set Task configuration text/dict. Multiple configurations are supported.

View File

@ -1121,7 +1121,7 @@ class Task(_Task):
raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__) raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__)
def connect_configuration(self, configuration, name=None, description=None): def connect_configuration(self, configuration, name=None, description=None):
# type: (Union[Mapping, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str] # type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
""" """
Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object. Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object.
This method should be called before reading the configuration file. This method should be called before reading the configuration file.
@ -1136,7 +1136,7 @@ class Task(_Task):
config_file = task.connect_configuration(config_file) config_file = task.connect_configuration(config_file)
my_params = json.load(open(config_file,'rt')) my_params = json.load(open(config_file,'rt'))
A parameter dictionary: A parameter dictionary/list:
.. code-block:: py .. code-block:: py
@ -1145,7 +1145,7 @@ class Task(_Task):
:param configuration: The configuration. This is usually the configuration used in the model training process. :param configuration: The configuration. This is usually the configuration used in the model training process.
Specify one of the following: Specify one of the following:
- A dictionary - A dictionary containing the configuration. ClearML stores the configuration in - A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in
the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable. the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable.
- A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file. - A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file.
A local path must be relative path. When executing a Task remotely in a worker, the contents brought A local path must be relative path. When executing a Task remotely in a worker, the contents brought
@ -1160,7 +1160,7 @@ class Task(_Task):
specified, then a path to a local configuration file is returned. Configuration object. specified, then a path to a local configuration file is returned. Configuration object.
""" """
pathlib_Path = None # noqa pathlib_Path = None # noqa
if not isinstance(configuration, (dict, Path, six.string_types)): if not isinstance(configuration, (dict, list, Path, six.string_types)):
try: try:
from pathlib import Path as pathlib_Path # noqa from pathlib import Path as pathlib_Path # noqa
except ImportError: except ImportError:
@ -1178,7 +1178,7 @@ class Task(_Task):
"please upgrade to the latest version") "please upgrade to the latest version")
# parameter dictionary # parameter dictionary
if isinstance(configuration, dict): if isinstance(configuration, (dict, list,)):
def _update_config_dict(task, config_dict): def _update_config_dict(task, config_dict):
if multi_config_support: if multi_config_support:
# noinspection PyProtectedMember # noinspection PyProtectedMember
@ -1194,6 +1194,7 @@ class Task(_Task):
name=name, description=description, config_type='dictionary', config_dict=configuration) name=name, description=description, config_type='dictionary', config_dict=configuration)
else: else:
self._set_model_config(config_dict=configuration) self._set_model_config(config_dict=configuration)
if isinstance(configuration, dict):
configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration) configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
else: else:
# noinspection PyBroadException # noinspection PyBroadException
@ -1214,9 +1215,14 @@ class Task(_Task):
config_type='dictionary', config_dict=configuration) config_type='dictionary', config_dict=configuration)
return configuration return configuration
if isinstance(configuration, dict):
configuration.clear() configuration.clear()
configuration.update(remote_configuration) configuration.update(remote_configuration)
configuration = ProxyDictPreWrite(False, False, **configuration) configuration = ProxyDictPreWrite(False, False, **configuration)
elif isinstance(configuration, list):
configuration.clear()
configuration.extend(remote_configuration)
return configuration return configuration
# it is a path to a local file # it is a path to a local file

View File

@ -56,8 +56,8 @@ def config_dict_to_text(config):
# if already string return as is # if already string return as is
if isinstance(config, six.string_types): if isinstance(config, six.string_types):
return config return config
if not isinstance(config, dict): if not isinstance(config, (dict, list)):
raise ValueError("Configuration only supports dictionary objects") raise ValueError("Configuration only supports dictionary/list objects")
try: try:
# noinspection PyBroadException # noinspection PyBroadException
try: try:
@ -79,7 +79,7 @@ def text_to_config_dict(text):
raise ValueError("Configuration parsing only supports string") raise ValueError("Configuration parsing only supports string")
# noinspection PyBroadException # noinspection PyBroadException
try: try:
return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict()) return hocon_unquote_key(ConfigFactory.parse_string(text))
except pyparsing.ParseBaseException as ex: except pyparsing.ParseBaseException as ex:
pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column) pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None) six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)

View File

@ -127,11 +127,17 @@ def merge_dicts(dict1, dict2):
return dict1 return dict1
def hocon_quote_key(a_dict): def hocon_quote_key(a_obj):
""" Recursively quote key with '.' to \"key\" """ """ Recursively quote key with '.' to \"key\" """
if not isinstance(a_dict, dict): if isinstance(a_obj, list):
return a_dict return [hocon_quote_key(a) for a in a_obj]
elif isinstance(a_obj, tuple):
return tuple(hocon_quote_key(a) for a in a_obj)
elif not isinstance(a_obj, dict):
return a_obj
# preserve dict type # preserve dict type
a_dict = a_obj
new_dict = type(a_dict)() new_dict = type(a_dict)()
for k, v in a_dict.items(): for k, v in a_dict.items():
if isinstance(k, str) and '.' in k: if isinstance(k, str) and '.' in k:
@ -141,10 +147,22 @@ def hocon_quote_key(a_dict):
return new_dict return new_dict
def hocon_unquote_key(a_dict): def hocon_unquote_key(a_obj):
""" Recursively unquote \"key\" with '.' to key """ """ Recursively unquote \"key\" with '.' to key """
if not isinstance(a_dict, dict):
return a_dict if isinstance(a_obj, list):
return [hocon_unquote_key(a) for a in a_obj]
elif isinstance(a_obj, tuple):
return tuple(hocon_unquote_key(a) for a in a_obj)
elif not isinstance(a_obj, dict):
return a_obj
a_dict = a_obj
# ConfigTree to dict
if hasattr(a_dict, 'as_plain_ordered_dict'):
a_dict = a_dict.as_plain_ordered_dict()
# preserve dict type # preserve dict type
new_dict = type(a_dict)() new_dict = type(a_dict)()
for k, v in a_dict.items(): for k, v in a_dict.items():