mirror of
https://github.com/clearml/clearml
synced 2025-05-29 09:38:15 +00:00
Add support for connect_configuration(list), not just dict
This commit is contained in:
parent
18ee6ed177
commit
774957797e
@ -937,18 +937,26 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
|
||||
return ""
|
||||
|
||||
str_value = str(value)
|
||||
if isinstance(value, (tuple, list, dict)) and 'None' in re.split(r'[ ,\[\]{}()]', str_value):
|
||||
# If we have None in the string we have to use json to replace it with null,
|
||||
# otherwise we end up with None as string when running remotely
|
||||
try:
|
||||
str_json = json.dumps(value)
|
||||
# verify we actually have a null in the string, otherwise prefer the str cast
|
||||
# This is because we prefer to have \' as in str and not \" used in json
|
||||
if 'null' in re.split(r'[ ,\[\]{}()]', str_json):
|
||||
if isinstance(value, (tuple, list, dict)):
|
||||
if 'None' in re.split(r'[ ,\[\]{}()]', str_value):
|
||||
# If we have None in the string we have to use json to replace it with null,
|
||||
# otherwise we end up with None as string when running remotely
|
||||
try:
|
||||
str_json = json.dumps(value)
|
||||
# verify we actually have a null in the string, otherwise prefer the str cast
|
||||
# This is because we prefer to have \' as in str and not \" used in json
|
||||
if 'null' in re.split(r'[ ,\[\]{}()]', str_json):
|
||||
return str_json
|
||||
except TypeError:
|
||||
# if we somehow failed to json serialize, revert to previous std casting
|
||||
pass
|
||||
elif any('\\' in str(v) for v in value):
|
||||
try:
|
||||
str_json = json.dumps(value)
|
||||
return str_json
|
||||
except TypeError:
|
||||
# if we somehow failed to json serialize, revert to previous std casting
|
||||
pass
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
return str_value
|
||||
|
||||
if not all(isinstance(x, (dict, Iterable)) for x in args):
|
||||
@ -1878,7 +1886,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin):
|
||||
self._edit(script=script)
|
||||
|
||||
def _set_configuration(self, name, description=None, config_type=None, config_text=None, config_dict=None):
|
||||
# type: (str, Optional[str], Optional[str], Optional[str], Optional[Mapping]) -> None
|
||||
# type: (str, Optional[str], Optional[str], Optional[str], Optional[Union[Mapping, list]]) -> None
|
||||
"""
|
||||
Set Task configuration text/dict. Multiple configurations are supported.
|
||||
|
||||
|
@ -1121,7 +1121,7 @@ class Task(_Task):
|
||||
raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__)
|
||||
|
||||
def connect_configuration(self, configuration, name=None, description=None):
|
||||
# type: (Union[Mapping, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
|
||||
# type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
|
||||
"""
|
||||
Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object.
|
||||
This method should be called before reading the configuration file.
|
||||
@ -1136,7 +1136,7 @@ class Task(_Task):
|
||||
config_file = task.connect_configuration(config_file)
|
||||
my_params = json.load(open(config_file,'rt'))
|
||||
|
||||
A parameter dictionary:
|
||||
A parameter dictionary/list:
|
||||
|
||||
.. code-block:: py
|
||||
|
||||
@ -1145,7 +1145,7 @@ class Task(_Task):
|
||||
:param configuration: The configuration. This is usually the configuration used in the model training process.
|
||||
Specify one of the following:
|
||||
|
||||
- A dictionary - A dictionary containing the configuration. ClearML stores the configuration in
|
||||
- A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in
|
||||
the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable.
|
||||
- A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file.
|
||||
A local path must be relative path. When executing a Task remotely in a worker, the contents brought
|
||||
@ -1160,7 +1160,7 @@ class Task(_Task):
|
||||
specified, then a path to a local configuration file is returned. Configuration object.
|
||||
"""
|
||||
pathlib_Path = None # noqa
|
||||
if not isinstance(configuration, (dict, Path, six.string_types)):
|
||||
if not isinstance(configuration, (dict, list, Path, six.string_types)):
|
||||
try:
|
||||
from pathlib import Path as pathlib_Path # noqa
|
||||
except ImportError:
|
||||
@ -1178,7 +1178,7 @@ class Task(_Task):
|
||||
"please upgrade to the latest version")
|
||||
|
||||
# parameter dictionary
|
||||
if isinstance(configuration, dict):
|
||||
if isinstance(configuration, (dict, list,)):
|
||||
def _update_config_dict(task, config_dict):
|
||||
if multi_config_support:
|
||||
# noinspection PyProtectedMember
|
||||
@ -1194,7 +1194,8 @@ class Task(_Task):
|
||||
name=name, description=description, config_type='dictionary', config_dict=configuration)
|
||||
else:
|
||||
self._set_model_config(config_dict=configuration)
|
||||
configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
|
||||
if isinstance(configuration, dict):
|
||||
configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
|
||||
else:
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
@ -1214,9 +1215,14 @@ class Task(_Task):
|
||||
config_type='dictionary', config_dict=configuration)
|
||||
return configuration
|
||||
|
||||
configuration.clear()
|
||||
configuration.update(remote_configuration)
|
||||
configuration = ProxyDictPreWrite(False, False, **configuration)
|
||||
if isinstance(configuration, dict):
|
||||
configuration.clear()
|
||||
configuration.update(remote_configuration)
|
||||
configuration = ProxyDictPreWrite(False, False, **configuration)
|
||||
elif isinstance(configuration, list):
|
||||
configuration.clear()
|
||||
configuration.extend(remote_configuration)
|
||||
|
||||
return configuration
|
||||
|
||||
# it is a path to a local file
|
||||
|
@ -56,8 +56,8 @@ def config_dict_to_text(config):
|
||||
# if already string return as is
|
||||
if isinstance(config, six.string_types):
|
||||
return config
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError("Configuration only supports dictionary objects")
|
||||
if not isinstance(config, (dict, list)):
|
||||
raise ValueError("Configuration only supports dictionary/list objects")
|
||||
try:
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
@ -79,7 +79,7 @@ def text_to_config_dict(text):
|
||||
raise ValueError("Configuration parsing only supports string")
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict())
|
||||
return hocon_unquote_key(ConfigFactory.parse_string(text))
|
||||
except pyparsing.ParseBaseException as ex:
|
||||
pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
|
||||
six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)
|
||||
|
@ -127,11 +127,17 @@ def merge_dicts(dict1, dict2):
|
||||
return dict1
|
||||
|
||||
|
||||
def hocon_quote_key(a_dict):
|
||||
def hocon_quote_key(a_obj):
|
||||
""" Recursively quote key with '.' to \"key\" """
|
||||
if not isinstance(a_dict, dict):
|
||||
return a_dict
|
||||
if isinstance(a_obj, list):
|
||||
return [hocon_quote_key(a) for a in a_obj]
|
||||
elif isinstance(a_obj, tuple):
|
||||
return tuple(hocon_quote_key(a) for a in a_obj)
|
||||
elif not isinstance(a_obj, dict):
|
||||
return a_obj
|
||||
|
||||
# preserve dict type
|
||||
a_dict = a_obj
|
||||
new_dict = type(a_dict)()
|
||||
for k, v in a_dict.items():
|
||||
if isinstance(k, str) and '.' in k:
|
||||
@ -141,10 +147,22 @@ def hocon_quote_key(a_dict):
|
||||
return new_dict
|
||||
|
||||
|
||||
def hocon_unquote_key(a_dict):
|
||||
def hocon_unquote_key(a_obj):
|
||||
""" Recursively unquote \"key\" with '.' to key """
|
||||
if not isinstance(a_dict, dict):
|
||||
return a_dict
|
||||
|
||||
if isinstance(a_obj, list):
|
||||
return [hocon_unquote_key(a) for a in a_obj]
|
||||
elif isinstance(a_obj, tuple):
|
||||
return tuple(hocon_unquote_key(a) for a in a_obj)
|
||||
elif not isinstance(a_obj, dict):
|
||||
return a_obj
|
||||
|
||||
a_dict = a_obj
|
||||
|
||||
# ConfigTree to dict
|
||||
if hasattr(a_dict, 'as_plain_ordered_dict'):
|
||||
a_dict = a_dict.as_plain_ordered_dict()
|
||||
|
||||
# preserve dict type
|
||||
new_dict = type(a_dict)()
|
||||
for k, v in a_dict.items():
|
||||
|
Loading…
Reference in New Issue
Block a user