ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support storing dict with "." in the keys using Task.connect_configuration

Browse Source
This commit is contained in:
allegroai 2021-04-10 22:22:03 +03:00
parent bf9ec770c6
commit cf28551d21
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clearml/utilities/config.py
View File

@ -4,6 +4,8 @@ import json
import six import six
import pyparsing import pyparsing
from .dicts import hocon_quote_key, hocon_unquote_key
from .pyhocon import ConfigFactory, HOCONConverter from .pyhocon import ConfigFactory, HOCONConverter
from ..storage.util import parse_size from ..storage.util import parse_size
@ -59,7 +61,7 @@ def config_dict_to_text(config):
try: try:
# noinspection PyBroadException # noinspection PyBroadException
try: try:
text = HOCONConverter.to_hocon(ConfigFactory.from_dict(config)) text = HOCONConverter.to_hocon(ConfigFactory.from_dict(hocon_quote_key(config)))
except Exception: except Exception:
# fallback json+pyhocon # fallback json+pyhocon
# hack, pyhocon is not very good with dict conversion so we pass through json # hack, pyhocon is not very good with dict conversion so we pass through json
@ -77,7 +79,7 @@ def text_to_config_dict(text):
raise ValueError("Configuration parsing only supports string") raise ValueError("Configuration parsing only supports string")
# noinspection PyBroadException # noinspection PyBroadException
try: try:
return ConfigFactory.parse_string(text).as_plain_ordered_dict() return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict())
except pyparsing.ParseBaseException as ex: except pyparsing.ParseBaseException as ex:
pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column) pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None) six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)

8
clearml/utilities/dicts.py
View File

@ -132,11 +132,11 @@ def hocon_quote_key(a_dict):
return a_dict return a_dict
# preserve dict type # preserve dict type
new_dict = type(a_dict)() new_dict = type(a_dict)()
for v, k in a_dict.items(): for k, v in a_dict.items():
if isinstance(k, str) and '.' in k: if isinstance(k, str) and '.' in k:
new_dict['"{}"'.format(k)] = hocon_quote_key(v) new_dict['"{}"'.format(k)] = hocon_quote_key(v)
else: else:
new_dict[k] = v new_dict[k] = hocon_quote_key(v)
return new_dict return new_dict
@ -146,9 +146,9 @@ def hocon_unquote_key(a_dict):
return a_dict return a_dict
# preserve dict type # preserve dict type
new_dict = type(a_dict)() new_dict = type(a_dict)()
for v, k in a_dict.items(): for k, v in a_dict.items():
if isinstance(k, str) and k[0] == '"' and k[-1] == '"' and '.' in k: if isinstance(k, str) and k[0] == '"' and k[-1] == '"' and '.' in k:
new_dict[k[1:-1]] = hocon_unquote_key(v) new_dict[k[1:-1]] = hocon_unquote_key(v)
else: else:
new_dict[k] = v new_dict[k] = hocon_unquote_key(v)
return new_dict return new_dict