From cf28551d211af73c1a7332a6ac72b1f3dd2a798c Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Sat, 10 Apr 2021 22:22:03 +0300 Subject: [PATCH] Support storing dict with "." in the keys using Task.connect_configuration --- clearml/utilities/config.py | 6 ++++-- clearml/utilities/dicts.py | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/clearml/utilities/config.py b/clearml/utilities/config.py index 50b2db26..cb9c09c2 100644 --- a/clearml/utilities/config.py +++ b/clearml/utilities/config.py @@ -4,6 +4,8 @@ import json import six import pyparsing + +from .dicts import hocon_quote_key, hocon_unquote_key from .pyhocon import ConfigFactory, HOCONConverter from ..storage.util import parse_size @@ -59,7 +61,7 @@ def config_dict_to_text(config): try: # noinspection PyBroadException try: - text = HOCONConverter.to_hocon(ConfigFactory.from_dict(config)) + text = HOCONConverter.to_hocon(ConfigFactory.from_dict(hocon_quote_key(config))) except Exception: # fallback json+pyhocon # hack, pyhocon is not very good with dict conversion so we pass through json @@ -77,7 +79,7 @@ def text_to_config_dict(text): raise ValueError("Configuration parsing only supports string") # noinspection PyBroadException try: - return ConfigFactory.parse_string(text).as_plain_ordered_dict() + return hocon_unquote_key(ConfigFactory.parse_string(text).as_plain_ordered_dict()) except pyparsing.ParseBaseException as ex: pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column) six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None) diff --git a/clearml/utilities/dicts.py b/clearml/utilities/dicts.py index 02f437f5..8d1bef26 100644 --- a/clearml/utilities/dicts.py +++ b/clearml/utilities/dicts.py @@ -132,11 +132,11 @@ def hocon_quote_key(a_dict): return a_dict # preserve dict type new_dict = type(a_dict)() - for v, k in a_dict.items(): + for k, v in a_dict.items(): if isinstance(k, str) and '.' in k: new_dict['"{}"'.format(k)] = hocon_quote_key(v) else: - new_dict[k] = v + new_dict[k] = hocon_quote_key(v) return new_dict @@ -146,9 +146,9 @@ def hocon_unquote_key(a_dict): return a_dict # preserve dict type new_dict = type(a_dict)() - for v, k in a_dict.items(): + for k, v in a_dict.items(): if isinstance(k, str) and k[0] == '"' and k[-1] == '"' and '.' in k: new_dict[k[1:-1]] = hocon_unquote_key(v) else: - new_dict[k] = v + new_dict[k] = hocon_unquote_key(v) return new_dict