Add support for configuration env and files section

clearml/backend_api/session/defs.py

@@ -15,3 +15,5 @@ ENV_OFFLINE_MODE = EnvEntry("CLEARML_OFFLINE_MODE", "TRAINS_OFFLINE_MODE", type=
 ENV_CLEARML_NO_DEFAULT_SERVER = EnvEntry("CLEARML_NO_DEFAULT_SERVER", "TRAINS_NO_DEFAULT_SERVER",
                                          converter=safe_text_to_bool, type=bool, default=True)
 ENV_DISABLE_VAULT_SUPPORT = EnvEntry('CLEARML_DISABLE_VAULT_SUPPORT', type=bool)
+ENV_ENABLE_ENV_CONFIG_SECTION = EnvEntry('CLEARML_ENABLE_ENV_CONFIG_SECTION', type=bool)
+ENV_ENABLE_FILES_CONFIG_SECTION = EnvEntry('CLEARML_ENABLE_FILES_CONFIG_SECTION', type=bool)

clearml/backend_api/session/session.py

@@ -22,6 +22,8 @@ from .defs import (
     ENV_CLEARML_NO_DEFAULT_SERVER,
     ENV_AUTH_TOKEN,
     ENV_DISABLE_VAULT_SUPPORT,
+    ENV_ENABLE_ENV_CONFIG_SECTION,
+    ENV_ENABLE_FILES_CONFIG_SECTION,
 )
 from .request import Request, BatchRequest  # noqa: F401
 from .token_manager import TokenManager
@@ -30,6 +32,7 @@ from ..utils import get_http_session_with_retry, urllib_log_warning_setup
 from ...debugging import get_logger
 from ...utilities.pyhocon import ConfigTree, ConfigFactory
 from ...version import __version__
+from ...backend_config.utils import apply_files, apply_environment
 
 try:
     from OpenSSL.SSL import Error as SSLError
@@ -197,6 +200,8 @@ class Session(TokenManager):
 
         self.refresh_token()
 
+        local_logger = self._LocalLogger(self._logger)
+
         # update api version from server response
         try:
             token_dict = TokenManager.get_decoded_token(self.token)
@@ -210,7 +215,7 @@ class Session(TokenManager):
             Session.max_api_version = Session.api_version = str(api_version)
             Session.feature_set = str(token_dict.get('feature_set', self.feature_set) or "basic")
         except (jwt.DecodeError, ValueError):
-            (self._logger or get_logger()).warning(
+            local_logger().warning(
                 "Failed parsing server API level, defaulting to {}".format(Session.api_version))
 
         # now setup the session reporting, so one consecutive retries will show warning
@@ -225,6 +230,8 @@ class Session(TokenManager):
 
         self._load_vaults()
 
+        self._apply_config_sections(local_logger)
+
     def _load_vaults(self):
         if not self.check_min_api_version("2.15") or self.feature_set == "basic":
             return
@@ -257,6 +264,24 @@ class Session(TokenManager):
         except Exception as ex:
             print("Failed getting vaults: {}".format(ex))
 
+    def _apply_config_sections(self, local_logger):
+        # type: (_LocalLogger) -> None
+        default = self.config.get("sdk.apply_environment", False)
+        if ENV_ENABLE_ENV_CONFIG_SECTION.get(default=default):
+            try:
+                keys = apply_environment(self.config)
+                if keys:
+                    print("Environment variables set from configuration: {}".format(keys))
+            except Exception as ex:
+                local_logger().warning("Failed applying environment from configuration: {}".format(ex))
+
+        default = self.config.get("sdk.apply_files", default=False)
+        if ENV_ENABLE_FILES_CONFIG_SECTION.get(default=default):
+            try:
+                apply_files(self.config)
+            except Exception as ex:
+                local_logger().warning("Failed applying files from configuration: {}".format(ex))
+
     def _send_request(
         self,
         service,
@@ -726,3 +751,12 @@ class Session(TokenManager):
         return "{self.__class__.__name__}[{self.host}, {self.access_key}/{secret_key}]".format(
             self=self, secret_key=self.secret_key[:5] + "*" * (len(self.secret_key) - 5)
         )
+
+    class _LocalLogger:
+        def __init__(self, l):
+            self.logger = l
+
+        def __call__(self):
+            if not self.logger:
+                self.logger = get_logger()
+            return self.logger

clearml/backend_config/config.py

@@ -11,7 +11,6 @@ from typing import Any
 
 import six
 from pathlib2 import Path
-from ..utilities.pyhocon import ConfigTree, ConfigFactory
 from pyparsing import (
     ParseFatalException,
     ParseException,
@@ -36,6 +35,7 @@ from .entry import Entry, NotSet
 from .errors import ConfigurationError
 from .log import initialize as initialize_log, logger
 from .utils import get_options
+from ..utilities.pyhocon import ConfigTree, ConfigFactory
 
 try:
     from typing import Text

clearml/backend_config/utils.py

@@ -1,3 +1,14 @@
+import base64
+import os
+from os.path import expandvars, expanduser
+from pathlib import Path
+from typing import List, TYPE_CHECKING
+
+from ..utilities.pyhocon import HOCONConverter, ConfigTree
+
+if TYPE_CHECKING:
+    from .config import Config
+
 
 def get_items(cls):
     """ get key/value items from an enum-like class (members represent enumeration key/value) """
@@ -7,3 +18,95 @@ def get_items(cls):
 def get_options(cls):
     """ get options from an enum-like class (members represent enumeration key/value) """
     return get_items(cls).values()
+
+
+def apply_environment(config):
+    # type: (Config) -> List[str]
+    env_vars = config.get("environment", None)
+    if not env_vars:
+        return []
+    if isinstance(env_vars, (list, tuple)):
+        env_vars = dict(env_vars)
+
+    keys = list(filter(None, env_vars.keys()))
+
+    for key in keys:
+        os.environ[str(key)] = str(env_vars[key] or "")
+
+    return keys
+
+
+def apply_files(config):
+    # type: (Config) -> None
+    files = config.get("files", None)
+    if not files:
+        return
+
+    if isinstance(files, (list, tuple)):
+        files = dict(files)
+
+    print("Creating files from configuration")
+    for key, data in files.items():
+        path = data.get("path")
+        fmt = data.get("format", "string")
+        target_fmt = data.get("target_format", "string")
+        overwrite = bool(data.get("overwrite", True))
+        contents = data.get("contents")
+
+        target = Path(expanduser(expandvars(path)))
+
+        # noinspection PyBroadException
+        try:
+            if target.is_dir():
+                print("Skipped [{}]: is a directory {}".format(key, target))
+                continue
+
+            if not overwrite and target.is_file():
+                print("Skipped [{}]: file exists {}".format(key, target))
+                continue
+        except Exception as ex:
+            print("Skipped [{}]: can't access {} ({})".format(key, target, ex))
+            continue
+
+        if contents:
+            try:
+                if fmt == "base64":
+                    contents = base64.b64decode(contents)
+                    if target_fmt != "bytes":
+                        contents = contents.decode("utf-8")
+            except Exception as ex:
+                print("Skipped [{}]: failed decoding {} ({})".format(key, fmt, ex))
+                continue
+
+        # noinspection PyBroadException
+        try:
+            target.parent.mkdir(parents=True, exist_ok=True)
+        except Exception as ex:
+            print("Skipped [{}]: failed creating path {} ({})".format(key, target.parent, ex))
+            continue
+
+        try:
+            if target_fmt == "bytes":
+                try:
+                    target.write_bytes(contents)
+                except TypeError:
+                    # simpler error so the user won't get confused
+                    raise TypeError("a bytes-like object is required")
+            else:
+                try:
+                    if target_fmt == "json":
+                        text = HOCONConverter.to_json(contents)
+                    elif target_fmt in ("yaml", "yml"):
+                        text = HOCONConverter.to_yaml(contents)
+                    else:
+                        if isinstance(contents, ConfigTree):
+                            contents = contents.as_plain_ordered_dict()
+                        text = str(contents)
+                except Exception as ex:
+                    print("Skipped [{}]: failed encoding to {} ({})".format(key, target_fmt, ex))
+                    continue
+                target.write_text(text)
+            print("Saved [{}]: {}".format(key, target))
+        except Exception as ex:
+            print("Skipped [{}]: failed saving file {} ({})".format(key, target, ex))
+            continue

clearml/config/default/sdk.conf

@@ -191,4 +191,38 @@
             report_global_mem_used: false
         }
     }
+
+    # Apply top-level environment section from configuration into os.environ
+    apply_environment: false
+    # Top-level environment section is in the form of:
+    #   environment {
+    #     key: value
+    #     ...
+    #   }
+    # and is applied to the OS environment as `key=value` for each key/value pair
+
+    # Apply top-level files section from configuration into local file system
+    apply_files: false
+    # Top-level files section allows auto-generating files at designated paths with a predefined contents
+    # and target format. Options include:
+    #  contents: the target file's content, typically a string (or any base type int/float/list/dict etc.)
+    #  format: a custom format for the contents. Currently supported value is `base64` to automatically decode a
+    #          base64-encoded contents string, otherwise ignored
+    #  path: the target file's path, may include ~ and inplace env vars
+    #  target_format: format used to encode contents before writing into the target file. Supported values are json,
+    #                 yaml, yml and bytes (in which case the file will be written in binary mode). Default is text mode.
+    #  overwrite: overwrite the target file in case it exists. Default is true.
+    #
+    # Example:
+    #   files {
+    #     myfile1 {
+    #       contents: "The quick brown fox jumped over the lazy dog"
+    #       path: "/tmp/fox.txt"
+    #     }
+    #     myjsonfile {
+    #       contents: {some: nested: value: [1, 2, 3, 4]}
+    #       path: "/tmp/test.json"
+    #       target_format: json
+    #     }
+    #   }
 }