Fix unsafe Google Storage delete object

clearml/backend_api/session/defs.py

@@ -14,3 +14,4 @@ ENV_HOST_VERIFY_CERT = EnvEntry("CLEARML_API_HOST_VERIFY_CERT", "TRAINS_API_HOST
 ENV_OFFLINE_MODE = EnvEntry("CLEARML_OFFLINE_MODE", "TRAINS_OFFLINE_MODE", type=bool, converter=safe_text_to_bool)
 ENV_CLEARML_NO_DEFAULT_SERVER = EnvEntry("CLEARML_NO_DEFAULT_SERVER", "TRAINS_NO_DEFAULT_SERVER",
                                          converter=safe_text_to_bool, type=bool, default=True)
+ENV_DISABLE_VAULT_SUPPORT = EnvEntry('CLEARML_DISABLE_VAULT_SUPPORT', type=bool)

clearml/backend_api/session/session.py

@@ -12,14 +12,23 @@ from six.moves.urllib.parse import urlparse, urlunparse
 
 from .callresult import CallResult
 from .defs import (
-    ENV_VERBOSE, ENV_HOST, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_WEB_HOST,
-    ENV_FILES_HOST, ENV_OFFLINE_MODE, ENV_CLEARML_NO_DEFAULT_SERVER, ENV_AUTH_TOKEN, )
+    ENV_VERBOSE,
+    ENV_HOST,
+    ENV_ACCESS_KEY,
+    ENV_SECRET_KEY,
+    ENV_WEB_HOST,
+    ENV_FILES_HOST,
+    ENV_OFFLINE_MODE,
+    ENV_CLEARML_NO_DEFAULT_SERVER,
+    ENV_AUTH_TOKEN,
+    ENV_DISABLE_VAULT_SUPPORT,
+)
 from .request import Request, BatchRequest  # noqa: F401
 from .token_manager import TokenManager
 from ..config import load
 from ..utils import get_http_session_with_retry, urllib_log_warning_setup
 from ...debugging import get_logger
-from ...utilities.pyhocon import ConfigTree
+from ...utilities.pyhocon import ConfigTree, ConfigFactory
 from ...version import __version__
 
 try:
@@ -59,6 +68,7 @@ class Session(TokenManager):
 
     api_version = '2.1'
     max_api_version = '2.1'
+    feature_set = 'basic'
     default_demo_host = "https://demoapi.demo.clear.ml"
     default_host = default_demo_host
     default_web = "https://demoapp.demo.clear.ml"
@@ -198,6 +208,7 @@ class Session(TokenManager):
                     Session._client.append(('clearml-server', token_dict.get('server_version'), ))
 
             Session.max_api_version = Session.api_version = str(api_version)
+            Session.feature_set = str(token_dict.get('feature_set', self.feature_set) or "basic")
         except (jwt.DecodeError, ValueError):
             (self._logger or get_logger()).warning(
                 "Failed parsing server API level, defaulting to {}".format(Session.api_version))
@@ -212,6 +223,38 @@ class Session(TokenManager):
         if self.force_max_api_version and self.check_min_api_version(self.force_max_api_version):
             Session.max_api_version = Session.api_version = str(self.force_max_api_version)
 
+    def _load_vaults(self):
+        if not self.check_min_api_version("2.15") or self.feature_set == "basic":
+            return
+
+        if ENV_DISABLE_VAULT_SUPPORT.get():
+            print("Vault support is disabled")
+            return
+
+        def parse(vault):
+            # noinspection PyBroadException
+            try:
+                d = vault.get('data', None)
+                if d:
+                    r = ConfigFactory.parse_string(d)
+                    if isinstance(r, (ConfigTree, dict)):
+                        return r
+            except Exception as e:
+                print("Failed parsing vault {}: {}".format(vault.get("description", "<unknown>"), e))
+
+        # noinspection PyBroadException
+        try:
+            res = self.send_request("users", "get_vaults", json={"enabled": True, "types": ["config"]})
+            if res.ok:
+                vaults = res.json().get("data", {}).get("vaults", [])
+                data = list(filter(None, map(parse, vaults)))
+                if data:
+                    self.config.set_overrides(*data)
+            elif res.status_code != 404:
+                raise Exception(res.json().get("meta", {}).get("result_msg", res.text))
+        except Exception as ex:
+            print("Failed getting vaults: {}".format(ex))
+
     def _send_request(
         self,
         service,

clearml/backend_config/config.py

@@ -86,6 +86,7 @@ class Config(object):
         self._env = env or os.environ.get("CLEARML_ENV", os.environ.get("TRAINS_ENV", Environment.default))
         self.config_paths = set()
         self.is_server = is_server
+        self._overrides_configs = None
 
         if self._verbose:
             print("Config env:%s" % str(self._env))
@@ -177,6 +178,15 @@ class Config(object):
                 config,
             )
 
+        if self._overrides_configs:
+            config = functools.reduce(
+                lambda cfg, override: ConfigTree.merge_configs(
+                    cfg, override, copy_trees=True
+                ),
+                self._overrides_configs,
+                config,
+            )
+
         config["env"] = env
         return config
 
@@ -401,3 +411,10 @@ class Config(object):
             bucket=bucket,
             host=host,
         )
+
+    def set_overrides(self, *dicts):
+        """ Set several override dictionaries or ConfigTree objects which should be merged onto the configuration """
+        self._overrides_configs = [
+            d if isinstance(d, ConfigTree) else ConfigFactory.from_dict(d) for d in dicts
+        ]
+        self.reload()

clearml/storage/helper.py

@@ -1543,7 +1543,19 @@ class _GoogleCloudStorageDriver(_Driver):
         return list(container.bucket.list_blobs())
 
     def delete_object(self, object, **kwargs):
-        object.delete()
+        try:
+            object.delete()
+        except Exception as ex:
+            try:
+                from google.cloud.exceptions import NotFound
+                if isinstance(ex, NotFound):
+                    return False
+            except ImportError:
+                pass
+            name = getattr(object, "name", "")
+            log.warning("Failed deleting object {}: {}".format(name, ex))
+            return False
+
         return not object.exists()
 
     def get_object(self, container_name, object_name, *args, **kwargs):