From 6e15349b7627bee3847a39e5bdce8c988e39cb38 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 2 Sep 2021 15:48:09 +0300 Subject: [PATCH] Fix unsafe Google Storage delete object --- clearml/backend_api/session/defs.py | 1 + clearml/backend_api/session/session.py | 49 ++++++++++++++++++++++++-- clearml/backend_config/config.py | 17 +++++++++ clearml/storage/helper.py | 14 +++++++- 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/clearml/backend_api/session/defs.py b/clearml/backend_api/session/defs.py index 612025d9..62c345a9 100644 --- a/clearml/backend_api/session/defs.py +++ b/clearml/backend_api/session/defs.py @@ -14,3 +14,4 @@ ENV_HOST_VERIFY_CERT = EnvEntry("CLEARML_API_HOST_VERIFY_CERT", "TRAINS_API_HOST ENV_OFFLINE_MODE = EnvEntry("CLEARML_OFFLINE_MODE", "TRAINS_OFFLINE_MODE", type=bool, converter=safe_text_to_bool) ENV_CLEARML_NO_DEFAULT_SERVER = EnvEntry("CLEARML_NO_DEFAULT_SERVER", "TRAINS_NO_DEFAULT_SERVER", converter=safe_text_to_bool, type=bool, default=True) +ENV_DISABLE_VAULT_SUPPORT = EnvEntry('CLEARML_DISABLE_VAULT_SUPPORT', type=bool) diff --git a/clearml/backend_api/session/session.py b/clearml/backend_api/session/session.py index 1a6c6a5c..4a952caf 100644 --- a/clearml/backend_api/session/session.py +++ b/clearml/backend_api/session/session.py @@ -12,14 +12,23 @@ from six.moves.urllib.parse import urlparse, urlunparse from .callresult import CallResult from .defs import ( - ENV_VERBOSE, ENV_HOST, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_WEB_HOST, - ENV_FILES_HOST, ENV_OFFLINE_MODE, ENV_CLEARML_NO_DEFAULT_SERVER, ENV_AUTH_TOKEN, ) + ENV_VERBOSE, + ENV_HOST, + ENV_ACCESS_KEY, + ENV_SECRET_KEY, + ENV_WEB_HOST, + ENV_FILES_HOST, + ENV_OFFLINE_MODE, + ENV_CLEARML_NO_DEFAULT_SERVER, + ENV_AUTH_TOKEN, + ENV_DISABLE_VAULT_SUPPORT, +) from .request import Request, BatchRequest # noqa: F401 from .token_manager import TokenManager from ..config import load from ..utils import get_http_session_with_retry, urllib_log_warning_setup from ...debugging import get_logger -from ...utilities.pyhocon import ConfigTree +from ...utilities.pyhocon import ConfigTree, ConfigFactory from ...version import __version__ try: @@ -59,6 +68,7 @@ class Session(TokenManager): api_version = '2.1' max_api_version = '2.1' + feature_set = 'basic' default_demo_host = "https://demoapi.demo.clear.ml" default_host = default_demo_host default_web = "https://demoapp.demo.clear.ml" @@ -198,6 +208,7 @@ class Session(TokenManager): Session._client.append(('clearml-server', token_dict.get('server_version'), )) Session.max_api_version = Session.api_version = str(api_version) + Session.feature_set = str(token_dict.get('feature_set', self.feature_set) or "basic") except (jwt.DecodeError, ValueError): (self._logger or get_logger()).warning( "Failed parsing server API level, defaulting to {}".format(Session.api_version)) @@ -212,6 +223,38 @@ class Session(TokenManager): if self.force_max_api_version and self.check_min_api_version(self.force_max_api_version): Session.max_api_version = Session.api_version = str(self.force_max_api_version) + def _load_vaults(self): + if not self.check_min_api_version("2.15") or self.feature_set == "basic": + return + + if ENV_DISABLE_VAULT_SUPPORT.get(): + print("Vault support is disabled") + return + + def parse(vault): + # noinspection PyBroadException + try: + d = vault.get('data', None) + if d: + r = ConfigFactory.parse_string(d) + if isinstance(r, (ConfigTree, dict)): + return r + except Exception as e: + print("Failed parsing vault {}: {}".format(vault.get("description", ""), e)) + + # noinspection PyBroadException + try: + res = self.send_request("users", "get_vaults", json={"enabled": True, "types": ["config"]}) + if res.ok: + vaults = res.json().get("data", {}).get("vaults", []) + data = list(filter(None, map(parse, vaults))) + if data: + self.config.set_overrides(*data) + elif res.status_code != 404: + raise Exception(res.json().get("meta", {}).get("result_msg", res.text)) + except Exception as ex: + print("Failed getting vaults: {}".format(ex)) + def _send_request( self, service, diff --git a/clearml/backend_config/config.py b/clearml/backend_config/config.py index 1fd10c2e..20b150b5 100644 --- a/clearml/backend_config/config.py +++ b/clearml/backend_config/config.py @@ -86,6 +86,7 @@ class Config(object): self._env = env or os.environ.get("CLEARML_ENV", os.environ.get("TRAINS_ENV", Environment.default)) self.config_paths = set() self.is_server = is_server + self._overrides_configs = None if self._verbose: print("Config env:%s" % str(self._env)) @@ -177,6 +178,15 @@ class Config(object): config, ) + if self._overrides_configs: + config = functools.reduce( + lambda cfg, override: ConfigTree.merge_configs( + cfg, override, copy_trees=True + ), + self._overrides_configs, + config, + ) + config["env"] = env return config @@ -401,3 +411,10 @@ class Config(object): bucket=bucket, host=host, ) + + def set_overrides(self, *dicts): + """ Set several override dictionaries or ConfigTree objects which should be merged onto the configuration """ + self._overrides_configs = [ + d if isinstance(d, ConfigTree) else ConfigFactory.from_dict(d) for d in dicts + ] + self.reload() diff --git a/clearml/storage/helper.py b/clearml/storage/helper.py index 604356d8..3c7a8e32 100644 --- a/clearml/storage/helper.py +++ b/clearml/storage/helper.py @@ -1543,7 +1543,19 @@ class _GoogleCloudStorageDriver(_Driver): return list(container.bucket.list_blobs()) def delete_object(self, object, **kwargs): - object.delete() + try: + object.delete() + except Exception as ex: + try: + from google.cloud.exceptions import NotFound + if isinstance(ex, NotFound): + return False + except ImportError: + pass + name = getattr(object, "name", "") + log.warning("Failed deleting object {}: {}".format(name, ex)) + return False + return not object.exists() def get_object(self, container_name, object_name, *args, **kwargs):