ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix misconfigured boto3 bucket credentials verify

Browse Source
This commit is contained in:
Alex Burlacu 2023-07-19 02:09:49 +03:00
parent c8c8a1224e
commit 75b4015fdb
2 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
clearml/backend_config/bucket_config.py
View File

@ -31,7 +31,7 @@ class S3BucketConfig(object):
acl = attrib(type=str, converter=_none_to_empty_string, default="") acl = attrib(type=str, converter=_none_to_empty_string, default="")
secure = attrib(type=bool, default=True) secure = attrib(type=bool, default=True)
region = attrib(type=str, converter=_none_to_empty_string, default="") region = attrib(type=str, converter=_none_to_empty_string, default="")
verify = attrib(type=bool, default=True) verify = attrib(type=bool, default=None)
use_credentials_chain = attrib(type=bool, default=False) use_credentials_chain = attrib(type=bool, default=False)
extra_args = attrib(type=dict, default=None) extra_args = attrib(type=dict, default=None)
@ -106,6 +106,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
default_use_credentials_chain=False, default_use_credentials_chain=False,
default_token="", default_token="",
default_extra_args=None, default_extra_args=None,
default_verify=None,
): ):
super(S3BucketConfigurations, self).__init__() super(S3BucketConfigurations, self).__init__()
self._buckets = buckets if buckets else list() self._buckets = buckets if buckets else list()
@ -116,6 +117,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
self._default_multipart = True self._default_multipart = True
self._default_use_credentials_chain = default_use_credentials_chain self._default_use_credentials_chain = default_use_credentials_chain
self._default_extra_args = default_extra_args self._default_extra_args = default_extra_args
self._default_verify = default_verify
@classmethod @classmethod
def from_config(cls, s3_configuration): def from_config(cls, s3_configuration):
@ -129,6 +131,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
default_region = s3_configuration.get("region", "") or getenv("AWS_DEFAULT_REGION", "") default_region = s3_configuration.get("region", "") or getenv("AWS_DEFAULT_REGION", "")
default_use_credentials_chain = s3_configuration.get("use_credentials_chain") or False default_use_credentials_chain = s3_configuration.get("use_credentials_chain") or False
default_extra_args = s3_configuration.get("extra_args") default_extra_args = s3_configuration.get("extra_args")
default_verify = s3_configuration.get("verify", None)
default_key = _none_to_empty_string(default_key) default_key = _none_to_empty_string(default_key)
default_secret = _none_to_empty_string(default_secret) default_secret = _none_to_empty_string(default_secret)
@ -142,7 +145,8 @@ class S3BucketConfigurations(BaseBucketConfigurations):
default_region, default_region,
default_use_credentials_chain, default_use_credentials_chain,
default_token, default_token,
default_extra_args default_extra_args,
default_verify,
) )
def add_config(self, bucket_config): def add_config(self, bucket_config):

8
clearml/storage/helper.py
View File

@ -437,12 +437,18 @@ class _Boto3Driver(_Driver):
self.name = name[5:] self.name = name[5:]
endpoint = (('https://' if cfg.secure else 'http://') + cfg.host) if cfg.host else None endpoint = (('https://' if cfg.secure else 'http://') + cfg.host) if cfg.host else None
verify = cfg.verify
if verify is True:
# True is a non-documented value for boto3, use None instead (which means verify)
print("Using boto3 verify=None instead of true")
verify = None
# boto3 client creation isn't thread-safe (client itself is) # boto3 client creation isn't thread-safe (client itself is)
with self._creation_lock: with self._creation_lock:
boto_kwargs = { boto_kwargs = {
"endpoint_url": endpoint, "endpoint_url": endpoint,
"use_ssl": cfg.secure, "use_ssl": cfg.secure,
"verify": cfg.verify, "verify": verify,
"region_name": cfg.region or None, # None in case cfg.region is an empty string "region_name": cfg.region or None, # None in case cfg.region is an empty string
"config": botocore.client.Config( "config": botocore.client.Config(
max_pool_connections=max( max_pool_connections=max(