ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix misconfigured boto3 bucket credentials verify

Browse Source
This commit is contained in:
Alex Burlacu 2023-07-19 02:09:49 +03:00
parent c8c8a1224e
commit 75b4015fdb
2 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
clearml/backend_config/bucket_config.py
View File

@ -31,7 +31,7 @@ class S3BucketConfig(object):
acl = attrib(type=str, converter=_none_to_empty_string, default="")
secure = attrib(type=bool, default=True)
region = attrib(type=str, converter=_none_to_empty_string, default="")
verify = attrib(type=bool, default=True)
verify = attrib(type=bool, default=None)
use_credentials_chain = attrib(type=bool, default=False)
extra_args = attrib(type=dict, default=None)
@ -106,6 +106,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
default_use_credentials_chain=False,
default_token="",
default_extra_args=None,
default_verify=None,
):
super(S3BucketConfigurations, self).__init__()
self._buckets = buckets if buckets else list()
@ -116,6 +117,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
self._default_multipart = True
self._default_use_credentials_chain = default_use_credentials_chain
self._default_extra_args = default_extra_args
self._default_verify = default_verify
@classmethod
def from_config(cls, s3_configuration):
@ -129,6 +131,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
default_region = s3_configuration.get("region", "") or getenv("AWS_DEFAULT_REGION", "")
default_use_credentials_chain = s3_configuration.get("use_credentials_chain") or False
default_extra_args = s3_configuration.get("extra_args")
default_verify = s3_configuration.get("verify", None)
default_key = _none_to_empty_string(default_key)
default_secret = _none_to_empty_string(default_secret)
@ -142,7 +145,8 @@ class S3BucketConfigurations(BaseBucketConfigurations):
default_region,
default_use_credentials_chain,
default_token,
default_extra_args
default_extra_args,
default_verify,
)
def add_config(self, bucket_config):

8
clearml/storage/helper.py
View File

@ -437,12 +437,18 @@ class _Boto3Driver(_Driver):
self.name = name[5:]
endpoint = (('https://' if cfg.secure else 'http://') + cfg.host) if cfg.host else None
verify = cfg.verify
if verify is True:
# True is a non-documented value for boto3, use None instead (which means verify)
print("Using boto3 verify=None instead of true")
verify = None
# boto3 client creation isn't thread-safe (client itself is)
with self._creation_lock:
boto_kwargs = {
"endpoint_url": endpoint,
"use_ssl": cfg.secure,
"verify": cfg.verify,
"verify": verify,
"region_name": cfg.region or None, # None in case cfg.region is an empty string
"config": botocore.client.Config(
max_pool_connections=max(