diff --git a/clearml/backend_config/bucket_config.py b/clearml/backend_config/bucket_config.py index 28c84141..0c35432c 100644 --- a/clearml/backend_config/bucket_config.py +++ b/clearml/backend_config/bucket_config.py @@ -30,12 +30,14 @@ class S3BucketConfig(object): secure = attrib(type=bool, default=True) region = attrib(type=str, converter=_none_to_empty_string, default="") verify = attrib(type=bool, default=True) + use_credentials_chain = attrib(type=bool, default=False) - def update(self, key, secret, multipart=True, region=None): + def update(self, key, secret, multipart=True, region=None, use_credentials_chain=False): self.key = key self.secret = secret self.multipart = multipart self.region = region + self.use_credentials_chain = use_credentials_chain def is_valid(self): return self.key and self.secret @@ -89,7 +91,7 @@ class BaseBucketConfigurations(object): class S3BucketConfigurations(BaseBucketConfigurations): def __init__( - self, buckets=None, default_key="", default_secret="", default_region="" + self, buckets=None, default_key="", default_secret="", default_region="", default_use_credentials_chain=False ): super(S3BucketConfigurations, self).__init__() self._buckets = buckets if buckets else list() @@ -97,6 +99,7 @@ class S3BucketConfigurations(BaseBucketConfigurations): self._default_secret = default_secret self._default_region = default_region self._default_multipart = True + self._default_use_credentials_chain = default_use_credentials_chain @classmethod def from_config(cls, s3_configuration): @@ -107,12 +110,13 @@ class S3BucketConfigurations(BaseBucketConfigurations): default_key = s3_configuration.get("key") or getenv("AWS_ACCESS_KEY_ID", "") default_secret = s3_configuration.get("secret") or getenv("AWS_SECRET_ACCESS_KEY", "") default_region = s3_configuration.get("region") or getenv("AWS_DEFAULT_REGION", "") + default_use_credentials_chain = s3_configuration.get("use_credentials_chain") or False default_key = _none_to_empty_string(default_key) default_secret = _none_to_empty_string(default_secret) default_region = _none_to_empty_string(default_region) - return cls(config_list, default_key, default_secret, default_region) + return cls(config_list, default_key, default_secret, default_region, default_use_credentials_chain) def add_config(self, bucket_config): self._buckets.insert(0, bucket_config) @@ -140,6 +144,7 @@ class S3BucketConfigurations(BaseBucketConfigurations): secret=self._default_secret, region=bucket_config.region or self._default_region, multipart=bucket_config.multipart or self._default_multipart, + use_credentials_chain=self._default_use_credentials_chain ) def _get_prefix_from_bucket_config(self, config): @@ -201,6 +206,7 @@ class S3BucketConfigurations(BaseBucketConfigurations): secret=self._default_secret, region=self._default_region, multipart=True, + use_credentials_chain=self._default_use_credentials_chain, bucket=bucket, host=host, ) diff --git a/clearml/backend_config/config.py b/clearml/backend_config/config.py index b79cb314..9389971b 100644 --- a/clearml/backend_config/config.py +++ b/clearml/backend_config/config.py @@ -401,6 +401,7 @@ class Config(object): key=self.get("sdk.aws.s3.key", None), secret=self.get("sdk.aws.s3.secret", None), region=self.get("sdk.aws.s3.region", None), + use_credentials_chain=self.get("sdk.aws.s3.use_credentials_chain", None), multipart=True, bucket=bucket, host=host, diff --git a/clearml/config/default/sdk.conf b/clearml/config/default/sdk.conf index 20c19a07..c3a42a75 100644 --- a/clearml/config/default/sdk.conf +++ b/clearml/config/default/sdk.conf @@ -63,10 +63,16 @@ s3 { # S3 credentials, used for read/write access by various SDK elements - # default, used for any bucket not specified below + # Default, used for any bucket not specified below + region: "" + # Specify explicit keys key: "" secret: "" - region: "" + # Or enable credentials chain to let Boto3 pick the right credentials. + # This includes picking credentials from environment variables, + # credential file and IAM role using metadata service. + # Refer to the latest Boto3 docs + use_credentials_chain: false credentials: [ # specifies key/secret credentials to use when handling s3 urls (read or write) diff --git a/clearml/storage/helper.py b/clearml/storage/helper.py index b3a8beb0..6ab44747 100644 --- a/clearml/storage/helper.py +++ b/clearml/storage/helper.py @@ -306,10 +306,14 @@ class StorageHelper(object): secret=secret or self._conf.secret, multipart=self._conf.multipart, region=final_region, + use_credentials_chain=self._conf.use_credentials_chain ) - if not self._conf.key or not self._conf.secret: - raise ValueError('Missing key and secret for S3 storage access (%s)' % base_url) + if not self._conf.use_credentials_chain: + if not self._conf.key or not self._conf.secret: + raise ValueError( + "Missing key and secret for S3 storage access (%s)" % base_url + ) self._driver = _Boto3Driver() self._container = self._driver.get_container(container_name=self._base_url, retries=retries, @@ -1237,18 +1241,23 @@ class _Boto3Driver(_Driver): # boto3 client creation isn't thread-safe (client itself is) with self._creation_lock: - self.resource = boto3.resource( - 's3', - aws_access_key_id=cfg.key, - aws_secret_access_key=cfg.secret, - endpoint_url=endpoint, - use_ssl=cfg.secure, - verify=cfg.verify, - config=botocore.client.Config( + boto_kwargs = { + "endpoint_url": endpoint, + "use_ssl": cfg.secure, + "verify": cfg.verify, + "config": botocore.client.Config( max_pool_connections=max( _Boto3Driver._min_pool_connections, _Boto3Driver._pool_connections) - ), + ) + } + if not cfg.use_credentials_chain: + boto_kwargs["aws_access_key_id"] = cfg.key + boto_kwargs["aws_secret_access_key"] = cfg.secret + + self.resource = boto3.resource( + 's3', + **boto_kwargs ) self.config = cfg diff --git a/docs/clearml.conf b/docs/clearml.conf index cda232ad..18f22dc7 100644 --- a/docs/clearml.conf +++ b/docs/clearml.conf @@ -73,10 +73,17 @@ sdk { s3 { # S3 credentials, used for read/write access by various SDK elements - # default, used for any bucket not specified below + # Default, used for any bucket not specified below + region: "" + # Specify explicit keys key: "" secret: "" - region: "" + # Or enable credentials chain to let Boto3 pick the right credentials. + # This includes picking credentials from environment variables, + # credential file and IAM role using metadata service. + # Refer to the latest Boto3 docs + use_credentials_chain: false + credentials: [ # specifies key/secret credentials to use when handling s3 urls (read or write) diff --git a/docs/trains.conf b/docs/trains.conf index 552a4488..d47ec7f6 100644 --- a/docs/trains.conf +++ b/docs/trains.conf @@ -73,10 +73,17 @@ sdk { s3 { # S3 credentials, used for read/write access by various SDK elements - # default, used for any bucket not specified below + # Default, used for any bucket not specified below + region: "" + # Specify explicit keys key: "" secret: "" - region: "" + # Or enable credentials chain to let Boto3 pick the right credentials. + # This includes picking credentials from environment variables, + # credential file and IAM role using metadata service. + # Refer to the latest Boto3 docs + use_credentials_chain: false + credentials: [ # specifies key/secret credentials to use when handling s3 urls (read or write)