Use default boto credential chain if no keys in the config (#342)

* Use default boto credential chain if no keys in the config

* Add use_credentials_chain config to explicitly enable boto cred chain

* Propagate the new config param in the right way

* PEP8 fixes

clearml/backend_config/bucket_config.py

@@ -30,12 +30,14 @@ class S3BucketConfig(object):
     secure = attrib(type=bool, default=True)
     region = attrib(type=str, converter=_none_to_empty_string, default="")
     verify = attrib(type=bool, default=True)
+    use_credentials_chain = attrib(type=bool, default=False)
 
-    def update(self, key, secret, multipart=True, region=None):
+    def update(self, key, secret, multipart=True, region=None, use_credentials_chain=False):
         self.key = key
         self.secret = secret
         self.multipart = multipart
         self.region = region
+        self.use_credentials_chain = use_credentials_chain
 
     def is_valid(self):
         return self.key and self.secret
@@ -89,7 +91,7 @@ class BaseBucketConfigurations(object):
 
 class S3BucketConfigurations(BaseBucketConfigurations):
     def __init__(
-        self, buckets=None, default_key="", default_secret="", default_region=""
+        self, buckets=None, default_key="", default_secret="", default_region="", default_use_credentials_chain=False
     ):
         super(S3BucketConfigurations, self).__init__()
         self._buckets = buckets if buckets else list()
@@ -97,6 +99,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
         self._default_secret = default_secret
         self._default_region = default_region
         self._default_multipart = True
+        self._default_use_credentials_chain = default_use_credentials_chain
 
     @classmethod
     def from_config(cls, s3_configuration):
@@ -107,12 +110,13 @@ class S3BucketConfigurations(BaseBucketConfigurations):
         default_key = s3_configuration.get("key") or getenv("AWS_ACCESS_KEY_ID", "")
         default_secret = s3_configuration.get("secret") or getenv("AWS_SECRET_ACCESS_KEY", "")
         default_region = s3_configuration.get("region") or getenv("AWS_DEFAULT_REGION", "")
+        default_use_credentials_chain = s3_configuration.get("use_credentials_chain") or False
 
         default_key = _none_to_empty_string(default_key)
         default_secret = _none_to_empty_string(default_secret)
         default_region = _none_to_empty_string(default_region)
 
-        return cls(config_list, default_key, default_secret, default_region)
+        return cls(config_list, default_key, default_secret, default_region, default_use_credentials_chain)
 
     def add_config(self, bucket_config):
         self._buckets.insert(0, bucket_config)
@@ -140,6 +144,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
             secret=self._default_secret,
             region=bucket_config.region or self._default_region,
             multipart=bucket_config.multipart or self._default_multipart,
+            use_credentials_chain=self._default_use_credentials_chain
         )
 
     def _get_prefix_from_bucket_config(self, config):
@@ -201,6 +206,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
             secret=self._default_secret,
             region=self._default_region,
             multipart=True,
+            use_credentials_chain=self._default_use_credentials_chain,
             bucket=bucket,
             host=host,
         )

clearml/backend_config/config.py

@@ -401,6 +401,7 @@ class Config(object):
             key=self.get("sdk.aws.s3.key", None),
             secret=self.get("sdk.aws.s3.secret", None),
             region=self.get("sdk.aws.s3.region", None),
+            use_credentials_chain=self.get("sdk.aws.s3.use_credentials_chain", None),
             multipart=True,
             bucket=bucket,
             host=host,

clearml/config/default/sdk.conf

@@ -63,10 +63,16 @@
         s3 {
             # S3 credentials, used for read/write access by various SDK elements
 
-            # default, used for any bucket not specified below
+            # Default, used for any bucket not specified below
+            region: ""
+            # Specify explicit keys
             key: ""
             secret: ""
-            region: ""
+            # Or enable credentials chain to let Boto3 pick the right credentials. 
+            # This includes picking credentials from environment variables, 
+            # credential file and IAM role using metadata service. 
+            # Refer to the latest Boto3 docs
+            use_credentials_chain: false
 
             credentials: [
                 # specifies key/secret credentials to use when handling s3 urls (read or write)

clearml/storage/helper.py

@@ -306,10 +306,14 @@ class StorageHelper(object):
                 secret=secret or self._conf.secret,
                 multipart=self._conf.multipart,
                 region=final_region,
+                use_credentials_chain=self._conf.use_credentials_chain
             )
 
+            if not self._conf.use_credentials_chain:
                 if not self._conf.key or not self._conf.secret:
-                raise ValueError('Missing key and secret for S3 storage access (%s)' % base_url)
+                    raise ValueError(
+                        "Missing key and secret for S3 storage access (%s)" % base_url
+                    )
 
             self._driver = _Boto3Driver()
             self._container = self._driver.get_container(container_name=self._base_url, retries=retries,
@@ -1237,18 +1241,23 @@ class _Boto3Driver(_Driver):
 
             # boto3 client creation isn't thread-safe (client itself is)
             with self._creation_lock:
-                self.resource = boto3.resource(
+                boto_kwargs = {
-                    's3',
+                    "endpoint_url": endpoint,
-                    aws_access_key_id=cfg.key,
+                    "use_ssl": cfg.secure,
-                    aws_secret_access_key=cfg.secret,
+                    "verify": cfg.verify,
-                    endpoint_url=endpoint,
+                    "config": botocore.client.Config(
-                    use_ssl=cfg.secure,
-                    verify=cfg.verify,
-                    config=botocore.client.Config(
                         max_pool_connections=max(
                             _Boto3Driver._min_pool_connections,
                             _Boto3Driver._pool_connections)
-                    ),
+                    )
+                }
+                if not cfg.use_credentials_chain:
+                    boto_kwargs["aws_access_key_id"] = cfg.key
+                    boto_kwargs["aws_secret_access_key"] = cfg.secret
+
+                self.resource = boto3.resource(
+                    's3',
+                    **boto_kwargs
                 )
 
                 self.config = cfg

docs/clearml.conf

@@ -73,10 +73,17 @@ sdk {
         s3 {
             # S3 credentials, used for read/write access by various SDK elements
 
-            # default, used for any bucket not specified below
+            # Default, used for any bucket not specified below
+            region: ""
+            # Specify explicit keys
             key: ""
             secret: ""
-            region: ""
+            # Or enable credentials chain to let Boto3 pick the right credentials. 
+            # This includes picking credentials from environment variables, 
+            # credential file and IAM role using metadata service. 
+            # Refer to the latest Boto3 docs
+            use_credentials_chain: false
+
 
             credentials: [
                 # specifies key/secret credentials to use when handling s3 urls (read or write)

docs/trains.conf

@@ -73,10 +73,17 @@ sdk {
         s3 {
             # S3 credentials, used for read/write access by various SDK elements
 
-            # default, used for any bucket not specified below
+            # Default, used for any bucket not specified below
+            region: ""
+            # Specify explicit keys
             key: ""
             secret: ""
-            region: ""
+            # Or enable credentials chain to let Boto3 pick the right credentials. 
+            # This includes picking credentials from environment variables, 
+            # credential file and IAM role using metadata service. 
+            # Refer to the latest Boto3 docs
+            use_credentials_chain: false
+
 
             credentials: [
                 # specifies key/secret credentials to use when handling s3 urls (read or write)