ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-13 17:10:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Black formatting

Browse Source
This commit is contained in:
clearml 2025-03-04 16:43:47 +02:00
parent ff5fe3b27c
commit 4fa233ff76
1 changed files with 22 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
clearml/backend_config/bucket_config.py
View File

@ -66,9 +66,7 @@ class S3BucketConfig(object):
@classmethod @classmethod
def from_list(cls, dict_list, log=None): def from_list(cls, dict_list, log=None):
if not isinstance(dict_list, (tuple, list)) or not all( if not isinstance(dict_list, (tuple, list)) or not all(isinstance(x, dict) for x in dict_list):
isinstance(x, dict) for x in dict_list
):
raise ValueError("Expecting a list of configurations dictionaries") raise ValueError("Expecting a list of configurations dictionaries")
configs = [cls(**entry) for entry in dict_list] configs = [cls(**entry) for entry in dict_list]
valid_configs = [conf for conf in configs if conf.is_valid()] valid_configs = [conf for conf in configs if conf.is_valid()]
@ -76,9 +74,7 @@ class S3BucketConfig(object):
log.warning( log.warning(
"Invalid bucket configurations detected for {}".format( "Invalid bucket configurations detected for {}".format(
", ".join( ", ".join(
"/".join((config.host, config.bucket)) "/".join((config.host, config.bucket)) for config in configs if config not in valid_configs
for config in configs
if config not in valid_configs
) )
) )
) )
@ -97,10 +93,7 @@ class BaseBucketConfigurations(object):
def _update_prefixes(self, refresh=True): def _update_prefixes(self, refresh=True):
if self._prefixes and not refresh: if self._prefixes and not refresh:
return return
prefixes = ( prefixes = ((config, self._get_prefix_from_bucket_config(config)) for config in self._buckets)
(config, self._get_prefix_from_bucket_config(config))
for config in self._buckets
)
self._prefixes = sorted(prefixes, key=itemgetter(1), reverse=True) self._prefixes = sorted(prefixes, key=itemgetter(1), reverse=True)
@abc.abstractmethod @abc.abstractmethod
@ -179,9 +172,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
def get_config_by_bucket(self, bucket, host=None): def get_config_by_bucket(self, bucket, host=None):
try: try:
return next( return next(
bucket_config bucket_config for bucket_config in self._buckets if (bucket, host) == bucket_config.get_bucket_host()
for bucket_config in self._buckets
if (bucket, host) == bucket_config.get_bucket_host()
) )
except StopIteration: except StopIteration:
pass pass
@ -198,7 +189,7 @@ class S3BucketConfigurations(BaseBucketConfigurations):
token=self._default_token, token=self._default_token,
extra_args=self._default_extra_args, extra_args=self._default_extra_args,
profile=self._default_profile, profile=self._default_profile,
secure=self._default_secure secure=self._default_secure,
) )
def _get_prefix_from_bucket_config(self, config): def _get_prefix_from_bucket_config(self, config):
@ -225,14 +216,11 @@ class S3BucketConfigurations(BaseBucketConfigurations):
:param uri: URI of bucket, directory or file :param uri: URI of bucket, directory or file
:return: S3BucketConfig: bucket config :return: S3BucketConfig: bucket config
""" """
def find_match(uri): def find_match(uri):
self._update_prefixes(refresh=False) self._update_prefixes(refresh=False)
uri = uri.lower() uri = uri.lower()
res = ( res = (config for config, prefix in self._prefixes if prefix is not None and uri.startswith(prefix))
config
for config, prefix in self._prefixes
if prefix is not None and uri.startswith(prefix)
)
try: try:
return next(res) return next(res)
@ -294,12 +282,12 @@ class GSBucketConfig(object):
class GSBucketConfigurations(BaseBucketConfigurations): class GSBucketConfigurations(BaseBucketConfigurations):
def __init__( def __init__(
self, self,
buckets=None, buckets=None,
default_project=None, default_project=None,
default_credentials=None, default_credentials=None,
default_pool_connections=None, default_pool_connections=None,
default_pool_maxsize=None default_pool_maxsize=None
): ):
super(GSBucketConfigurations, self).__init__(buckets) super(GSBucketConfigurations, self).__init__(buckets)
self._default_project = default_project self._default_project = default_project
@ -339,8 +327,7 @@ class GSBucketConfigurations(BaseBucketConfigurations):
def update_config_with_defaults(self, bucket_config): def update_config_with_defaults(self, bucket_config):
bucket_config.update( bucket_config.update(
project=bucket_config.project or self._default_project, project=bucket_config.project or self._default_project,
credentials_json=bucket_config.credentials_json credentials_json=bucket_config.credentials_json or self._default_credentials,
or self._default_credentials,
pool_connections=bucket_config.pool_connections or self._default_pool_connections, pool_connections=bucket_config.pool_connections or self._default_pool_connections,
pool_maxsize=bucket_config.pool_maxsize or self._default_pool_maxsize pool_maxsize=bucket_config.pool_maxsize or self._default_pool_maxsize
) )
@ -354,11 +341,7 @@ class GSBucketConfigurations(BaseBucketConfigurations):
:return: GSBucketConfig: bucket config :return: GSBucketConfig: bucket config
""" """
res = ( res = (config for config, prefix in self._prefixes if prefix is not None and uri.lower().startswith(prefix))
config
for config, prefix in self._prefixes
if prefix is not None and uri.lower().startswith(prefix)
)
try: try:
return next(res) return next(res)
@ -413,16 +396,12 @@ class AzureContainerConfigurations(object):
default_container_configs = [] default_container_configs = []
if default_account and default_key: if default_account and default_key:
default_container_configs.append(AzureContainerConfig( default_container_configs.append(
account_name=default_account, account_key=default_key AzureContainerConfig(account_name=default_account, account_key=default_key)
)) )
if configuration is None: if configuration is None:
return cls( return cls(default_container_configs, default_account=default_account, default_key=default_key)
default_container_configs,
default_account=default_account,
default_key=default_key
)
containers = configuration.get("containers", list()) containers = configuration.get("containers", list())
container_configs = [AzureContainerConfig(**entry) for entry in containers] + default_container_configs container_configs = [AzureContainerConfig(**entry) for entry in containers] + default_container_configs
@ -441,9 +420,7 @@ class AzureContainerConfigurations(object):
if not f.path.segments: if not f.path.segments:
raise ValueError( raise ValueError(
"URI {} is missing a container name (expected " "URI {} is missing a container name (expected "
"[https/azure]://<account-name>.../<container-name>)".format( "[https/azure]://<account-name>.../<container-name>)".format(uri)
uri
)
) )
container = f.path.segments[0] container = f.path.segments[0]
@ -460,10 +437,8 @@ class AzureContainerConfigurations(object):
( (
config config
for config in self._container_configs for config in self._container_configs
if config.account_name == account_name and ( if config.account_name == account_name
not config.container_name and (not config.container_name or config.container_name == container)
or config.container_name == container
)
), ),
None None
) )