ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-04 13:02:13 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Added boto3 parameters for multipart configurations

Browse Source 
- multipart threshold and chunksize
This commit is contained in:
AbdulHamid Merii 2023-06-25 06:05:53 +01:00 committed by Colin Gaudreau
parent cbd14e3d6a
commit 9e8ed16b3e
1 changed files with 20 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
clearml/storage/helper.py
View File

@ -401,6 +401,8 @@ class _Boto3Driver(_Driver):
_min_pool_connections = 512 _min_pool_connections = 512
_max_multipart_concurrency = deferred_config('aws.boto3.max_multipart_concurrency', 16) _max_multipart_concurrency = deferred_config('aws.boto3.max_multipart_concurrency', 16)
_multipart_threshold = deferred_config('aws.boto3.multipart_threshold', (1024 ** 2) * 8) # 8 MB
_multipart_chunksize = deferred_config('aws.boto3.multipart_chunksize', (1024 ** 2) * 8)
_pool_connections = deferred_config('aws.boto3.pool_connections', 512) _pool_connections = deferred_config('aws.boto3.pool_connections', 512)
_connect_timeout = deferred_config('aws.boto3.connect_timeout', 60) _connect_timeout = deferred_config('aws.boto3.connect_timeout', 60)
_read_timeout = deferred_config('aws.boto3.read_timeout', 60) _read_timeout = deferred_config('aws.boto3.read_timeout', 60)
@ -498,7 +500,9 @@ class _Boto3Driver(_Driver):
container.bucket.upload_fileobj(stream, object_name, Config=boto3.s3.transfer.TransferConfig( container.bucket.upload_fileobj(stream, object_name, Config=boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
num_download_attempts=container.config.retries), num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold,
multipart_chunksize=self._multipart_chunksize),
Callback=callback, Callback=callback,
ExtraArgs=extra_args, ExtraArgs=extra_args,
) )
@ -512,6 +516,8 @@ class _Boto3Driver(_Driver):
Config=boto3.s3.transfer.TransferConfig( Config=boto3.s3.transfer.TransferConfig(
use_threads=False, use_threads=False,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold,
multipart_chunksize=self._multipart_chunksize,
), ),
Callback=callback, Callback=callback,
ExtraArgs=extra_args ExtraArgs=extra_args
@ -535,7 +541,9 @@ class _Boto3Driver(_Driver):
container.bucket.upload_file(file_path, object_name, Config=boto3.s3.transfer.TransferConfig( container.bucket.upload_file(file_path, object_name, Config=boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
num_download_attempts=container.config.retries), num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold,
multipart_chunksize=self._multipart_chunksize),
Callback=callback, Callback=callback,
ExtraArgs=extra_args, ExtraArgs=extra_args,
) )
@ -547,7 +555,10 @@ class _Boto3Driver(_Driver):
file_path, file_path,
object_name, object_name,
Config=boto3.s3.transfer.TransferConfig( Config=boto3.s3.transfer.TransferConfig(
use_threads=False, num_download_attempts=container.config.retries use_threads=False,
num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold,
multipart_chunksize=self._multipart_chunksize
), ),
Callback=callback, Callback=callback,
ExtraArgs=extra_args ExtraArgs=extra_args
@ -600,7 +611,9 @@ class _Boto3Driver(_Driver):
config = boto3.s3.transfer.TransferConfig( config = boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
num_download_attempts=container.config.retries) num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold,
multipart_chunksize=self._multipart_chunksize)
total_size_mb = obj.content_length / (1024. * 1024.) total_size_mb = obj.content_length / (1024. * 1024.)
remote_path = os.path.join(obj.container_name, obj.key) remote_path = os.path.join(obj.container_name, obj.key)
cb = DownloadProgressReport(total_size_mb, verbose, remote_path, log) cb = DownloadProgressReport(total_size_mb, verbose, remote_path, log)
@ -618,7 +631,9 @@ class _Boto3Driver(_Driver):
Config = boto3.s3.transfer.TransferConfig( Config = boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
num_download_attempts=container.config.retries num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold,
multipart_chunksize=self._multipart_chunksize
) )
obj.download_file(str(p), Callback=callback, Config=Config) obj.download_file(str(p), Callback=callback, Config=Config)