ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-16 11:28:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix S3 uploads fail with LazyEvalWrapper type error (#1081)

Browse Source
This commit is contained in:
allegroai 2023-07-30 13:04:42 +03:00
parent b1f99abc96
commit e467c05fb4
1 changed files with 31 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
clearml/storage/helper.py
View File

@ -504,12 +504,16 @@ class _Boto3Driver(_Driver):
'ContentType': get_file_mimetype(object_name) 'ContentType': get_file_mimetype(object_name)
} }
extra_args.update(container.config.extra_args or {}) extra_args.update(container.config.extra_args or {})
container.bucket.upload_fileobj(stream, object_name, Config=boto3.s3.transfer.TransferConfig( container.bucket.upload_fileobj(
stream,
object_name,
Config=boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=int(self._max_multipart_concurrency) if container.config.multipart else 1,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold, multipart_threshold=int(self._multipart_threshold),
multipart_chunksize=self._multipart_chunksize), multipart_chunksize=int(self._multipart_chunksize),
),
Callback=callback, Callback=callback,
ExtraArgs=extra_args, ExtraArgs=extra_args,
) )
@ -523,8 +527,8 @@ class _Boto3Driver(_Driver):
Config=boto3.s3.transfer.TransferConfig( Config=boto3.s3.transfer.TransferConfig(
use_threads=False, use_threads=False,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold, multipart_threshold=int(self._multipart_threshold),
multipart_chunksize=self._multipart_chunksize, multipart_chunksize=int(self._multipart_chunksize),
), ),
Callback=callback, Callback=callback,
ExtraArgs=extra_args ExtraArgs=extra_args
@ -545,12 +549,16 @@ class _Boto3Driver(_Driver):
'ContentType': get_file_mimetype(object_name or file_path) 'ContentType': get_file_mimetype(object_name or file_path)
} }
extra_args.update(container.config.extra_args or {}) extra_args.update(container.config.extra_args or {})
container.bucket.upload_file(file_path, object_name, Config=boto3.s3.transfer.TransferConfig( container.bucket.upload_file(
file_path,
object_name,
Config=boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=int(self._max_multipart_concurrency) if container.config.multipart else 1,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold, multipart_threshold=int(self._multipart_threshold),
multipart_chunksize=self._multipart_chunksize), multipart_chunksize=int(self._multipart_chunksize),
),
Callback=callback, Callback=callback,
ExtraArgs=extra_args, ExtraArgs=extra_args,
) )
@ -564,8 +572,8 @@ class _Boto3Driver(_Driver):
Config=boto3.s3.transfer.TransferConfig( Config=boto3.s3.transfer.TransferConfig(
use_threads=False, use_threads=False,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold, multipart_threshold=int(self._multipart_threshold),
multipart_chunksize=self._multipart_chunksize multipart_chunksize=int(self._multipart_chunksize)
), ),
Callback=callback, Callback=callback,
ExtraArgs=extra_args ExtraArgs=extra_args
@ -617,10 +625,11 @@ class _Boto3Driver(_Driver):
container = self._containers[obj.container_name] container = self._containers[obj.container_name]
config = boto3.s3.transfer.TransferConfig( config = boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=int(self._max_multipart_concurrency) if container.config.multipart else 1,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold, multipart_threshold=int(self._multipart_threshold),
multipart_chunksize=self._multipart_chunksize) multipart_chunksize=int(self._multipart_chunksize),
)
total_size_mb = obj.content_length / (1024. * 1024.) total_size_mb = obj.content_length / (1024. * 1024.)
remote_path = os.path.join(obj.container_name, obj.key) remote_path = os.path.join(obj.container_name, obj.key)
cb = DownloadProgressReport(total_size_mb, verbose, remote_path, log) cb = DownloadProgressReport(total_size_mb, verbose, remote_path, log)
@ -637,10 +646,10 @@ class _Boto3Driver(_Driver):
container = self._containers[obj.container_name] container = self._containers[obj.container_name]
Config = boto3.s3.transfer.TransferConfig( Config = boto3.s3.transfer.TransferConfig(
use_threads=container.config.multipart, use_threads=container.config.multipart,
max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1, max_concurrency=int(self._max_multipart_concurrency) if container.config.multipart else 1,
num_download_attempts=container.config.retries, num_download_attempts=container.config.retries,
multipart_threshold=self._multipart_threshold, multipart_threshold=int(self._multipart_threshold),
multipart_chunksize=self._multipart_chunksize multipart_chunksize=int(self._multipart_chunksize)
) )
obj.download_file(str(p), Callback=callback, Config=Config) obj.download_file(str(p), Callback=callback, Config=Config)