ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix uploading files to S3 when _at_exit() is called fails (work around https://github.com/boto/boto3/issues/3113)

Browse Source
This commit is contained in:
allegroai 2022-12-13 15:43:18 +02:00
parent 09a53bce48
commit 62a5ef102a
1 changed files with 34 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
clearml/storage/helper.py
View File

@ -1591,6 +1591,23 @@ class _Boto3Driver(_Driver):
Callback=callback, Callback=callback,
ExtraArgs=extra_args, ExtraArgs=extra_args,
) )
except RuntimeError:
# one might get an error similar to: "RuntimeError: cannot schedule new futures after interpreter shutdown"
# In this case, retry the upload without threads
try:
container.bucket.upload_fileobj(
stream,
object_name,
Config=boto3.s3.transfer.TransferConfig(
use_threads=False,
num_download_attempts=container.config.retries,
),
Callback=callback,
ExtraArgs=extra_args,
)
except Exception as ex:
self.get_logger().error("Failed uploading: %s" % ex)
return False
except Exception as ex: except Exception as ex:
self.get_logger().error('Failed uploading: %s' % ex) self.get_logger().error('Failed uploading: %s' % ex)
return False return False
@ -1610,8 +1627,24 @@ class _Boto3Driver(_Driver):
Callback=callback, Callback=callback,
ExtraArgs=extra_args, ExtraArgs=extra_args,
) )
except RuntimeError:
# one might get an error similar to: "RuntimeError: cannot schedule new futures after interpreter shutdown"
# In this case, retry the upload without threads
try:
container.bucket.upload_file(
file_path,
object_name,
Config=boto3.s3.transfer.TransferConfig(
use_threads=False, num_download_attempts=container.config.retries
),
Callback=callback,
ExtraArgs=extra_args,
)
except Exception as ex:
self.get_logger().error("Failed uploading: %s" % ex)
return False
except Exception as ex: except Exception as ex:
self.get_logger().error('Failed uploading: %s' % ex) self.get_logger().error("Failed uploading: %s" % ex)
return False return False
return True return True