ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix Storage Helper rename partial file throwing errors on multiple access

Browse Source
This commit is contained in:
allegroai 2021-12-17 11:56:24 +02:00
parent a4b507598b
commit 6f6c5e9f5e
1 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
clearml/storage/helper.py
View File

@ -711,7 +711,20 @@ class StorageHelper(object):
pass
# rename temp file to local_file
os.rename(temp_local_path, local_path)
# noinspection PyBroadException
try:
os.rename(temp_local_path, local_path)
except Exception:
# noinspection PyBroadException
try:
os.unlink(temp_local_path)
except Exception:
pass
# file was downloaded by a parallel process, check we have the final output and delete the partial copy
path_local_path = Path(local_path)
if not path_local_path.is_file() or path_local_path.stat().st_size <= 0:
raise Exception('Failed renaming partial file, downloaded file exists and a 0-sized file')
# report download if we are on the second chunk
if verbose or download_reported:
self._log.info(