ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-01-31 00:56:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix Windows PermissionError (WinError 5) while uploading datasets (#1349)

Browse Source
This commit is contained in:
Slava Eliseev 2024-12-19 11:08:25 +03:00 committed by GitHub
parent 075f8f0bf9
commit aed1b46612
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
clearml/binding/artifacts.py
View File

@ -257,6 +257,7 @@ class Artifacts(object):
max_preview_size_bytes = 65536
_flush_frequency_sec = 300.
_max_tmp_file_replace_attemps = 3
# notice these two should match
_save_format = '.csv.gz'
_compression = 'gzip'
@ -1138,6 +1139,20 @@ class Artifacts(object):
temp_folder, prefix, suffix = self._temp_files_lookup.pop(local_filename)
fd, temp_filename = mkstemp(prefix=prefix, suffix=suffix)
os.close(fd)
for i in range(self._max_tmp_file_replace_attemps):
try:
os.replace(local_filename, temp_filename)
break
except PermissionError:
LoggerRoot.get_base_logger().warning(
"Failed to replace {} with {}. Attemps left: {}".format(
local_filename, temp_filename, self._max_tmp_file_replace_attemps - i
)
)
else:
# final attempt, and if it fails, throw an exception
# exception could be thrown on some Windows systems
os.replace(local_filename, temp_filename)
local_filename = temp_filename
os.rmdir(temp_folder)