ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix renaming/deleting model file right after saving will break async upload (trains/issues#123)

Browse Source
This commit is contained in:
allegroai 2020-04-13 19:03:15 +03:00
parent 12659307a8
commit 4bb17ca420
1 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
trains/binding/frameworks/__init__.py
View File

@ -1,5 +1,9 @@
import os
import shutil
import sys
import threading import threading
import weakref import weakref
from tempfile import mkstemp
import six import six
from pathlib2 import Path from pathlib2 import Path
@ -200,7 +204,17 @@ class WeightsFileHandler(object):
trains_out_model.update_weights_package(weights_filenames=files, auto_delete_file=False, trains_out_model.update_weights_package(weights_filenames=files, auto_delete_file=False,
target_filename=target_filename) target_filename=target_filename)
else: else:
trains_out_model.update_weights(weights_filename=files[0], auto_delete_file=False) # create a copy of the stored file,
# protect against someone deletes/renames the file before async upload finish is done
target_filename = Path(files[0]).name
# HACK: if pytorch-lightning is used, remove the temp '.part' file extension
if sys.modules.get('pytorch_lightning') and target_filename.lower().endswith('.part'):
target_filename = target_filename[:-len('.part')]
fd, temp_file = mkstemp(prefix='.trains.upload_model_', suffix='.tmp')
os.close(fd)
shutil.copy(files[0], temp_file)
trains_out_model.update_weights(weights_filename=temp_file, auto_delete_file=True,
target_filename=target_filename)
else: else:
trains_out_model.update_weights(weights_filename=None, register_uri=saved_path) trains_out_model.update_weights(weights_filename=None, register_uri=saved_path)
except Exception as ex: except Exception as ex: