ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #935 from john-zielke-snkeos/bugfix-path-substitutions

Browse Source 
Fixes for path substitution
This commit is contained in:
alex-burlacu-clear-ml 2023-03-02 16:40:41 +02:00 committed by GitHub
commit fda5095ab2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 37 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
clearml/backend_interface/model.py
View File

@ -90,6 +90,7 @@ class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin):
dest_path=dest_path, dest_path=dest_path,
async_enable=async_enable, async_enable=async_enable,
cb=partial(self._upload_callback, cb=cb), cb=partial(self._upload_callback, cb=cb),
return_canonized=False
) )
if async_enable: if async_enable:
def msg(num_results): def msg(num_results):

45
clearml/storage/helper.py
View File

@ -180,6 +180,10 @@ class StorageHelper(object):
def dest_path(self): def dest_path(self):
return self._dest_path return self._dest_path
@property
def canonized_dest_path(self):
return self._canonized_dest_path
@property @property
def extra(self): def extra(self):
return self._extra return self._extra
@ -192,12 +196,18 @@ class StorageHelper(object):
def retries(self): def retries(self):
return self._retries return self._retries
def __init__(self, src_path, dest_path, extra, callback, retries): @property
def return_canonized(self):
return self._return_canonized
def __init__(self, src_path, dest_path, canonized_dest_path, extra, callback, retries, return_canonized):
self._src_path = src_path self._src_path = src_path
self._dest_path = dest_path self._dest_path = dest_path
self._canonized_dest_path = canonized_dest_path
self._extra = extra self._extra = extra
self._callback = callback self._callback = callback
self._retries = retries self._retries = retries
self._return_canonized = return_canonized
def __str__(self): def __str__(self):
return "src=%s" % self.src_path return "src=%s" % self.src_path
@ -770,12 +780,26 @@ class StorageHelper(object):
if async_enable: if async_enable:
data = self._UploadData( data = self._UploadData(
src_path=src_path, dest_path=canonized_dest_path, extra=extra, callback=cb, retries=retries src_path=src_path,
dest_path=dest_path,
canonized_dest_path=canonized_dest_path,
extra=extra,
callback=cb,
retries=retries,
return_canonized=return_canonized
) )
StorageHelper._initialize_upload_pool() StorageHelper._initialize_upload_pool()
return StorageHelper._upload_pool.apply_async(self._do_async_upload, args=(data,)) return StorageHelper._upload_pool.apply_async(self._do_async_upload, args=(data,))
else: else:
res = self._do_upload(src_path, canonized_dest_path, extra, cb, verbose=False, retries=retries) res = self._do_upload(
src_path=src_path,
dest_path=dest_path,
canonized_dest_path=canonized_dest_path,
extra=extra,
cb=cb,
verbose=False,
retries=retries,
return_canonized=return_canonized)
if res: if res:
result_path = quote_url(result_path) result_path = quote_url(result_path)
return result_path return result_path
@ -1006,6 +1030,7 @@ class StorageHelper(object):
self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e))) self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e)))
def delete(self, path): def delete(self, path):
path = self._canonize_url(path)
return self._driver.delete_object(self.get_object(path)) return self._driver.delete_object(self.get_object(path))
def check_write_permissions(self, dest_path=None): def check_write_permissions(self, dest_path=None):
@ -1152,8 +1177,8 @@ class StorageHelper(object):
def _do_async_upload(self, data): def _do_async_upload(self, data):
assert isinstance(data, self._UploadData) assert isinstance(data, self._UploadData)
return self._do_upload(data.src_path, data.dest_path, extra=data.extra, cb=data.callback, return self._do_upload(data.src_path, data.dest_path, data.canonized_dest_path, extra=data.extra, cb=data.callback,
verbose=True, retries=data.retries) verbose=True, retries=data.retries, return_canonized=data.return_canonized)
def _upload_from_file(self, local_path, dest_path, extra=None): def _upload_from_file(self, local_path, dest_path, extra=None):
if not hasattr(self._driver, 'upload_object'): if not hasattr(self._driver, 'upload_object'):
@ -1172,8 +1197,8 @@ class StorageHelper(object):
extra=extra) extra=extra)
return res return res
def _do_upload(self, src_path, dest_path, extra=None, cb=None, verbose=False, retries=1): def _do_upload(self, src_path, dest_path, canonized_dest_path, extra=None, cb=None, verbose=False, retries=1, return_canonized=False):
object_name = self._normalize_object_name(dest_path) object_name = self._normalize_object_name(canonized_dest_path)
if cb: if cb:
try: try:
cb(None) cb(None)
@ -1191,7 +1216,7 @@ class StorageHelper(object):
last_ex = None last_ex = None
for i in range(max(1, int(retries))): for i in range(max(1, int(retries))):
try: try:
if not self._upload_from_file(local_path=src_path, dest_path=dest_path, extra=extra): if not self._upload_from_file(local_path=src_path, dest_path=canonized_dest_path, extra=extra):
# retry if failed # retry if failed
last_ex = ValueError("Upload failed") last_ex = ValueError("Upload failed")
continue continue
@ -1213,11 +1238,11 @@ class StorageHelper(object):
self._log.debug("Finished upload: %s => %s" % (src_path, object_name)) self._log.debug("Finished upload: %s => %s" % (src_path, object_name))
if cb: if cb:
try: try:
cb(dest_path) cb(canonized_dest_path if return_canonized else dest_path)
except Exception as e: except Exception as e:
self._log.warning("Exception on upload callback: %s" % str(e)) self._log.warning("Exception on upload callback: %s" % str(e))
return dest_path return canonized_dest_path if return_canonized else dest_path
def get_object(self, path, silence_errors=False): def get_object(self, path, silence_errors=False):
# type: (str, bool) -> object # type: (str, bool) -> object