ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 21:33:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bugfix for storage with path substitutions (#877)

Browse Source 
* Bugfix: canonize url in StorageHelper.exists_file()

* Added fixes for path substitutions:
StorageManager to use orginal path in metadata
Helper to use path rewrites in list operations

* Fixed a copy-paste error

Co-authored-by: eajechiloae <97950284+eugen-ajechiloae-clearml@users.noreply.github.com>
This commit is contained in:
john-zielke-snkeos 2023-01-23 14:51:21 +01:00 committed by GitHub
parent b4cb44bf1b
commit 7d80406290
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
clearml/storage/helper.py
View File

@ -784,6 +784,7 @@ class StorageHelper(object):
Listed relative to the base path. Listed relative to the base path.
""" """
if prefix: if prefix:
prefix = self._canonize_url(prefix)
if prefix.startswith(self._base_url): if prefix.startswith(self._base_url):
prefix = prefix[len(self._base_url):] prefix = prefix[len(self._base_url):]
if self._base_url != "file://": if self._base_url != "file://":
@ -1240,6 +1241,7 @@ class StorageHelper(object):
pass pass
def exists_file(self, remote_url): def exists_file(self, remote_url):
remote_url = self._canonize_url(remote_url)
object_name = self._normalize_object_name(remote_url) object_name = self._normalize_object_name(remote_url)
return self._driver.exists_file( return self._driver.exists_file(
container_name=self._container.name if self._container else "", object_name=object_name container_name=self._container.name if self._container else "", object_name=object_name

5
clearml/storage/manager.py
View File

@ -481,8 +481,9 @@ class StorageManager(object):
if not obj: if not obj:
return None return None
metadata = helper.get_object_metadata(obj) metadata = helper.get_object_metadata(obj)
if return_full_path and not metadata["name"].startswith(helper.base_url): base_url = helper._resolve_base_url(remote_url)
metadata["name"] = helper.base_url + ("/" if not helper.base_url.endswith("/") else "") + metadata["name"] if return_full_path and not metadata["name"].startswith(base_url):
metadata["name"] = base_url + ("/" if not base_url.endswith("/") else "") + metadata["name"]
return metadata return metadata
@classmethod @classmethod