From 7d80406290c0fc4d46035ec47bbc0f0e8fb4950e Mon Sep 17 00:00:00 2001
From: john-zielke-snkeos <john.zielke@snkeos.com>
Date: Mon, 23 Jan 2023 14:51:21 +0100
Subject: [PATCH] Bugfix for storage with path substitutions (#877)

* Bugfix: canonize url in StorageHelper.exists_file()

* Added fixes for path substitutions:
StorageManager to use orginal path in metadata
Helper to use path rewrites in list operations

* Fixed a copy-paste error

Co-authored-by: eajechiloae <97950284+eugen-ajechiloae-clearml@users.noreply.github.com>
---
 clearml/storage/helper.py  | 2 ++
 clearml/storage/manager.py | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/clearml/storage/helper.py b/clearml/storage/helper.py
index e0e26d86..274c0c1b 100644
--- a/clearml/storage/helper.py
+++ b/clearml/storage/helper.py
@@ -784,6 +784,7 @@ class StorageHelper(object):
             Listed relative to the base path.
         """
         if prefix:
+            prefix = self._canonize_url(prefix)
             if prefix.startswith(self._base_url):
                 prefix = prefix[len(self._base_url):]
                 if self._base_url != "file://":
@@ -1240,6 +1241,7 @@ class StorageHelper(object):
                 pass
 
     def exists_file(self, remote_url):
+        remote_url = self._canonize_url(remote_url)
         object_name = self._normalize_object_name(remote_url)
         return self._driver.exists_file(
             container_name=self._container.name if self._container else "", object_name=object_name
diff --git a/clearml/storage/manager.py b/clearml/storage/manager.py
index 200dfb2b..c0bcd9ed 100644
--- a/clearml/storage/manager.py
+++ b/clearml/storage/manager.py
@@ -481,8 +481,9 @@ class StorageManager(object):
         if not obj:
             return None
         metadata = helper.get_object_metadata(obj)
-        if return_full_path and not metadata["name"].startswith(helper.base_url):
-            metadata["name"] = helper.base_url + ("/" if not helper.base_url.endswith("/") else "") + metadata["name"]
+        base_url = helper._resolve_base_url(remote_url)
+        if return_full_path and not metadata["name"].startswith(base_url):
+            metadata["name"] = base_url + ("/" if not base_url.endswith("/") else "") + metadata["name"]
         return metadata
 
     @classmethod