ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix StorageManager list() is not working for Windows full local path

Browse Source
This commit is contained in:
allegroai 2022-07-28 18:36:52 +03:00
parent 0397f2b41e
commit f9895a8e3b
1 changed files with 30 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
clearml/storage/helper.py
View File

@ -372,10 +372,8 @@ class StorageHelper(object):
# if this is not a known scheme assume local file # if this is not a known scheme assume local file
# url2pathname is specifically intended to operate on (urlparse result).path # url2pathname is specifically intended to operate on (urlparse result).path
# and returns a cross-platform compatible result # and returns a cross-platform compatible result
url = parsed.path new_url = normalize_local_path(url)
if parsed.netloc: self._driver = _FileStorageDriver(new_url)
url = os.path.join(parsed.netloc, url.lstrip(os.path.sep))
self._driver = _FileStorageDriver(Path(url))
# noinspection PyBroadException # noinspection PyBroadException
try: try:
self._container = self._driver.get_container("") self._container = self._driver.get_container("")
@ -392,6 +390,7 @@ class StorageHelper(object):
remaining_timeout = timeout remaining_timeout = timeout
for thread in cls._async_upload_threads: for thread in cls._async_upload_threads:
t = time() t = time()
# noinspection PyBroadException
try: try:
thread.join(timeout=remaining_timeout) thread.join(timeout=remaining_timeout)
except Exception: except Exception:
@ -698,6 +697,14 @@ class StorageHelper(object):
remote_path = self._canonize_url(remote_path) remote_path = self._canonize_url(remote_path)
verbose = self._verbose if verbose is None else verbose verbose = self._verbose if verbose is None else verbose
tmp_remote_path = remote_path
# noinspection PyBroadException
try:
tmp_remote_path = normalize_local_path(tmp_remote_path)
if tmp_remote_path.exists():
remote_path = "file://{}".format(str(tmp_remote_path))
except Exception:
pass
# Check if driver type supports direct access: # Check if driver type supports direct access:
direct_access_path = self.get_driver_direct_access(remote_path) direct_access_path = self.get_driver_direct_access(remote_path)
if direct_access_path and direct_access: if direct_access_path and direct_access:
@ -2631,6 +2638,25 @@ class _FileStorageDriver(_Driver):
return True return True
def normalize_local_path(local_path):
"""
Get a normalized local path
:param local_path: Path of the local file/dir
:type local_path: str
:return: Normalized local path
:rtype: Path
"""
local_path = os.path.normpath(local_path)
local_path = os.path.expanduser(local_path)
local_path = os.path.expandvars(local_path)
local_path = os.path.realpath(local_path)
local_path = os.path.abspath(local_path)
local_path = Path(local_path)
return local_path
def get_file_mimetype(file_path): def get_file_mimetype(file_path):
""" """
Get MIME types of a file Get MIME types of a file