ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-13 00:50:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix local file uploads without scheme (#1326)
Some checks failed
CodeQL / Analyze (python) (push) Has been cancelled
Details

Browse Source 
* handle scenarios of local file upload without file:// scheme. Fetch the absolute path for the file in such cases and use it as source url

* Update README gifs (#1327)

* skip adding scheme to the path if the scheme is already provided

* handle scenarios of local file upload without file:// scheme. Fetch the absolute path for the file in such cases and use it as source url

* skip adding scheme to the path if the scheme is already provided

---------

Co-authored-by: Vignesh D <vignesh_d@epam.com>
Co-authored-by: pollfly <75068813+pollfly@users.noreply.github.com>
This commit is contained in:
vignesh 2025-04-18 18:41:47 +05:30 committed by GitHub
parent 3616265bc5
commit 7f822e7626
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clearml/datasets/dataset.py
View File

@ -3639,6 +3639,8 @@ class Dataset(object):
# noinspection PyBroadException # noinspection PyBroadException
try: try:
if StorageManager.exists_file(source_url): if StorageManager.exists_file(source_url):
# handle local path provided without scheme
source_url = StorageHelper.sanitize_url(source_url)
remote_objects = [StorageManager.get_metadata(source_url, return_full_path=True)] remote_objects = [StorageManager.get_metadata(source_url, return_full_path=True)]
elif not source_url.startswith(("http://", "https://")): elif not source_url.startswith(("http://", "https://")):
if source_url[-1] != "/": if source_url[-1] != "/":
@ -3655,7 +3657,7 @@ class Dataset(object):
link = remote_object.get("name") link = remote_object.get("name")
relative_path = link[len(source_url) :] relative_path = link[len(source_url) :]
if not relative_path: if not relative_path:
relative_path = source_url.split("/")[-1] relative_path = os.path.basename(source_url)
if not matches_any_wildcard(relative_path, wildcard, recursive=recursive): if not matches_any_wildcard(relative_path, wildcard, recursive=recursive):
continue continue
try: try:

8
clearml/storage/helper.py
View File

@ -3507,6 +3507,14 @@ class StorageHelper(object):
container_name=self._container.name if self._container else "", container_name=self._container.name if self._container else "",
object_name=object_name, object_name=object_name,
) )
@classmethod
def sanitize_url(cls, remote_url):
base_url = cls._resolve_base_url(remote_url)
if base_url != 'file://' or remote_url.startswith("file://"):
return remote_url
absoulte_path = os.path.abspath(remote_url)
return base_url + absoulte_path
def normalize_local_path(local_path: str) -> Path: def normalize_local_path(local_path: str) -> Path: