ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Adding appropriate error when inputing URL's in add_files function (instead of using add_external_files)

Browse Source
This commit is contained in:
morzahavi 2024-09-25 17:03:12 +03:00 committed by GitHub
parent eaeadb18e3
commit 50ad7b09a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 62 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
clearml/datasets/dataset.py
View File

@ -380,6 +380,16 @@ class Dataset(object):
:param tags: A list of tags which describe the Task to add.
"""
self._task.add_tags(tags)
import re
def is_url(path):
"""
Helper function to check if the provided path is an external URL (e.g., s3://, http://).
"""
url_regex = re.compile(
r'^(?:http|ftp|s3|gs|azure)://' # schemes: http, ftp, s3, gs, azure
)
return url_regex.match(path) is not None
def add_files(
self,
@ -406,6 +416,13 @@ class Dataset(object):
:param max_workers: The number of threads to add the files with. Defaults to the number of logical cores
:return: number of files added
"""
# Check if the path provided is a URL, if so, raise an error and suggest using add_external_files
if is_url(path):
raise ValueError(
"The path provided seems to be an external URL (e.g., s3://, http://). "
"Please use `add_external_files()` to add external files to the dataset."
)
max_workers = max_workers or psutil.cpu_count()
self._dirty = True
self._task.get_logger().report_text(