ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-06 21:54:36 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge 50ad7b09a1 into 342e1b35f8

Browse Source
This commit is contained in:
AuricResin 2025-02-22 16:30:06 -08:00 committed by GitHub
commit 3d84b712f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 62 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
clearml/datasets/dataset.py
View File

@ -380,6 +380,16 @@ class Dataset(object):
:param tags: A list of tags which describe the Task to add. :param tags: A list of tags which describe the Task to add.
""" """
self._task.add_tags(tags) self._task.add_tags(tags)
import re
def is_url(path):
"""
Helper function to check if the provided path is an external URL (e.g., s3://, http://).
"""
url_regex = re.compile(
r'^(?:http|ftp|s3|gs|azure)://' # schemes: http, ftp, s3, gs, azure
)
return url_regex.match(path) is not None
def add_files( def add_files(
self, self,
@ -406,6 +416,13 @@ class Dataset(object):
:param max_workers: The number of threads to add the files with. Defaults to the number of logical cores :param max_workers: The number of threads to add the files with. Defaults to the number of logical cores
:return: number of files added :return: number of files added
""" """
# Check if the path provided is a URL, if so, raise an error and suggest using add_external_files
if is_url(path):
raise ValueError(
"The path provided seems to be an external URL (e.g., s3://, http://). "
"Please use `add_external_files()` to add external files to the dataset."
)
max_workers = max_workers or psutil.cpu_count() max_workers = max_workers or psutil.cpu_count()
self._dirty = True self._dirty = True
self._task.get_logger().report_text( self._task.get_logger().report_text(