Merge 50ad7b09a1 into 342e1b35f8

clearml/datasets/dataset.py

@@ -380,6 +380,16 @@ class Dataset(object):
         :param tags: A list of tags which describe the Task to add.
         """
         self._task.add_tags(tags)
+import re
+
+def is_url(path):
+    """
+    Helper function to check if the provided path is an external URL (e.g., s3://, http://).
+    """
+    url_regex = re.compile(
+        r'^(?:http|ftp|s3|gs|azure)://'  # schemes: http, ftp, s3, gs, azure
+    )
+    return url_regex.match(path) is not None
 
 def add_files(
         self,
@@ -406,6 +416,13 @@ class Dataset(object):
     :param max_workers: The number of threads to add the files with. Defaults to the number of logical cores
     :return: number of files added
     """
+    # Check if the path provided is a URL, if so, raise an error and suggest using add_external_files
+    if is_url(path):
+        raise ValueError(
+            "The path provided seems to be an external URL (e.g., s3://, http://). "
+            "Please use `add_external_files()` to add external files to the dataset."
+        )
+
     max_workers = max_workers or psutil.cpu_count()
     self._dirty = True
     self._task.get_logger().report_text(