From 46019e7e3ac6abfa059f5df427db03e1499fa895 Mon Sep 17 00:00:00 2001
From: clearml <>
Date: Sun, 13 Oct 2024 22:09:11 +0300
Subject: [PATCH] Add sdk.development.artifacts.auto_pickle to support changing
 the default behavior when uploading artifacts

---
 clearml/automation/controller.py |  5 +++--
 clearml/binding/artifacts.py     |  7 +++++--
 clearml/config/default/sdk.conf  | 11 +++++++++++
 clearml/task.py                  |  7 ++++---
 4 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/clearml/automation/controller.py b/clearml/automation/controller.py
index 0b932796..08612282 100644
--- a/clearml/automation/controller.py
+++ b/clearml/automation/controller.py
@@ -1177,7 +1177,7 @@ class PipelineController(object):
         artifact_object,  # type: Any
         metadata=None,  # type: Optional[Mapping]
         delete_after_upload=False,  # type: bool
-        auto_pickle=True,  # type: bool
+        auto_pickle=None,  # type: Optional[bool]
         preview=None,  # type: Any
         wait_on_upload=False,  # type: bool
         serialization_function=None  # type: Optional[Callable[[Any], Union[bytes, bytearray]]]
@@ -1214,9 +1214,10 @@ class PipelineController(object):
             - ``True`` - Delete the local copy of the artifact.
             - ``False`` - Do not delete. (default)
 
-        :param bool auto_pickle: If True (default), and the artifact_object is not one of the following types:
+        :param bool auto_pickle: If True, and the artifact_object is not one of the following types:
             pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string)
             the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)
+            If set to None (default) the sdk.development.artifacts.auto_pickle configuration value will be used.
 
         :param Any preview: The artifact preview
 
diff --git a/clearml/binding/artifacts.py b/clearml/binding/artifacts.py
index 7d4b382e..093c80d7 100644
--- a/clearml/binding/artifacts.py
+++ b/clearml/binding/artifacts.py
@@ -28,7 +28,7 @@ from ..storage.helper import remote_driver_schemes
 from ..storage.util import sha256sum, format_size, get_common_path
 from ..utilities.process.mp import SafeEvent, ForkSafeRLock
 from ..utilities.proxy_object import LazyEvalWrapper
-from ..config import deferred_config
+from ..config import deferred_config, config
 
 try:
     import pandas as pd
@@ -365,7 +365,7 @@ class Artifacts(object):
         metadata=None,  # type: Optional[dict]
         preview=None,  # type: Optional[str]
         delete_after_upload=False,  # type: bool
-        auto_pickle=True,  # type: bool
+        auto_pickle=None,  # type: Optional[bool]
         wait_on_upload=False,  # type: bool
         extension_name=None,  # type: Optional[str]
         serialization_function=None,  # type: Optional[Callable[[Any], Union[bytes, bytearray]]]
@@ -381,6 +381,9 @@ class Artifacts(object):
         if name in self._artifacts_container:
             raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name))
 
+        if auto_pickle is None:
+            auto_pickle = bool(config.get("development.artifacts.auto_pickle", False))
+
         # cast preview to string
         if preview is not None and not (isinstance(preview, bool) and preview is False):
             preview = str(preview)
diff --git a/clearml/config/default/sdk.conf b/clearml/config/default/sdk.conf
index 39f33849..5f6b8d64 100644
--- a/clearml/config/default/sdk.conf
+++ b/clearml/config/default/sdk.conf
@@ -186,6 +186,17 @@
         # Example: log_os_environments: ["AWS_*", "CUDA_VERSION"]
         log_os_environments: []
 
+        artifacts {
+            # Default value for the auto_pickle behavior when uploading artifacts.
+            # Auto-picle will not pickle artifacts of specific types (pathlib2.Path, dict, pandas.DataFrame,
+            # numpy.ndarray, PIL.Image, url string, local_file string) but if this setting is set to true and the
+            # auto_pickle argument is not explicitly set to False when uploading an artifact, the artifact will be
+            # pickled and uploaded as a pickle file artifact (with the .pkl file extension).
+            # Setting this to false will disable the auto-pickle behavior for any artifact upload which does not
+            # explicitly request this behavior.
+            auto_pickle: true
+        }
+
         # Development mode worker
         worker {
             # Status report period in seconds
diff --git a/clearml/task.py b/clearml/task.py
index d96a67af..fede03d5 100644
--- a/clearml/task.py
+++ b/clearml/task.py
@@ -2468,7 +2468,7 @@ class Task(_Task):
             artifact_object,  # type: Union[str, Mapping, pandas.DataFrame, numpy.ndarray, Image.Image, Any]
             metadata=None,  # type: Optional[Mapping]
             delete_after_upload=False,  # type: bool
-            auto_pickle=True,  # type: bool
+            auto_pickle=None,  # type: Optional[bool]
             preview=None,  # type: Any
             wait_on_upload=False,  # type: bool
             extension_name=None,  # type: Optional[str]
@@ -2503,9 +2503,10 @@ class Task(_Task):
             - ``True`` - Delete the local copy of the artifact.
             - ``False`` - Do not delete. (default)
 
-        :param bool auto_pickle: If True (default) and the artifact_object is not one of the following types:
+        :param bool auto_pickle: If True and the artifact_object is not one of the following types:
             pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string),
-            the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)
+            the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl).
+            If set to None (default) the sdk.development.artifacts.auto_pickle configuration value will be used.
 
         :param Any preview: The artifact preview