Add sdk.development.artifacts.auto_pickle to support changing the default behavior when uploading artifacts

clearml/automation/controller.py

@@ -1177,7 +1177,7 @@ class PipelineController(object):
         artifact_object,  # type: Any
         metadata=None,  # type: Optional[Mapping]
         delete_after_upload=False,  # type: bool
-        auto_pickle=True,  # type: bool
+        auto_pickle=None,  # type: Optional[bool]
         preview=None,  # type: Any
         wait_on_upload=False,  # type: bool
         serialization_function=None  # type: Optional[Callable[[Any], Union[bytes, bytearray]]]
@@ -1214,9 +1214,10 @@ class PipelineController(object):
             - ``True`` - Delete the local copy of the artifact.
             - ``False`` - Do not delete. (default)
 
-        :param bool auto_pickle: If True (default), and the artifact_object is not one of the following types:
+        :param bool auto_pickle: If True, and the artifact_object is not one of the following types:
             pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string)
             the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)
+            If set to None (default) the sdk.development.artifacts.auto_pickle configuration value will be used.
 
         :param Any preview: The artifact preview
 

clearml/binding/artifacts.py

@@ -28,7 +28,7 @@ from ..storage.helper import remote_driver_schemes
 from ..storage.util import sha256sum, format_size, get_common_path
 from ..utilities.process.mp import SafeEvent, ForkSafeRLock
 from ..utilities.proxy_object import LazyEvalWrapper
-from ..config import deferred_config
+from ..config import deferred_config, config
 
 try:
     import pandas as pd
@@ -365,7 +365,7 @@ class Artifacts(object):
         metadata=None,  # type: Optional[dict]
         preview=None,  # type: Optional[str]
         delete_after_upload=False,  # type: bool
-        auto_pickle=True,  # type: bool
+        auto_pickle=None,  # type: Optional[bool]
         wait_on_upload=False,  # type: bool
         extension_name=None,  # type: Optional[str]
         serialization_function=None,  # type: Optional[Callable[[Any], Union[bytes, bytearray]]]
@@ -381,6 +381,9 @@ class Artifacts(object):
         if name in self._artifacts_container:
             raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name))
 
+        if auto_pickle is None:
+            auto_pickle = bool(config.get("development.artifacts.auto_pickle", False))
+
         # cast preview to string
         if preview is not None and not (isinstance(preview, bool) and preview is False):
             preview = str(preview)

clearml/config/default/sdk.conf

@@ -186,6 +186,17 @@
         # Example: log_os_environments: ["AWS_*", "CUDA_VERSION"]
         log_os_environments: []
 
+        artifacts {
+            # Default value for the auto_pickle behavior when uploading artifacts.
+            # Auto-picle will not pickle artifacts of specific types (pathlib2.Path, dict, pandas.DataFrame,
+            # numpy.ndarray, PIL.Image, url string, local_file string) but if this setting is set to true and the
+            # auto_pickle argument is not explicitly set to False when uploading an artifact, the artifact will be
+            # pickled and uploaded as a pickle file artifact (with the .pkl file extension).
+            # Setting this to false will disable the auto-pickle behavior for any artifact upload which does not
+            # explicitly request this behavior.
+            auto_pickle: true
+        }
+
         # Development mode worker
         worker {
             # Status report period in seconds

clearml/task.py

@@ -2468,7 +2468,7 @@ class Task(_Task):
             artifact_object,  # type: Union[str, Mapping, pandas.DataFrame, numpy.ndarray, Image.Image, Any]
             metadata=None,  # type: Optional[Mapping]
             delete_after_upload=False,  # type: bool
-            auto_pickle=True,  # type: bool
+            auto_pickle=None,  # type: Optional[bool]
             preview=None,  # type: Any
             wait_on_upload=False,  # type: bool
             extension_name=None,  # type: Optional[str]
@@ -2503,9 +2503,10 @@ class Task(_Task):
             - ``True`` - Delete the local copy of the artifact.
             - ``False`` - Do not delete. (default)
 
-        :param bool auto_pickle: If True (default) and the artifact_object is not one of the following types:
+        :param bool auto_pickle: If True and the artifact_object is not one of the following types:
             pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string),
-            the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)
+            the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl).
+            If set to None (default) the sdk.development.artifacts.auto_pickle configuration value will be used.
 
         :param Any preview: The artifact preview