ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-01-31 00:56:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add sdk.development.artifacts.auto_pickle to support changing the default behavior when uploading artifacts

Browse Source
This commit is contained in:
clearml 2024-10-13 22:09:11 +03:00
parent 5e201334ba
commit 46019e7e3a
4 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
clearml/automation/controller.py
View File

@ -1177,7 +1177,7 @@ class PipelineController(object):
artifact_object, # type: Any
metadata=None, # type: Optional[Mapping]
delete_after_upload=False, # type: bool
auto_pickle=True, # type: bool
auto_pickle=None, # type: Optional[bool]
preview=None, # type: Any
wait_on_upload=False, # type: bool
serialization_function=None # type: Optional[Callable[[Any], Union[bytes, bytearray]]]
@ -1214,9 +1214,10 @@ class PipelineController(object):
- ``True`` - Delete the local copy of the artifact.
- ``False`` - Do not delete. (default)
:param bool auto_pickle: If True (default), and the artifact_object is not one of the following types:
:param bool auto_pickle: If True, and the artifact_object is not one of the following types:
pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string)
the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)
If set to None (default) the sdk.development.artifacts.auto_pickle configuration value will be used.
:param Any preview: The artifact preview

7
clearml/binding/artifacts.py
View File

@ -28,7 +28,7 @@ from ..storage.helper import remote_driver_schemes
from ..storage.util import sha256sum, format_size, get_common_path
from ..utilities.process.mp import SafeEvent, ForkSafeRLock
from ..utilities.proxy_object import LazyEvalWrapper
from ..config import deferred_config
from ..config import deferred_config, config
try:
import pandas as pd
@ -365,7 +365,7 @@ class Artifacts(object):
metadata=None, # type: Optional[dict]
preview=None, # type: Optional[str]
delete_after_upload=False, # type: bool
auto_pickle=True, # type: bool
auto_pickle=None, # type: Optional[bool]
wait_on_upload=False, # type: bool
extension_name=None, # type: Optional[str]
serialization_function=None, # type: Optional[Callable[[Any], Union[bytes, bytearray]]]
@ -381,6 +381,9 @@ class Artifacts(object):
if name in self._artifacts_container:
raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name))
if auto_pickle is None:
auto_pickle = bool(config.get("development.artifacts.auto_pickle", False))
# cast preview to string
if preview is not None and not (isinstance(preview, bool) and preview is False):
preview = str(preview)

11
clearml/config/default/sdk.conf
View File

@ -186,6 +186,17 @@
# Example: log_os_environments: ["AWS_*", "CUDA_VERSION"]
log_os_environments: []
artifacts {
# Default value for the auto_pickle behavior when uploading artifacts.
# Auto-picle will not pickle artifacts of specific types (pathlib2.Path, dict, pandas.DataFrame,
# numpy.ndarray, PIL.Image, url string, local_file string) but if this setting is set to true and the
# auto_pickle argument is not explicitly set to False when uploading an artifact, the artifact will be
# pickled and uploaded as a pickle file artifact (with the .pkl file extension).
# Setting this to false will disable the auto-pickle behavior for any artifact upload which does not
# explicitly request this behavior.
auto_pickle: true
}
# Development mode worker
worker {
# Status report period in seconds

7
clearml/task.py
View File

@ -2468,7 +2468,7 @@ class Task(_Task):
artifact_object, # type: Union[str, Mapping, pandas.DataFrame, numpy.ndarray, Image.Image, Any]
metadata=None, # type: Optional[Mapping]
delete_after_upload=False, # type: bool
auto_pickle=True, # type: bool
auto_pickle=None, # type: Optional[bool]
preview=None, # type: Any
wait_on_upload=False, # type: bool
extension_name=None, # type: Optional[str]
@ -2503,9 +2503,10 @@ class Task(_Task):
- ``True`` - Delete the local copy of the artifact.
- ``False`` - Do not delete. (default)
:param bool auto_pickle: If True (default) and the artifact_object is not one of the following types:
:param bool auto_pickle: If True and the artifact_object is not one of the following types:
pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string),
the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)
the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl).
If set to None (default) the sdk.development.artifacts.auto_pickle configuration value will be used.
:param Any preview: The artifact preview