ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix Task.upload_artifact() does not upload empty lists/tuples

Browse Source
This commit is contained in:
allegroai 2022-09-29 17:53:38 +03:00
parent a8175ac2e8
commit 17a5ca91d8
3 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
clearml/automation/controller.py
View File

@ -29,7 +29,7 @@ from ..model import BaseModel, OutputModel
from ..storage.util import hash_dict
from ..task import Task
from ..utilities.process.mp import leave_process
from ..utilities.proxy_object import LazyEvalWrapper, flatten_dictionary, walk_nested_dict_tuple_list
from ..utilities.proxy_object import LazyEvalWrapper, flatten_dictionary, walk_nested_dict_tuple_list, verify_basic_type
class PipelineController(object):
@ -701,7 +701,7 @@ class PipelineController(object):
function_input_artifacts = {}
# go over function_kwargs, split it into string and input artifacts
for k, v in function_kwargs.items():
if v and self._step_ref_pattern.match(str(v)):
if v is not None and self._step_ref_pattern.match(str(v)):
# check for step artifacts
step, _, artifact = v[2:-1].partition('.')
if step in self._nodes and artifact in self._nodes[step].return_artifacts:
@ -711,6 +711,9 @@ class PipelineController(object):
# steps from tasks the _nodes is till empty, only after deserializing we will have the full DAG)
if self._task.running_locally():
self.__verify_step_reference(node=self.Node(name=name), step_ref_string=v)
elif not verify_basic_type(v):
function_input_artifacts[k] = "{}.{}.{}".format(self._task.id, name, k)
self._task.upload_artifact("{}.{}".format(name, k), artifact_object=v, wait_on_upload=True)
function_kwargs = {k: v for k, v in function_kwargs.items() if k not in function_input_artifacts}
parameters = {"{}/{}".format(CreateFromFunction.kwargs_section, k): v for k, v in function_kwargs.items()}
@ -3622,9 +3625,7 @@ class PipelineDecorator(PipelineController):
x for x in cls._evaluated_return_values.get(tid, []) if x in leaves
]
for k, v in kwargs.items():
if v is None or isinstance(v, (bool, int, float, str)):
_node.parameters["{}/{}".format(CreateFromFunction.kwargs_section, k)] = v
elif isinstance(v, (list, tuple)) and all(isinstance(i, (bool, int, float, str)) for i in v):
if v is None or verify_basic_type(v):
_node.parameters["{}/{}".format(CreateFromFunction.kwargs_section, k)] = v
else:
# we need to create an artifact

7
clearml/automation/job.py
View File

@ -18,6 +18,7 @@ from ..backend_interface.util import get_or_create_project, exact_match_regex
from ..storage.util import hash_dict
from ..task import Task
from ..backend_api.services import tasks as tasks_service
from ..utilities.proxy_object import verify_basic_type, get_basic_type
logger = getLogger('clearml.automation.job')
@ -573,7 +574,11 @@ class ClearmlJob(BaseJob):
self.task.set_tags(list(set(self.task.get_tags()) | set(tags)))
if task_params:
self.task.set_parameters(task_params)
param_types = {}
for key, value in task_params.items():
if verify_basic_type(value):
param_types[key] = get_basic_type(value)
self.task.set_parameters(task_params, __parameters_types=param_types)
# store back Task configuration object into backend
if task_configurations:

6
clearml/binding/artifacts.py
View File

@ -635,7 +635,11 @@ class Artifacts(object):
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
local_filename = artifact_object
elif isinstance(artifact_object, (list, tuple)) and all(isinstance(p, pathlib_types) for p in artifact_object):
elif (
artifact_object
and isinstance(artifact_object, (list, tuple))
and all(isinstance(p, pathlib_types) for p in artifact_object)
):
# find common path if exists
list_files = [Path(p) for p in artifact_object]
override_filename_ext_in_uri = '.zip'