diff --git a/clearml/automation/controller.py b/clearml/automation/controller.py index 51c1a25a..998318fc 100644 --- a/clearml/automation/controller.py +++ b/clearml/automation/controller.py @@ -29,7 +29,7 @@ from ..model import BaseModel, OutputModel from ..storage.util import hash_dict from ..task import Task from ..utilities.process.mp import leave_process -from ..utilities.proxy_object import LazyEvalWrapper, flatten_dictionary, walk_nested_dict_tuple_list +from ..utilities.proxy_object import LazyEvalWrapper, flatten_dictionary, walk_nested_dict_tuple_list, verify_basic_type class PipelineController(object): @@ -701,7 +701,7 @@ class PipelineController(object): function_input_artifacts = {} # go over function_kwargs, split it into string and input artifacts for k, v in function_kwargs.items(): - if v and self._step_ref_pattern.match(str(v)): + if v is not None and self._step_ref_pattern.match(str(v)): # check for step artifacts step, _, artifact = v[2:-1].partition('.') if step in self._nodes and artifact in self._nodes[step].return_artifacts: @@ -711,6 +711,9 @@ class PipelineController(object): # steps from tasks the _nodes is till empty, only after deserializing we will have the full DAG) if self._task.running_locally(): self.__verify_step_reference(node=self.Node(name=name), step_ref_string=v) + elif not verify_basic_type(v): + function_input_artifacts[k] = "{}.{}.{}".format(self._task.id, name, k) + self._task.upload_artifact("{}.{}".format(name, k), artifact_object=v, wait_on_upload=True) function_kwargs = {k: v for k, v in function_kwargs.items() if k not in function_input_artifacts} parameters = {"{}/{}".format(CreateFromFunction.kwargs_section, k): v for k, v in function_kwargs.items()} @@ -3622,9 +3625,7 @@ class PipelineDecorator(PipelineController): x for x in cls._evaluated_return_values.get(tid, []) if x in leaves ] for k, v in kwargs.items(): - if v is None or isinstance(v, (bool, int, float, str)): - _node.parameters["{}/{}".format(CreateFromFunction.kwargs_section, k)] = v - elif isinstance(v, (list, tuple)) and all(isinstance(i, (bool, int, float, str)) for i in v): + if v is None or verify_basic_type(v): _node.parameters["{}/{}".format(CreateFromFunction.kwargs_section, k)] = v else: # we need to create an artifact diff --git a/clearml/automation/job.py b/clearml/automation/job.py index 19de52cc..6a2b7e08 100644 --- a/clearml/automation/job.py +++ b/clearml/automation/job.py @@ -18,6 +18,7 @@ from ..backend_interface.util import get_or_create_project, exact_match_regex from ..storage.util import hash_dict from ..task import Task from ..backend_api.services import tasks as tasks_service +from ..utilities.proxy_object import verify_basic_type, get_basic_type logger = getLogger('clearml.automation.job') @@ -573,7 +574,11 @@ class ClearmlJob(BaseJob): self.task.set_tags(list(set(self.task.get_tags()) | set(tags))) if task_params: - self.task.set_parameters(task_params) + param_types = {} + for key, value in task_params.items(): + if verify_basic_type(value): + param_types[key] = get_basic_type(value) + self.task.set_parameters(task_params, __parameters_types=param_types) # store back Task configuration object into backend if task_configurations: diff --git a/clearml/binding/artifacts.py b/clearml/binding/artifacts.py index 0c2c6872..b63d2b4a 100644 --- a/clearml/binding/artifacts.py +++ b/clearml/binding/artifacts.py @@ -635,7 +635,11 @@ class Artifacts(object): artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0] local_filename = artifact_object - elif isinstance(artifact_object, (list, tuple)) and all(isinstance(p, pathlib_types) for p in artifact_object): + elif ( + artifact_object + and isinstance(artifact_object, (list, tuple)) + and all(isinstance(p, pathlib_types) for p in artifact_object) + ): # find common path if exists list_files = [Path(p) for p in artifact_object] override_filename_ext_in_uri = '.zip'