diff --git a/clearml/automation/controller.py b/clearml/automation/controller.py index 26cb9516..8326b399 100644 --- a/clearml/automation/controller.py +++ b/clearml/automation/controller.py @@ -142,7 +142,7 @@ class PipelineController(object): try: self.job.task.reload() self.job_ended = self.job_started + self.job.task.data.active_duration - except Exception as e: + except Exception: pass def set_job_started(self): @@ -154,7 +154,6 @@ class PipelineController(object): except Exception: pass - def __init__( self, name, # type: str @@ -558,7 +557,7 @@ class PipelineController(object): previous_status # type: str ): pass - + :param output_uri: The storage / output url for this step. This is the default location for output models and other artifacts. Check Task.init reference docs for more info (output_uri is a parameter). @@ -1631,7 +1630,7 @@ class PipelineController(object): 'target_project': self._target_project, } pipeline_dag = self._serialize() - + # serialize pipeline state if self._task and self._auto_connect_task: # check if we are either running locally or that we are running remotely, @@ -2733,7 +2732,7 @@ class PipelineController(object): if node_failed and self._abort_running_steps_on_failure and not node.continue_on_fail: nodes_failed_stop_pipeline.append(node.name) elif node.timeout: - node.set_job_started() + started = node.job.task.data.started if (datetime.now().astimezone(started.tzinfo) - started).total_seconds() > node.timeout: node.job.abort() completed_jobs.append(j) diff --git a/clearml/backend_interface/base.py b/clearml/backend_interface/base.py index afbe67a7..8fe54b4b 100644 --- a/clearml/backend_interface/base.py +++ b/clearml/backend_interface/base.py @@ -178,8 +178,9 @@ class IdObjectBase(InterfaceBase): # noinspection PyBroadException try: self._data = self._reload() - except Exception: - self.log.error("Failed reloading task {}".format(self.id)) + except Exception as ex: + self.log.error("Failed reloading {} {}".format(type(self).__name__.lower(), self.id)) + self.log.debug("Failed reloading {} {}: {}".format(type(self).__name__.lower(), self.id, ex)) @classmethod def normalize_id(cls, id): diff --git a/clearml/binding/hydra_bind.py b/clearml/binding/hydra_bind.py index b89f1dbc..bece5188 100644 --- a/clearml/binding/hydra_bind.py +++ b/clearml/binding/hydra_bind.py @@ -1,7 +1,6 @@ import io import sys from functools import partial -import yaml from ..config import running_remotely, get_remote_task_id, DEV_TASK_NO_REUSE from ..debugging.log import LoggerRoot diff --git a/clearml/model.py b/clearml/model.py index 7f529bda..59dc4ba5 100644 --- a/clearml/model.py +++ b/clearml/model.py @@ -1,9 +1,8 @@ import abc import os -import tarfile import zipfile import shutil -from tempfile import mkdtemp, mkstemp +from tempfile import mkstemp import six import math diff --git a/clearml/task.py b/clearml/task.py index ac896798..5d8fbc78 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -1524,6 +1524,7 @@ class Task(_Task): specified, then a path to a local configuration file is returned. Configuration object. """ pathlib_Path = None # noqa + cast_Path = Path if not isinstance(configuration, (dict, list, Path, six.string_types)): try: from pathlib import Path as pathlib_Path # noqa @@ -1532,6 +1533,8 @@ class Task(_Task): if not pathlib_Path or not isinstance(configuration, pathlib_Path): raise ValueError("connect_configuration supports `dict`, `str` and 'Path' types, " "{} is not supported".format(type(configuration))) + if pathlib_Path and isinstance(configuration, pathlib_Path): + cast_Path = pathlib_Path multi_config_support = Session.check_min_api_version('2.9') if multi_config_support and not name: @@ -1599,7 +1602,7 @@ class Task(_Task): # it is a path to a local file if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()): # check if not absolute path - configuration_path = Path(configuration) + configuration_path = cast_Path(configuration) if not configuration_path.is_file(): ValueError("Configuration file does not exist") try: @@ -1626,7 +1629,7 @@ class Task(_Task): "Using default configuration: {}".format(name, str(configuration))) # update back configuration section if multi_config_support: - configuration_path = Path(configuration) + configuration_path = cast_Path(configuration) if configuration_path.is_file(): with open(configuration_path.as_posix(), 'rt') as f: configuration_text = f.read() @@ -1638,15 +1641,13 @@ class Task(_Task): config_text=configuration_text) return configuration - configuration_path = Path(configuration) + configuration_path = cast_Path(configuration) fd, local_filename = mkstemp(prefix='clearml_task_config_', suffix=configuration_path.suffixes[-1] if configuration_path.suffixes else '.txt') with open(fd, "w") as f: f.write(configuration_text) - if pathlib_Path: - return pathlib_Path(local_filename) - return Path(local_filename) if isinstance(configuration, Path) else local_filename + return cast_Path(local_filename) if isinstance(configuration, cast_Path) else local_filename def connect_label_enumeration(self, enumeration): # type: (Dict[str, int]) -> Dict[str, int] diff --git a/examples/frameworks/pytorch/requirements.txt b/examples/frameworks/pytorch/requirements.txt index e74fea99..7d2cdfe9 100644 --- a/examples/frameworks/pytorch/requirements.txt +++ b/examples/frameworks/pytorch/requirements.txt @@ -2,9 +2,11 @@ clearml jsonschema==3.2.0 ; python_version <= '3.5' matplotlib pytorch-ignite -tensorboard>=1.14.0 +tensorboard<=2.11.2 ; python_version <= '3.7' +tensorboard>2.11.2 ; python_version > '3.7' tensorboardX torch>=1.1.0 torchvision>=0.3.0 tqdm -protobuf>=4.21.1 +protobuf==3.20.* ; python_version <= '3.7' +protobuf>=4.21.1 ; python_version > '3.7' diff --git a/examples/reporting/config_files.py b/examples/reporting/config_files.py index 1a18c610..d3b43463 100644 --- a/examples/reporting/config_files.py +++ b/examples/reporting/config_files.py @@ -21,14 +21,14 @@ task = Task.init(project_name='FirstTrial', task_name='config_files_example') config_file = task.connect_configuration(Path("data_samples") / "sample.json", name='json config file') -with open(config_file, "rt") as f: +with open(config_file.as_posix(), "rt") as f: config_json = json.load(f) print(config_json) config_file = task.connect_configuration(Path("data_samples") / "config_yaml.yaml", name='yaml config file') -with open(config_file, "rt") as f: +with open(config_file.as_posix(), "rt") as f: config_yaml = yaml.load(f, Loader=yaml.SafeLoader) print(config_yaml)