ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-27 17:51:45 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix various CI and tests issues

Browse Source
This commit is contained in:
allegroai 2023-11-05 21:00:04 +02:00
parent 2f05a830b5
commit d6d8aa9318
7 changed files with 21 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
clearml/automation/controller.py
View File

@ -142,7 +142,7 @@ class PipelineController(object):
try:
self.job.task.reload()
self.job_ended = self.job_started + self.job.task.data.active_duration
except Exception as e:
except Exception:
pass
def set_job_started(self):
@ -154,7 +154,6 @@ class PipelineController(object):
except Exception:
pass
def __init__(
self,
name, # type: str
@ -558,7 +557,7 @@ class PipelineController(object):
previous_status # type: str
):
pass
:param output_uri: The storage / output url for this step. This is the default location for output
models and other artifacts. Check Task.init reference docs for more info (output_uri is a parameter).
@ -1631,7 +1630,7 @@ class PipelineController(object):
'target_project': self._target_project,
}
pipeline_dag = self._serialize()
# serialize pipeline state
if self._task and self._auto_connect_task:
# check if we are either running locally or that we are running remotely,
@ -2733,7 +2732,7 @@ class PipelineController(object):
if node_failed and self._abort_running_steps_on_failure and not node.continue_on_fail:
nodes_failed_stop_pipeline.append(node.name)
elif node.timeout:
node.set_job_started()
started = node.job.task.data.started
if (datetime.now().astimezone(started.tzinfo) - started).total_seconds() > node.timeout:
node.job.abort()
completed_jobs.append(j)

5
clearml/backend_interface/base.py
View File

@ -178,8 +178,9 @@ class IdObjectBase(InterfaceBase):
# noinspection PyBroadException
try:
self._data = self._reload()
except Exception:
self.log.error("Failed reloading task {}".format(self.id))
except Exception as ex:
self.log.error("Failed reloading {} {}".format(type(self).__name__.lower(), self.id))
self.log.debug("Failed reloading {} {}: {}".format(type(self).__name__.lower(), self.id, ex))
@classmethod
def normalize_id(cls, id):

1
clearml/binding/hydra_bind.py
View File

@ -1,7 +1,6 @@
import io
import sys
from functools import partial
import yaml
from ..config import running_remotely, get_remote_task_id, DEV_TASK_NO_REUSE
from ..debugging.log import LoggerRoot

3
clearml/model.py
View File

@ -1,9 +1,8 @@
import abc
import os
import tarfile
import zipfile
import shutil
from tempfile import mkdtemp, mkstemp
from tempfile import mkstemp
import six
import math

13
clearml/task.py
View File

@ -1524,6 +1524,7 @@ class Task(_Task):
specified, then a path to a local configuration file is returned. Configuration object.
"""
pathlib_Path = None # noqa
cast_Path = Path
if not isinstance(configuration, (dict, list, Path, six.string_types)):
try:
from pathlib import Path as pathlib_Path # noqa
@ -1532,6 +1533,8 @@ class Task(_Task):
if not pathlib_Path or not isinstance(configuration, pathlib_Path):
raise ValueError("connect_configuration supports `dict`, `str` and 'Path' types, "
"{} is not supported".format(type(configuration)))
if pathlib_Path and isinstance(configuration, pathlib_Path):
cast_Path = pathlib_Path
multi_config_support = Session.check_min_api_version('2.9')
if multi_config_support and not name:
@ -1599,7 +1602,7 @@ class Task(_Task):
# it is a path to a local file
if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
# check if not absolute path
configuration_path = Path(configuration)
configuration_path = cast_Path(configuration)
if not configuration_path.is_file():
ValueError("Configuration file does not exist")
try:
@ -1626,7 +1629,7 @@ class Task(_Task):
"Using default configuration: {}".format(name, str(configuration)))
# update back configuration section
if multi_config_support:
configuration_path = Path(configuration)
configuration_path = cast_Path(configuration)
if configuration_path.is_file():
with open(configuration_path.as_posix(), 'rt') as f:
configuration_text = f.read()
@ -1638,15 +1641,13 @@ class Task(_Task):
config_text=configuration_text)
return configuration
configuration_path = Path(configuration)
configuration_path = cast_Path(configuration)
fd, local_filename = mkstemp(prefix='clearml_task_config_',
suffix=configuration_path.suffixes[-1] if
configuration_path.suffixes else '.txt')
with open(fd, "w") as f:
f.write(configuration_text)
if pathlib_Path:
return pathlib_Path(local_filename)
return Path(local_filename) if isinstance(configuration, Path) else local_filename
return cast_Path(local_filename) if isinstance(configuration, cast_Path) else local_filename
def connect_label_enumeration(self, enumeration):
# type: (Dict[str, int]) -> Dict[str, int]

6
examples/frameworks/pytorch/requirements.txt
View File

@ -2,9 +2,11 @@ clearml
jsonschema==3.2.0 ; python_version <= '3.5'
matplotlib
pytorch-ignite
tensorboard>=1.14.0
tensorboard<=2.11.2 ; python_version <= '3.7'
tensorboard>2.11.2 ; python_version > '3.7'
tensorboardX
torch>=1.1.0
torchvision>=0.3.0
tqdm
protobuf>=4.21.1
protobuf==3.20.* ; python_version <= '3.7'
protobuf>=4.21.1 ; python_version > '3.7'

4
examples/reporting/config_files.py
View File

@ -21,14 +21,14 @@ task = Task.init(project_name='FirstTrial', task_name='config_files_example')
config_file = task.connect_configuration(Path("data_samples") / "sample.json", name='json config file')
with open(config_file, "rt") as f:
with open(config_file.as_posix(), "rt") as f:
config_json = json.load(f)
print(config_json)
config_file = task.connect_configuration(Path("data_samples") / "config_yaml.yaml", name='yaml config file')
with open(config_file, "rt") as f:
with open(config_file.as_posix(), "rt") as f:
config_yaml = yaml.load(f, Loader=yaml.SafeLoader)
print(config_yaml)