diff --git a/clearml/automation/aws_driver.py b/clearml/automation/aws_driver.py index 8c4f0768..1f0b75de 100644 --- a/clearml/automation/aws_driver.py +++ b/clearml/automation/aws_driver.py @@ -20,6 +20,7 @@ except ImportError as err: "install with: pip install boto3" ) from err + @attr.s class AWSDriver(CloudDriver): """AWS Driver""" @@ -48,7 +49,7 @@ class AWSDriver(CloudDriver): def __attrs_post_init__(self): super().__attrs_post_init__() self.tags = parse_tags(self.tags) - + def spin_up_worker(self, resource_conf, worker_prefix, queue_name, task_id): # user_data script will automatically run when the instance is started. it will install the required packages # for clearml-agent, configure it using environment variables and run clearml-agent on the required queue @@ -59,7 +60,7 @@ class AWSDriver(CloudDriver): launch_specification = ConfigFactory.from_dict( { "ImageId": resource_conf["ami_id"], - "Monitoring": {'Enabled':bool(resource_conf.get('enable_monitoring', False))}, + "Monitoring": {'Enabled': bool(resource_conf.get('enable_monitoring', False))}, "InstanceType": resource_conf["instance_type"], } ) @@ -84,7 +85,7 @@ class AWSDriver(CloudDriver): } } ] - + if resource_conf.get("subnet_id", None): launch_specification["SubnetId"] = resource_conf["subnet_id"] elif resource_conf.get("availability_zone", None): diff --git a/clearml/backend_interface/task/task.py b/clearml/backend_interface/task/task.py index 531c8a4c..5f4323f3 100644 --- a/clearml/backend_interface/task/task.py +++ b/clearml/backend_interface/task/task.py @@ -3,7 +3,6 @@ import itertools import json import logging import os -import re import sys import warnings from copy import copy @@ -34,12 +33,12 @@ from ...backend_interface.task.development.worker import DevWorker from ...backend_interface.session import SendError from ...backend_api import Session from ...backend_api.services import tasks, models, events, projects -from ...backend_api.session.defs import ENV_OFFLINE_MODE +# from ...backend_api.session.defs import ENV_OFFLINE_MODE from ...utilities.pyhocon import ConfigTree, ConfigFactory from ...utilities.config import config_dict_to_text, text_to_config_dict from ...errors import ArtifactUriDeleteError -from ..base import IdObjectBase, InterfaceBase +from ..base import IdObjectBase # , InterfaceBase from ..metrics import Metrics, Reporter from ..model import Model from ..setupuploadmixin import SetupUploadMixin @@ -376,7 +375,6 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): self._edit(type=tasks.TaskTypeEnum(task_type)) return id - def _set_storage_uri(self, value): value = value.rstrip('/') if value else None self._storage_uri = StorageHelper.conform_url(value) @@ -1406,7 +1404,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): Remove input models from the current task. Note that the models themselves are not deleted, but the tasks' reference to the models is removed. To delete the models themselves, see `Models.remove` - + :param models_to_remove: The models to remove from the task. Can be a list of ids, or of `BaseModel` (including its subclasses: `Model` and `InputModel`) """ @@ -2543,7 +2541,7 @@ class Task(IdObjectBase, AccessMixin, SetupUploadMixin): # type: (**Any) -> () for k, v in kwargs.items(): setattr(self.data, k, v) - offline_mode_folder = self.get_offline_mode_folder() + offline_mode_folder = self.get_offline_mode_folder() if not offline_mode_folder: return Path(offline_mode_folder).mkdir(parents=True, exist_ok=True) diff --git a/clearml/binding/frameworks/tensorflow_bind.py b/clearml/binding/frameworks/tensorflow_bind.py index 65b9e06d..eb0366dc 100644 --- a/clearml/binding/frameworks/tensorflow_bind.py +++ b/clearml/binding/frameworks/tensorflow_bind.py @@ -725,7 +725,7 @@ class EventTrainsWriter(object): else: step = int(step) step = tweak_step(step) - + self._max_step = max(self._max_step, step) if value_dicts is None: LoggerRoot.get_base_logger(TensorflowBinding).debug("Summary arrived without 'value'") @@ -2299,6 +2299,6 @@ def tweak_step(step): # unlike other frameworks, tensorflow already accounts for the iteration number # when continuing the training. we substract the smallest iteration such that we # don't increment the step twice number - return step - EventTrainsWriter._current_task.get_initial_iteration() + return step - EventTrainsWriter._current_task.get_initial_iteration() except Exception: return step diff --git a/clearml/binding/gradio_bind.py b/clearml/binding/gradio_bind.py index d074302e..26d1d99c 100644 --- a/clearml/binding/gradio_bind.py +++ b/clearml/binding/gradio_bind.py @@ -70,7 +70,7 @@ class PatchGradio: # noinspection PyBroadException try: return original_fn(*args, **kwargs) - except Exception as e: + except Exception: del kwargs["root_path"] return original_fn(*args, **kwargs) diff --git a/clearml/cli/config/__main__.py b/clearml/cli/config/__main__.py index de006d3d..46a4ccb9 100644 --- a/clearml/cli/config/__main__.py +++ b/clearml/cli/config/__main__.py @@ -242,7 +242,7 @@ def parse_known_host(parsed_host): print('Assuming files and api ports are unchanged and use the same (' + parsed_host.scheme + ') protocol') api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path - files_host = parsed_host.scheme + "://" + parsed_host.netloc+ ':8081' + parsed_host.path + files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path else: print("Warning! Could not parse host name") api_host = '' diff --git a/clearml/datasets/dataset.py b/clearml/datasets/dataset.py index 05ae2d62..569fb743 100644 --- a/clearml/datasets/dataset.py +++ b/clearml/datasets/dataset.py @@ -2981,7 +2981,7 @@ class Dataset(object): task = Task.get_task(task_id=id_) dataset_struct_entry = { - "job_id": id_[len("offline-"):] if id_.startswith("offline-") else id_, # .removeprefix not supported < Python 3.9 + "job_id": id_[len("offline-"):] if id_.startswith("offline-") else id_, # .removeprefix not supported < Python 3.9 "status": task.status } # noinspection PyProtectedMember diff --git a/clearml/storage/helper.py b/clearml/storage/helper.py index f7ef6008..b3a407a7 100644 --- a/clearml/storage/helper.py +++ b/clearml/storage/helper.py @@ -116,6 +116,7 @@ class _Driver(object): cls._file_server_hosts = hosts return cls._file_server_hosts + class _HttpDriver(_Driver): """ LibCloud http/https adapter (simple, enough for now) """ @@ -1797,7 +1798,6 @@ class _FileStorageDriver(_Driver): return os.path.isfile(object_name) - class StorageHelper(object): """ Storage helper. Used by the entire system to download/upload files. @@ -2433,7 +2433,7 @@ class StorageHelper(object): result_dest_path = canonized_dest_path if return_canonized else dest_path - if self.scheme in StorageHelper._quotable_uri_schemes: # TODO: fix-driver-schema + if self.scheme in StorageHelper._quotable_uri_schemes: # TODO: fix-driver-schema # quote link result_dest_path = quote_url(result_dest_path, StorageHelper._quotable_uri_schemes) @@ -2451,7 +2451,7 @@ class StorageHelper(object): result_path = canonized_dest_path if return_canonized else dest_path - if cb and self.scheme in StorageHelper._quotable_uri_schemes: # TODO: fix-driver-schema + if cb and self.scheme in StorageHelper._quotable_uri_schemes: # TODO: fix-driver-schema # store original callback a_cb = cb @@ -2976,7 +2976,6 @@ class StorageHelper(object): ) - def normalize_local_path(local_path): """ Get a normalized local path