From 8e9f422ec5ffacbb8155e1bab17b8f99bc595032 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Fri, 11 Nov 2022 13:30:15 +0200 Subject: [PATCH] PEP8 --- clearml/backend_interface/metrics/reporter.py | 6 +++--- clearml/task.py | 3 +++ clearml/utilities/parallel.py | 13 ++++++++++--- examples/cicd/README.md | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/clearml/backend_interface/metrics/reporter.py b/clearml/backend_interface/metrics/reporter.py index d545da67..1af08d1b 100644 --- a/clearml/backend_interface/metrics/reporter.py +++ b/clearml/backend_interface/metrics/reporter.py @@ -50,11 +50,11 @@ class BackgroundReportService(BackgroundMonitor, AsyncManagerMixin): # We need this list because on close, the daemon thread might call _write. # _write will pop everything from queue and add the events to a list, # then attempt to send the list of events to the backend. - # But it’s possible on close for the daemon thread to die in the middle of all that. + # But it's possible on close for the daemon thread to die in the middle of all that. # So we have to preserve the list the daemon thread attempted to send to the backend # such that we can retry this. - # Is is possible that we send the same events twice or that we are missing exactly one event. - # Both of these cases should be very rare and I don’t really see how we can do better. + # It is possible that we send the same events twice or that we are missing exactly one event. + # Both of these cases should be very rare and I don't really see how we can do better. self._processing_events = [] def set_storage_uri(self, uri): diff --git a/clearml/task.py b/clearml/task.py index e5638c9a..53190229 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -611,6 +611,9 @@ class Task(_Task): # set defaults if cls._offline_mode: task.output_uri = None + # create target data folder for logger / artifacts + # noinspection PyProtectedMember + Path(task._get_default_report_storage_uri()).mkdir(parents=True, exist_ok=True) elif output_uri is not None: if output_uri is True: output_uri = task.get_project_object().default_output_destination or True diff --git a/clearml/utilities/parallel.py b/clearml/utilities/parallel.py index 2b7c3db0..0579c623 100644 --- a/clearml/utilities/parallel.py +++ b/clearml/utilities/parallel.py @@ -377,8 +377,12 @@ class ParallelZipper(object): self._zipper_queue = PriorityQueue() self._zipper_results = Queue() - def zip_iter(self, file_paths, arcnames={}): - # type: (List[Union(str, Path)], Optional[dict[Union(str, Path), str]]) -> Generator[ParallelZipper.ZipperObject] + def zip_iter( + self, + file_paths, # type: List[Union[str, Path]] + arcnames=None # type: Optional[dict[Union[str, Path], str]] + ): + # type: (...) -> Generator[ParallelZipper.ZipperObject] """ Generator function that returns zip files as soon as they are available. The zipping is done in parallel @@ -388,6 +392,9 @@ class ParallelZipper(object): :return: Generator of ParallelZipper.ZipperObjects """ + if arcnames is None: + arcnames = dict() + while not self._zipper_queue.empty(): self._zipper_queue.get_nowait() for _ in range(self._max_workers): @@ -427,7 +434,7 @@ class ParallelZipper(object): for task in pooled: task.result() if not self._pool: - pool.close() + pool.shutdown() for result in self._yield_zipper_results(): yield result diff --git a/examples/cicd/README.md b/examples/cicd/README.md index 6ca7b594..56bf1ae3 100644 --- a/examples/cicd/README.md +++ b/examples/cicd/README.md @@ -35,7 +35,7 @@ The job simply starts a Github Actions instance and runs `task_stats_to_comment. ## Job 2: Compare model performance ### When to use -The second job is similar to the first, but now we want to make sure that we never merge a code change that will worsen the model’s performance. So we can again get the ClearML task corresponding to the current PR but this time compare the model metrics to the ones from the previous best ClearML task. We’ll only allow the pipeline to succeed, if the metrics are equal or better. In this way we can guarantee the quality of our main branch. +The second job is similar to the first, but now we want to make sure that we never merge a code change that will worsen the model's performance. So we can again get the ClearML task corresponding to the current PR but this time compare the model metrics to the ones from the previous best ClearML task. We’ll only allow the pipeline to succeed, if the metrics are equal or better. In this way we can guarantee the quality of our main branch. ### Technical details Similarly to Job 1, we have put all logic into the `compare_models.py` file. Please note here: this script imports a function from Job 1, so if you only want to include this job into your own project, make sure to copy the function over as well.