This commit is contained in:
allegroai 2022-11-11 13:30:15 +02:00
parent 74614bad6d
commit 8e9f422ec5
4 changed files with 17 additions and 7 deletions

View File

@ -50,11 +50,11 @@ class BackgroundReportService(BackgroundMonitor, AsyncManagerMixin):
# We need this list because on close, the daemon thread might call _write.
# _write will pop everything from queue and add the events to a list,
# then attempt to send the list of events to the backend.
# But its possible on close for the daemon thread to die in the middle of all that.
# But it's possible on close for the daemon thread to die in the middle of all that.
# So we have to preserve the list the daemon thread attempted to send to the backend
# such that we can retry this.
# Is is possible that we send the same events twice or that we are missing exactly one event.
# Both of these cases should be very rare and I dont really see how we can do better.
# It is possible that we send the same events twice or that we are missing exactly one event.
# Both of these cases should be very rare and I don't really see how we can do better.
self._processing_events = []
def set_storage_uri(self, uri):

View File

@ -611,6 +611,9 @@ class Task(_Task):
# set defaults
if cls._offline_mode:
task.output_uri = None
# create target data folder for logger / artifacts
# noinspection PyProtectedMember
Path(task._get_default_report_storage_uri()).mkdir(parents=True, exist_ok=True)
elif output_uri is not None:
if output_uri is True:
output_uri = task.get_project_object().default_output_destination or True

View File

@ -377,8 +377,12 @@ class ParallelZipper(object):
self._zipper_queue = PriorityQueue()
self._zipper_results = Queue()
def zip_iter(self, file_paths, arcnames={}):
# type: (List[Union(str, Path)], Optional[dict[Union(str, Path), str]]) -> Generator[ParallelZipper.ZipperObject]
def zip_iter(
self,
file_paths, # type: List[Union[str, Path]]
arcnames=None # type: Optional[dict[Union[str, Path], str]]
):
# type: (...) -> Generator[ParallelZipper.ZipperObject]
"""
Generator function that returns zip files as soon as they are available.
The zipping is done in parallel
@ -388,6 +392,9 @@ class ParallelZipper(object):
:return: Generator of ParallelZipper.ZipperObjects
"""
if arcnames is None:
arcnames = dict()
while not self._zipper_queue.empty():
self._zipper_queue.get_nowait()
for _ in range(self._max_workers):
@ -427,7 +434,7 @@ class ParallelZipper(object):
for task in pooled:
task.result()
if not self._pool:
pool.close()
pool.shutdown()
for result in self._yield_zipper_results():
yield result

View File

@ -35,7 +35,7 @@ The job simply starts a Github Actions instance and runs `task_stats_to_comment.
## Job 2: Compare model performance
### When to use
The second job is similar to the first, but now we want to make sure that we never merge a code change that will worsen the models performance. So we can again get the ClearML task corresponding to the current PR but this time compare the model metrics to the ones from the previous best ClearML task. Well only allow the pipeline to succeed, if the metrics are equal or better. In this way we can guarantee the quality of our main branch.
The second job is similar to the first, but now we want to make sure that we never merge a code change that will worsen the model's performance. So we can again get the ClearML task corresponding to the current PR but this time compare the model metrics to the ones from the previous best ClearML task. Well only allow the pipeline to succeed, if the metrics are equal or better. In this way we can guarantee the quality of our main branch.
### Technical details
Similarly to Job 1, we have put all logic into the `compare_models.py` file. Please note here: this script imports a function from Job 1, so if you only want to include this job into your own project, make sure to copy the function over as well.