ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix TensorBoard negative iterations were zeroed

Browse Source
This commit is contained in:
allegroai 2023-04-19 15:06:10 +03:00
parent 107bb93ab0
commit 9f9af85a2a
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clearml/binding/frameworks/tensorflow_bind.py
View File

@ -727,12 +727,14 @@ class EventTrainsWriter(object):
# unlike other frameworks, tensorflow already accounts for the iteration number # unlike other frameworks, tensorflow already accounts for the iteration number
# when continuing the training. we substract the smallest iteration such that we # when continuing the training. we substract the smallest iteration such that we
# don't increment the step twice number # don't increment the step twice number
original_step = step
if EventTrainsWriter._current_task: if EventTrainsWriter._current_task:
step -= EventTrainsWriter._current_task.get_initial_iteration() step -= EventTrainsWriter._current_task.get_initial_iteration()
# there can be a few metrics getting reported again, so the step can be negative # there can be a few metrics getting reported again, so the step can be negative
# for the first few reports # for the first few reports
if step < 0: if step < 0 and original_step > 0:
step = 0 step = 0
self._max_step = max(self._max_step, step) self._max_step = max(self._max_step, step)
if value_dicts is None: if value_dicts is None:
LoggerRoot.get_base_logger(TensorflowBinding).debug("Summary arrived without 'value'") LoggerRoot.get_base_logger(TensorflowBinding).debug("Summary arrived without 'value'")