ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-07 14:14:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix TensorFlow NaN/Inf values support

Browse Source
This commit is contained in:
allegroai 2020-03-20 10:26:43 +02:00
parent babaf9f1ce
commit b4050ecf25
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
trains/binding/frameworks/tensorflow_bind.py
View File

@ -287,7 +287,16 @@ class EventTrainsWriter(object):
series = possible_tag or series series = possible_tag or series
# update scalar cache # update scalar cache
num, value = self._scalar_report_cache.get((title, series), (0, 0)) num, value = self._scalar_report_cache.get((title, series), (0, 0))
self._scalar_report_cache[(title, series)] = (num + 1, value + scalar_data) # nan outputs is a string, it's probably a NaN
if isinstance(scalar_data, six.string_types):
try:
scalar_data = float(scalar_data)
except:
scalar_data = float('nan')
# nan outputs nan
self._scalar_report_cache[(title, series)] = \
(num + 1,
(value + scalar_data) if scalar_data == scalar_data else scalar_data)
# only report images every specific interval # only report images every specific interval
if step % self.report_freq != 0: if step % self.report_freq != 0: