From 62c614225019d8039f70c52eaafc082f29f49a34 Mon Sep 17 00:00:00 2001 From: Wernervanrun Date: Fri, 9 Aug 2024 13:31:51 +0200 Subject: [PATCH] Update previous value for the next iteration only if the value is not NaN --- infer/lib/train/graph_generation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infer/lib/train/graph_generation.py b/infer/lib/train/graph_generation.py index 0708aba..c9155df 100644 --- a/infer/lib/train/graph_generation.py +++ b/infer/lib/train/graph_generation.py @@ -72,7 +72,9 @@ def extract_scalar_data(log_dir): else: scalar_data[tag][event.step].append(value) - previous_value = value # Update previous value for the next iteration + # Update previous value for the next iteration only if the value is not NaN + if not np.isnan(value): + previous_value = value # Calculate the average for each step. Restarting training can cause multiple events for the same step. scalar_data[tag] = {step: sum(values) / len(values) for step, values in scalar_data[tag].items()}