From 82fbc36df607853362e3fe8a3b44f639158e435b Mon Sep 17 00:00:00 2001 From: Ammar Qammaz Date: Fri, 3 Nov 2023 10:02:33 +0200 Subject: [PATCH] ... --- train.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/train.py b/train.py index deead1c..0f2f642 100644 --- a/train.py +++ b/train.py @@ -31,7 +31,7 @@ except ImportError: def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoint_iterations, checkpoint, debug_from): bestLossEncountered = 1000000 bestIterationEncountered = 0 - pickBetweenFinalNLosses = 500 #Closely monitor last 500 solutions + goodLossThreshold = 0.07 first_iter = 0 tb_writer = prepare_output_and_logger(dataset) @@ -107,13 +107,14 @@ def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoi if iteration == opt.iterations: progress_bar.close() - if (bestLossEncountered>ema_loss_for_log) and (pickBetweenFinalNLosses+iteration > opt.iterations): + + if (bestLossEncountered>ema_loss_for_log) and (goodLossThreshold > ema_loss_for_log): if (bestIterationEncountered!=0): - print("\n[ITER {}] Erasing this iteration..".format(bestIterationEncountered)) - os.system("rm point_cloud/iteration_%u/*.ply && rmdir point_cloud/iteration_%u/"%iteration) + print("\n[GOOD ITER {}] Erasing previous best iteration..".format(bestIterationEncountered)) + os.system("rm point_cloud/iteration_%u/*.ply && rmdir point_cloud/iteration_%u/"%bestIterationEncountered) bestLossEncountered = ema_loss_for_log bestIterationEncountered = iteration - print("\n[ITER {}] Now remembering this iteration..".format(iteration)) + print("\n[GOOD ITER {}] Now remembering this iteration..".format(iteration)) saving_iterations.append(iteration) # Log and save