This commit is contained in:
Ammar Qammaz 2023-11-03 10:02:33 +02:00
parent 7c6d2ca956
commit 82fbc36df6

View File

@ -31,7 +31,7 @@ except ImportError:
def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoint_iterations, checkpoint, debug_from): def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoint_iterations, checkpoint, debug_from):
bestLossEncountered = 1000000 bestLossEncountered = 1000000
bestIterationEncountered = 0 bestIterationEncountered = 0
pickBetweenFinalNLosses = 500 #Closely monitor last 500 solutions goodLossThreshold = 0.07
first_iter = 0 first_iter = 0
tb_writer = prepare_output_and_logger(dataset) tb_writer = prepare_output_and_logger(dataset)
@ -107,13 +107,14 @@ def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoi
if iteration == opt.iterations: if iteration == opt.iterations:
progress_bar.close() progress_bar.close()
if (bestLossEncountered>ema_loss_for_log) and (pickBetweenFinalNLosses+iteration > opt.iterations):
if (bestLossEncountered>ema_loss_for_log) and (goodLossThreshold > ema_loss_for_log):
if (bestIterationEncountered!=0): if (bestIterationEncountered!=0):
print("\n[ITER {}] Erasing this iteration..".format(bestIterationEncountered)) print("\n[GOOD ITER {}] Erasing previous best iteration..".format(bestIterationEncountered))
os.system("rm point_cloud/iteration_%u/*.ply && rmdir point_cloud/iteration_%u/"%iteration) os.system("rm point_cloud/iteration_%u/*.ply && rmdir point_cloud/iteration_%u/"%bestIterationEncountered)
bestLossEncountered = ema_loss_for_log bestLossEncountered = ema_loss_for_log
bestIterationEncountered = iteration bestIterationEncountered = iteration
print("\n[ITER {}] Now remembering this iteration..".format(iteration)) print("\n[GOOD ITER {}] Now remembering this iteration..".format(iteration))
saving_iterations.append(iteration) saving_iterations.append(iteration)
# Log and save # Log and save