diff --git a/docker/run.sh b/docker/run.sh index 8210625..64735fa 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -21,7 +21,7 @@ fi python3.10 convert.py -s $1-data/ --camera SIMPLE_RADIAL --no_gpu #GPU produces worse results (?) -python3.10 train.py -s $1-data/ -r 1 --model_path=$1-data/output/ --position_lr_init 0.000016 --position_lr_final 0.000001 --scaling_lr 0.001 --iterations 100000 #Test more training budget +python3.10 train.py -s $1-data/ -r 1 --model_path=$1-data/output/ --position_lr_init 0.000016 --scaling_lr 0.001 --iterations 35000 #Test more training budget python3.10 3dgsconverter/3dgsconverter.py -i $1-data/output/point_cloud/iteration_30000/point_cloud.ply -o $1-data/output/point_cloud/iteration_30000/output_cc.ply -f cc --rgb --density_filter --remove_flyers python3.10 3dgsconverter/3dgsconverter.py -i $1-data/output/output/point_cloud/iteration_30000/output_cc.ply -o $1-data/output/output/point_cloud/iteration_30000/point_cloud_clean.ply -f 3dgs diff --git a/train.py b/train.py index 5d819b3..c0f8b22 100644 --- a/train.py +++ b/train.py @@ -29,6 +29,9 @@ except ImportError: TENSORBOARD_FOUND = False def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoint_iterations, checkpoint, debug_from): + bestLossEncountered = 1000000 + pickBetweenFinalNLosses = 10 + first_iter = 0 tb_writer = prepare_output_and_logger(dataset) gaussians = GaussianModel(dataset.sh_degree) @@ -103,6 +106,10 @@ def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoi if iteration == opt.iterations: progress_bar.close() + if (bestLossEncountered>ema_loss_for_log) and (pickBetweenFinalNLosses+iteration > opt.iterations): + print("\n[ITER {}] Also remembering this iteration..".format(iteration)) + saving_iterations.append(iteration) + # Log and save training_report(tb_writer, iteration, Ll1, loss, l1_loss, iter_start.elapsed_time(iter_end), testing_iterations, scene, render, (pipe, background)) if (iteration in saving_iterations):