From 26942586b1c90a72ed13245089350004dbadc937 Mon Sep 17 00:00:00 2001 From: Bernhard Kerbl Date: Wed, 5 Jul 2023 12:22:16 +0200 Subject: [PATCH 1/2] Added licenses, bumped submodule versions --- SIBR_viewers_windows | 2 +- arguments/__init__.py | 11 +++++++++++ convert.py | 11 +++++++++++ full_eval.py | 11 +++++++++++ gaussian_renderer/__init__.py | 11 +++++++++++ gaussian_renderer/network_gui.py | 11 +++++++++++ metrics.py | 11 +++++++++++ render.py | 11 +++++++++++ scene/__init__.py | 11 +++++++++++ scene/cameras.py | 11 +++++++++++ scene/colmap_loader.py | 11 +++++++++++ scene/dataset_readers.py | 11 +++++++++++ scene/gaussian_model.py | 11 +++++++++++ submodules/diff-gaussian-rasterization | 2 +- submodules/simple-knn | 2 +- train.py | 11 +++++++++++ utils/camera_utils.py | 11 +++++++++++ utils/general_utils.py | 11 +++++++++++ utils/graphics_utils.py | 11 +++++++++++ utils/image_utils.py | 11 +++++++++++ utils/loss_utils.py | 11 +++++++++++ utils/system_utils.py | 11 +++++++++++ 22 files changed, 212 insertions(+), 3 deletions(-) diff --git a/SIBR_viewers_windows b/SIBR_viewers_windows index 71ced00..3c35b80 160000 --- a/SIBR_viewers_windows +++ b/SIBR_viewers_windows @@ -1 +1 @@ -Subproject commit 71ced0023fd0c0aaaa83bb1ab32bce1d4ed630c7 +Subproject commit 3c35b80b65fdfca4ba580a6d7d11e9a8ede29f8a diff --git a/arguments/__init__.py b/arguments/__init__.py index 424d381..5b057cc 100644 --- a/arguments/__init__.py +++ b/arguments/__init__.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + from argparse import ArgumentParser, Namespace import sys import os diff --git a/convert.py b/convert.py index a9d827c..0f386a4 100644 --- a/convert.py +++ b/convert.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import os from argparse import ArgumentParser import shutil diff --git a/full_eval.py b/full_eval.py index 01be106..c83efbd 100644 --- a/full_eval.py +++ b/full_eval.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import os from argparse import ArgumentParser diff --git a/gaussian_renderer/__init__.py b/gaussian_renderer/__init__.py index 492318c..7ca4cf3 100644 --- a/gaussian_renderer/__init__.py +++ b/gaussian_renderer/__init__.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch import math from diff_gaussian_rasterization import GaussianRasterizationSettings, GaussianRasterizer diff --git a/gaussian_renderer/network_gui.py b/gaussian_renderer/network_gui.py index a136f93..df2f9da 100644 --- a/gaussian_renderer/network_gui.py +++ b/gaussian_renderer/network_gui.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch import traceback import socket diff --git a/metrics.py b/metrics.py index f59949a..9eed113 100644 --- a/metrics.py +++ b/metrics.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + from pathlib import Path import os from PIL import Image diff --git a/render.py b/render.py index 29b8a71..fc6b82d 100644 --- a/render.py +++ b/render.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch from scene import Scene import os diff --git a/scene/__init__.py b/scene/__init__.py index 335d497..c1799f2 100644 --- a/scene/__init__.py +++ b/scene/__init__.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import os import random import json diff --git a/scene/cameras.py b/scene/cameras.py index c3dcc32..b57d351 100644 --- a/scene/cameras.py +++ b/scene/cameras.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch from torch import nn import numpy as np diff --git a/scene/colmap_loader.py b/scene/colmap_loader.py index ebdb14e..0f32d23 100644 --- a/scene/colmap_loader.py +++ b/scene/colmap_loader.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import numpy as np import collections import struct diff --git a/scene/dataset_readers.py b/scene/dataset_readers.py index 8939046..babc56b 100644 --- a/scene/dataset_readers.py +++ b/scene/dataset_readers.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import os import sys from PIL import Image diff --git a/scene/gaussian_model.py b/scene/gaussian_model.py index 6b2a249..8cea495 100644 --- a/scene/gaussian_model.py +++ b/scene/gaussian_model.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch import numpy as np from utils.general_utils import inverse_sigmoid, get_expon_lr_func, build_rotation diff --git a/submodules/diff-gaussian-rasterization b/submodules/diff-gaussian-rasterization index 3a07ac2..4aedd82 160000 --- a/submodules/diff-gaussian-rasterization +++ b/submodules/diff-gaussian-rasterization @@ -1 +1 @@ -Subproject commit 3a07ac2e39b9ba7043ffc8bb98397c3ba6e2532d +Subproject commit 4aedd8226f7257935891049f5a378b0e21d0aa37 diff --git a/submodules/simple-knn b/submodules/simple-knn index 76eff30..10b4e40 160000 --- a/submodules/simple-knn +++ b/submodules/simple-knn @@ -1 +1 @@ -Subproject commit 76eff30b1533718fe652799729203821b7f54d73 +Subproject commit 10b4e40c4e73330ceb1adcb3bae4a494e7d3b9c1 diff --git a/train.py b/train.py index 0e0c0c4..6192c8b 100644 --- a/train.py +++ b/train.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import os import torch from random import randint diff --git a/utils/camera_utils.py b/utils/camera_utils.py index c778980..4d1b76f 100644 --- a/utils/camera_utils.py +++ b/utils/camera_utils.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + from scene.cameras import Camera import numpy as np from utils.general_utils import PILtoTorch diff --git a/utils/general_utils.py b/utils/general_utils.py index 0dc50dc..541c082 100644 --- a/utils/general_utils.py +++ b/utils/general_utils.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch import sys from datetime import datetime diff --git a/utils/graphics_utils.py b/utils/graphics_utils.py index 545fc41..b4627d8 100644 --- a/utils/graphics_utils.py +++ b/utils/graphics_utils.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch import math import numpy as np diff --git a/utils/image_utils.py b/utils/image_utils.py index 23e1cb7..cdeaa1b 100644 --- a/utils/image_utils.py +++ b/utils/image_utils.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch def mse(img1, img2): diff --git a/utils/loss_utils.py b/utils/loss_utils.py index 08c98a6..9defc23 100644 --- a/utils/loss_utils.py +++ b/utils/loss_utils.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + import torch import torch.nn.functional as F from torch.autograd import Variable diff --git a/utils/system_utils.py b/utils/system_utils.py index 1335538..90ca6d7 100644 --- a/utils/system_utils.py +++ b/utils/system_utils.py @@ -1,3 +1,14 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + from errno import EEXIST from os import makedirs, path import os From 17d0a6cba2c2c692fa99aa7ad2600e557a1393f5 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Wed, 5 Jul 2023 15:42:08 +0200 Subject: [PATCH 2/2] Added flag description to readme --- README.md | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ded3077..85f543b 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The optimizer uses PyTorch and CUDA extensions in a Python environment to produc - 24 GB VRAM to train the largest scenes in our test suite ### Software Requirements -- C++ Compiler (Visual Studio 2019 for Windows) +- C++ Compiler (we *recommend* Visual Studio 2019 for Windows) - CUDA 11 SDK for PyTorch extensions (we used 11.8) - Conda (recommended for easy setup) @@ -96,8 +96,76 @@ To run the optimizer, simply use python train.py -s ``` +
+ Command Line Arguments for train.py + + ### --source_path / -s + Path to the source directory containing a COLMAP or Synthetic NeRF data set. + ### --model_path / -m + Path where the trained model should be stored (```output/``` by default). + ### --images / -i + Alternative subdirectory for COLMAP images (```images``` by default). + ### --eval + Add this flag to use a MipNeRF360-style training/test split for evaluation. + ### --resolution / -r + Changes the resolution of the loaded images before training. If provided ```1, 2, 4``` or ```8```, uses original, 1/2, 1/4 or 1/8 resolution, respectively. For all other values, rescales the width to the given number while maintaining image aspect. ```1``` by default. + ### --white_background / -w + Add this flag to use white background instead of black (default), e.g., for evaluation of NeRF Synthetic dataset. + ### --sh_degree + Order of spherical harmonics to be used (no larger than 3). ```3``` by default. + ### --convert_SHs_python + Flag to make pipeline compute forward and backward of SHs with PyTorch instead of ours. + ### --convert_cov3D_python + Flag to make pipeline compute forward and backward of the 3D covariance with PyTorch instead of ours. + ### --iterations + Number of total iterations to train for, ```30_000``` by default. + ### --feature_lr + Spherical harmonics features learning rate, ```0.0025``` by default. + ### --opacity_lr + Opacity learning rate, ```0.05``` by default. + ### --scaling_lr + Scaling learning rate, ```0.001``` by default. + ### --rotation_lr + Rotation learning rate, ```0.001``` by default. + ### --position_lr_max_steps + Number of steps (from 0) where position learning rate goes from ```initial``` to ```final```. ```30_000``` by default. + ### --position_lr_init + Initial 3D position learning rate, ```0.00016``` by default. + ### --position_lr_final + Final 3D position learning rate, ```0.0000016``` by default. + ### --position_lr_delay_mult + Position learning rate multiplier (cf. Plenoxels), ```0.01``` by default. + ### --densify_from_iter + Iteration where densification starts, ```500``` by default. + ### --densify_until_iter + Iteration where densification stops, ```15_000``` by default. + ### --densify_grad_threshold + Limit that decides if points should be densified based on 2D position gradient, ```0.0002``` by default. + ### --densification_interal + How frequently to densify, ```100``` (every 100 iterations) by default. + ### --opacity_reset_interval + How frequently to reset opacity, ```3_000``` by default. + ### --lambda_dssim + Influence of SSIM on total loss from 0 to 1, ```0.2``` by default. + ### --percent_dense + Percentage of scene extent (0--1) a point must exceed to be forcibly densified, ```0.1``` by default. + ### --ip + IP to start GUI server on, ```127.0.0.1``` by default. + ### --port + Port to use for GUI server, ```6009``` by default. + ### --test_iterations + Space-separated iterations at which the training script computes L1 and PSNR over test set, ```7000 30000``` by default. + ### --save_iterations + Space-separated iterations at which the training script saves the Gaussian model, ```7000 30000 ``` by default. + ### --quiet + Flag to omit any text written to standard out pipe. + +
+
+ The MipNeRF360 scenes are hosted by the paper authors [here](https://jonbarron.info/mipnerf360/). You can find our SfM data sets for Tanks&Temples and Deep Blending [here](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/input/tandt+db.zip). If you do not provide an output model directory (```-m```), trained models are written to folders with randomized unique names inside the ```output``` directory. At this point, the trained models may be viewed with the real-time viewer (see further below). + ### Evaluation By default, the trained models use all available images in the dataset. To train them while withholding a test set for evaluation, use the ```--eval``` flag. This way, you can render training/test sets and produce error metrics as follows: ```shell @@ -106,6 +174,45 @@ python render.py -m # Generate renderings python metrics.py -m # Compute error metrics on renderings ``` +
+ Command Line Arguments for render.py + + ### --model_path / -m + Path where the trained model should be stored (```output/``` by default). + ### --skip_train + Flag to skip rendering the training set. + ### --skip_test + Flag to skip rendering the test set. + ### --quiet + Flag to omit any text written to standard out pipe. + + **The below parameters will be read automatically from the model path, based on what was used for training. However, you may override them by providing them explicitly on the command line.** + + ### --source_path / -s + Path to the source directory containing a COLMAP or Synthetic NeRF data set. + ### --images / -i + Alternative subdirectory for COLMAP images (```images``` by default). + ### --eval + Add this flag to use a MipNeRF360-style training/test split for evaluation. + ### --resolution / -r + Changes the resolution of the loaded images before training. If provided ```1, 2, 4``` or ```8```, uses original, 1/2, 1/4 or 1/8 resolution, respectively. For all other values, rescales the width to the given number while maintaining image aspect. ```1``` by default. + ### --white_background / -w + Add this flag to use white background instead of black (default), e.g., for evaluation of NeRF Synthetic dataset. + ### --convert_SHs_python + Flag to make pipeline render with computed SHs from PyTorch instead of ours. + ### --convert_cov3D_python + Flag to make pipeline render with computed 3D covariance from PyTorch instead of ours. + +
+ +
+ Command Line Arguments for metrics.py + + ### --model_paths / -m + Space-separated list of model paths for which metrics should be computed. +
+
+ We further provide the ```full_eval.py``` script. This script specifies the routine used in our evaluation and demonstrates the use of some additional parameters, e.g., ```--images (-i)``` to define alternative image directories within COLMAP data sets. If you have downloaded and extracted all the training data, you can run it like this: ```shell python full_eval.py -m360 -tat -db @@ -122,7 +229,7 @@ The Network Viewer can be used to observe the training process and watch the mod - 8 GB VRAM ### Software Requirements -- C++ Compiler (Visual Studio 2019 for Windows) +- C++ Compiler (we *recommend* Visual Studio 2019 for Windows) - CUDA 11 Developer SDK (we used 11.8) - CMake (recent version, we used 3.24) - 7zip (Windows) @@ -161,7 +268,6 @@ You may run the compiled ```SIBR_remoteGaussian_app_``` either by openin The network viewer allows you to connect to a running training process on the same or a different machine. If you are training on the same machine and OS, no command line parameters should be required: the optimizer communicates the location of the training data to the network viewer. By default, optimizer and network viewer will try to establish a connection on **localhost** on port **6009**. You can change this behavior by providing matching ```--ip``` and ```--port``` parameters to both the optimizer and the network viewer. If for some reason the path used by the optimizer to find the training data is not reachable by the network viewer (e.g., due to them running on different (virtual) machines), you may specify an override location to the viewer by using ```--path ```. ### Navigation - The SIBR interface provides several methods of navigating the scene. By default, you will be started with an FPS navigator, which you can control with ```W, A, S, D``` for camera translation and ```Q, E, I, K, J, L``` for rotation. Alternatively, you may want to use a Trackball-style navigator (select from the floating menu). You can also snap to a camera from the data set with the ```Snap to``` button or find the closest camera with ```Snap to closest```. The floating menues also allow you to change the navigation speed. You can use the ```Scaling Modifier``` to control the size of the displayed Gaussians, or show the initial point cloud. ## Real-Time Viewer @@ -175,10 +281,7 @@ The Real-Time Viewer can be used to render trained models with real-time frame r - 8 GB VRAM ### Software Requirements -- C++ Compiler (Visual Studio 2019 for Windows) -- CUDA 11 Developer SDK -- CMake (recent version) -- 7zip (Windows) +The requirements are the same as for the remote viewer. ### Setup @@ -202,7 +305,7 @@ We provide a converter script ```convert.py```, which uses COLMAP to extract SfM ```shell python convert.py -s [--resize] #If not resizing, ImageMagick is not needed ``` -Alternatively, you can use the optional parameters ```--colmap_executable``` and ```--magick_executable``` to point to the respective paths. Please not that on Windows, the executable should point to the COLMAP ```.bat``` file that takes care of setting the execution environment. Once done, `````` will contain the expected COLMAP data set structure with undistorted, differently sized input images, in addition to your original images and temporary data in the directory ```distorted```. +Alternatively, you can use the optional parameters ```--colmap_executable``` and ```--magick_executable``` to point to the respective paths. Please note that on Windows, the executable should point to the COLMAP ```.bat``` file that takes care of setting the execution environment. Once done, `````` will contain the expected COLMAP data set structure with undistorted, differently sized input images, in addition to your original images and temporary data in the directory ```distorted```. ## FAQ - *Where do I get data sets, e.g., those referenced in ```full_eval.py```?* The MipNeRF360 data set is provided by the authors of the original paper on the project site. Note that two of the data sets cannot be openly shared and require you to consult the authors directly. For Tanks&Temples and Deep Blending, please use the download links provided at the top of the page.