From e321295e8278fa3ad86625c113c700c198690329 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 01:41:39 +0200 Subject: [PATCH 01/18] Bumped SIBR version --- SIBR_viewers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIBR_viewers b/SIBR_viewers index aa1455e..9e371f9 160000 --- a/SIBR_viewers +++ b/SIBR_viewers @@ -1 +1 @@ -Subproject commit aa1455e23ac90a2aff94fe64b319e29051ede325 +Subproject commit 9e371f9ac5f3a413d5c606d931cf73e055b774e4 From e32275e3f7c006f2f0e6fb4b2d0f7914b9e61038 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 01:55:15 +0200 Subject: [PATCH 02/18] Bumped sibr version --- SIBR_viewers | 2 +- submodules/diff-gaussian-rasterization | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SIBR_viewers b/SIBR_viewers index 9e371f9..29dd2f3 160000 --- a/SIBR_viewers +++ b/SIBR_viewers @@ -1 +1 @@ -Subproject commit 9e371f9ac5f3a413d5c606d931cf73e055b774e4 +Subproject commit 29dd2f3a5dc866664b8e0f04bce34dc81e5e6088 diff --git a/submodules/diff-gaussian-rasterization b/submodules/diff-gaussian-rasterization index c78d81f..573e428 160000 --- a/submodules/diff-gaussian-rasterization +++ b/submodules/diff-gaussian-rasterization @@ -1 +1 @@ -Subproject commit c78d81f56cd5da3cf39f6201984570060128c1aa +Subproject commit 573e428645a98d74fb0d533bdbc6f174f3490e52 From 298894b3170038a4e2ad5ab2a82f4ef65ef008e1 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 01:56:35 +0200 Subject: [PATCH 03/18] back to main for gaussian_splatting --- submodules/diff-gaussian-rasterization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/diff-gaussian-rasterization b/submodules/diff-gaussian-rasterization index 573e428..c78d81f 160000 --- a/submodules/diff-gaussian-rasterization +++ b/submodules/diff-gaussian-rasterization @@ -1 +1 @@ -Subproject commit 573e428645a98d74fb0d533bdbc6f174f3490e52 +Subproject commit c78d81f56cd5da3cf39f6201984570060128c1aa From 285602801be081c0d13b2e925dc75616fa9ac7b8 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 02:15:22 +0200 Subject: [PATCH 04/18] Readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff9166d..c902f8b 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ cmake --build build --target install ``` ### Navigation in SIBR Viewers -The SIBR interface provides several methods of navigating the scene. By default, you will be started with an FPS navigator, which you can control with ```W, A, S, D, Q, E``` for camera translation and ```I, K, J, L, U, O``` for rotation. Alternatively, you may want to use a Trackball-style navigator (select from the floating menu). You can also snap to a camera from the data set with the ```Snap to``` button or find the closest camera with ```Snap to closest```. The floating menues also allow you to change the navigation speed. You can use the ```Scaling Modifier``` to control the size of the displayed Gaussians, or show the initial point cloud. +The SIBR interface provides several methods of navigating the scene. By default, you will be started with an FPS navigator, which you can control with ```W, A, S, D, Q, E``` for camera translation and ```I, K, J, L, U, O``` for rotation. Alternatively, you may want to use a Trackball-style navigator (select from the floating menu). You can also snap to a camera from the data set with the ```Snap to``` button or find the closest camera with ```Snap to closest```. The floating menues also allow you to change the navigation speed. You can use the ```Scaling Modifier``` to control the size of the displayed Gaussians, or show the initial point cloud. The real-time viewer also uses slightly more aggressive, fast culling, which can be toggled in the floating menu. If you ever encounter an issue that can be solved by turning fast culling off, please let us know. ### Running the Network Viewer From af8651989624400220859688c11be7ef4c57fddf Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 02:18:28 +0200 Subject: [PATCH 05/18] Readme update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c902f8b..51af8fe 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ cmake --build build --target install ``` ### Navigation in SIBR Viewers -The SIBR interface provides several methods of navigating the scene. By default, you will be started with an FPS navigator, which you can control with ```W, A, S, D, Q, E``` for camera translation and ```I, K, J, L, U, O``` for rotation. Alternatively, you may want to use a Trackball-style navigator (select from the floating menu). You can also snap to a camera from the data set with the ```Snap to``` button or find the closest camera with ```Snap to closest```. The floating menues also allow you to change the navigation speed. You can use the ```Scaling Modifier``` to control the size of the displayed Gaussians, or show the initial point cloud. The real-time viewer also uses slightly more aggressive, fast culling, which can be toggled in the floating menu. If you ever encounter an issue that can be solved by turning fast culling off, please let us know. +The SIBR interface provides several methods of navigating the scene. By default, you will be started with an FPS navigator, which you can control with ```W, A, S, D, Q, E``` for camera translation and ```I, K, J, L, U, O``` for rotation. Alternatively, you may want to use a Trackball-style navigator (select from the floating menu). You can also snap to a camera from the data set with the ```Snap to``` button or find the closest camera with ```Snap to closest```. The floating menues also allow you to change the navigation speed. You can use the ```Scaling Modifier``` to control the size of the displayed Gaussians, or show the initial point cloud. ### Running the Network Viewer @@ -354,7 +354,7 @@ It should suffice to provide the ```-m``` parameter pointing to a trained model **To unlock the full frame rate, please disable V-Sync on your machine and also in the application (Menu → Display).** In addition to the intial point cloud and the splats, you also have the option to visualize the Gaussians by rendering them as ellipsoids from the floating menu. -SIBR has many other functionalities, please see the [documentation](https://sibr.gitlabpages.inria.fr/) for more details on the viewer, navigation options etc. There is also a Top View (available from the menu) that shows the placement of the input cameras and the original SfM point cloud; please note that Top View slows rendering when enabled. +SIBR has many other functionalities, please see the [documentation](https://sibr.gitlabpages.inria.fr/) for more details on the viewer, navigation options etc. There is also a Top View (available from the menu) that shows the placement of the input cameras and the original SfM point cloud; please note that Top View slows rendering when enabled. The real-time viewer also uses slightly more aggressive, fast culling, which can be toggled in the floating menu. If you ever encounter an issue that can be solved by turning fast culling off, please let us know.
Primary Command Line Arguments for Real-Time Viewer From b81fc19c347ca915276226c0c07906415c8b63d2 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 02:51:34 +0200 Subject: [PATCH 06/18] Typo fix, better explanation in reamde --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 51af8fe..5940b1f 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ After extracting or installing the viewers, you may run the compiled ```SIBR_gau .//bin/SIBR_gaussianViewer_app -m ``` -It should suffice to provide the ```-m``` parameter pointing to a trained model directory. Alternatively, you can specify an override location for training input data using ```-s```. To use a specific resolution other than the auto-chosen one, specify ```--rendering-size ```. +It should suffice to provide the ```-m``` parameter pointing to a trained model directory. Alternatively, you can specify an override location for training input data using ```-s```. To use a specific resolution other than the auto-chosen one, specify ```--rendering-size ```. Combine it with ```--force-aspect-ratio``` if you want the exact resolution and don't mind image distortion. **To unlock the full frame rate, please disable V-Sync on your machine and also in the application (Menu → Display).** @@ -384,7 +384,7 @@ Alternatively, you can use the optional parameters ```--colmap_executable``` and ## FAQ - *Where do I get data sets, e.g., those referenced in ```full_eval.py```?* The MipNeRF360 data set is provided by the authors of the original paper on the project site. Note that two of the data sets cannot be openly shared and require you to consult the authors directly. For Tanks&Temples and Deep Blending, please use the download links provided at the top of the page. -- *I don't have 24 GB of VRAM for training, what do I do?* The VRAM consumption is determined by the number of points that are being optimized, which increases over time. If you only want to train to 7k iterations, you will need significantly less. To do the full training routine and avoid running out of memory, you can increase the ```---densify_grad_threshold```, ```--densification_interval``` or reduce the value of ```--densify_until_iter```. Note however that this will affect the quality of the result. Also try setting ```--test_iterations``` to ```-1``` to avoid memory spikes during testing. If ```--densify_grad_threshold``` is very high, no densification should occur and training should complete if the scene itself loads successfully. +- *I don't have 24 GB of VRAM for training, what do I do?* The VRAM consumption is determined by the number of points that are being optimized, which increases over time. If you only want to train to 7k iterations, you will need significantly less. To do the full training routine and avoid running out of memory, you can increase the ```--densify_grad_threshold```, ```--densification_interval``` or reduce the value of ```--densify_until_iter```. Note however that this will affect the quality of the result. Also try setting ```--test_iterations``` to ```-1``` to avoid memory spikes during testing. If ```--densify_grad_threshold``` is very high, no densification should occur and training should complete if the scene itself loads successfully. - *24 GB of VRAM for reference quality training is still a lot! Can't we do it with less?* Yes, most likely. By our calculations it should be possible with **way** less memory (~8GB). If we can find the time we will try to achieve this. If some PyTorch veteran out there wants to tackle this, we look forward to your pull request! From f1d543c44fafe665af6c976b05901a05988bdbc2 Mon Sep 17 00:00:00 2001 From: Zhensheng Yuan <564361+yzslab@users.noreply.github.com> Date: Mon, 10 Jul 2023 14:20:49 +0800 Subject: [PATCH 07/18] fix FileNotFoundError when reading points3D --- scene/dataset_readers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scene/dataset_readers.py b/scene/dataset_readers.py index babc56b..5a64a9b 100644 --- a/scene/dataset_readers.py +++ b/scene/dataset_readers.py @@ -154,9 +154,9 @@ def readColmapSceneInfo(path, images, eval, llffhold=8): nerf_normalization = getNerfppNorm(train_cam_infos) - ply_path = os.path.join(path, "sparse/0/points3d.ply") - bin_path = os.path.join(path, "sparse/0/points3d.bin") - txt_path = os.path.join(path, "sparse/0/points3d.txt") + ply_path = os.path.join(path, "sparse/0/points3D.ply") + bin_path = os.path.join(path, "sparse/0/points3D.bin") + txt_path = os.path.join(path, "sparse/0/points3D.txt") if not os.path.exists(ply_path): print("Converting point3d.bin to .ply, will happen only the first time you open the scene.") try: From 06afb7e616999437fad2b4a56cf0f8005ff9fd06 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 11:12:54 +0200 Subject: [PATCH 08/18] Advice for extensive data sets --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5940b1f..df15b24 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,8 @@ Alternatively, you can use the optional parameters ```--colmap_executable``` and - *24 GB of VRAM for reference quality training is still a lot! Can't we do it with less?* Yes, most likely. By our calculations it should be possible with **way** less memory (~8GB). If we can find the time we will try to achieve this. If some PyTorch veteran out there wants to tackle this, we look forward to your pull request! +- *How can I use this for a much larger dataset, like a city district?* Given enough memory, this should work out fine, but it will require to adapt the ```--scaling_lr``` and ```--position_lr_init/final```. To avoid manual tuning, a suggestion is to check the loss: if it diverges early on, reset and use a lower scaling / position learning rate. + - *How can I use the differentiable Gaussian rasterizer for my own project?* Easy, it is included in this repo as a submodule ```diff-gaussian-rasterization```. Feel free to check out and install the package. It's not really documented, but using it from the Python side is very straightforward (cf. ```gaussian_renderer/__init__.py```). - *Something is broken, how did this happen?* We tried hard to provide a solid and comprehensible basis to make use of the paper's method. We have refactored the code quite a bit, but we have limited capacity to test all possible usage scenarios. Thus, if part of the website, the code or the performance is lacking, please create an issue. If we find the time, we will do our best to address it. From 6a50784106739ad8424e5ad1c6eb422a58b250e8 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 14:40:51 +0200 Subject: [PATCH 09/18] Better camera model error message --- scene/dataset_readers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/dataset_readers.py b/scene/dataset_readers.py index 5a64a9b..31b4bba 100644 --- a/scene/dataset_readers.py +++ b/scene/dataset_readers.py @@ -92,7 +92,7 @@ def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder): FovY = focal2fov(focal_length_y, height) FovX = focal2fov(focal_length_x, width) else: - assert False, "Colmap camera model not handled!" + assert False, "Colmap camera model not handled: only undistorted datasets (PINHOLE or SIMPLE_PINHOLE cameras) supported!" image_path = os.path.join(images_folder, os.path.basename(extr.name)) image_name = os.path.basename(image_path).split(".")[0] From b9fa8d150821901a11b65f3b18c1192c2158da5b Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 15:50:38 +0200 Subject: [PATCH 10/18] Extra comment --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index df15b24..29ec3ec 100644 --- a/README.md +++ b/README.md @@ -392,4 +392,6 @@ Alternatively, you can use the optional parameters ```--colmap_executable``` and - *How can I use the differentiable Gaussian rasterizer for my own project?* Easy, it is included in this repo as a submodule ```diff-gaussian-rasterization```. Feel free to check out and install the package. It's not really documented, but using it from the Python side is very straightforward (cf. ```gaussian_renderer/__init__.py```). +- *Wait, but `````` isn't optimized and could be much better?* There are several parts we didn't even have time to think about improving (yet). The performance you get with this prototype is probably a rather slow baseline for what is physically possible. + - *Something is broken, how did this happen?* We tried hard to provide a solid and comprehensible basis to make use of the paper's method. We have refactored the code quite a bit, but we have limited capacity to test all possible usage scenarios. Thus, if part of the website, the code or the performance is lacking, please create an issue. If we find the time, we will do our best to address it. From c1b61aad155e43541e234306718a6125272dfcaf Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 17:58:17 +0200 Subject: [PATCH 11/18] Clearer conversion explanation --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++++++---------- convert.py | 42 ++++++++++++++++--------------- 2 files changed, 81 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 29ec3ec..01e698d 100644 --- a/README.md +++ b/README.md @@ -320,16 +320,16 @@ The network viewer allows you to connect to a running training process on the sa
Primary Command Line Arguments for Network Viewer - ### --path / -s + #### --path / -s Argument to override model's path to source dataset. - ### --ip + #### --ip IP to use for connection to a running training script. - ### --port + #### --port Port to use for connection to a running training script. - ### --rendering-size + #### --rendering-size Takes two space separated numbers to define the resolution at which network rendering occurs, ```1200``` width by default. Note that to enforce an aspect that differs from the input images, you need ```--force-aspect-ratio``` too. - ### --load_images + #### --load_images Flag to load source dataset images to be displayed in the top view for each camera.

@@ -359,28 +359,74 @@ SIBR has many other functionalities, please see the [documentation](https://sibr
Primary Command Line Arguments for Real-Time Viewer - ### --model-path / -m + #### --model-path / -m Path to trained model. - ### --iteration + #### --iteration Specifies which of state to load if multiple are available. Defaults to latest available iteration. - ### --path / -s + #### --path / -s Argument to override model's path to source dataset. - ### --rendering-size + #### --rendering-size Takes two space separated numbers to define the resolution at which real-time rendering occurs, ```1200``` width by default. Note that to enforce an aspect that differs from the input images, you need ```--force-aspect-ratio``` too. - ### --load_images + #### --load_images Flag to load source dataset images to be displayed in the top view for each camera. - ### --device + #### --device Index of CUDA device to use for rasterization if multiple are available, ```0``` by default.

## Preprocessing your own Scenes -We provide a converter script ```convert.py```, which uses COLMAP to extract SfM information. Optionally, you can use ImageMagick to resize the undistorted images. This rescaling is similar to MipNeRF360, i.e., it creates images with 1/2, 1/4 and 1/8 the original resolution in corresponding folders. To use them, please first install a recent version of COLMAP (ideally CUDA-powered) and ImageMagick. Put the images you want to use in a directory ```/input```. If you have COLMAP and ImageMagick on your system path, you can simply run +Our rasterization requires a SIMPLE_PINHOLE or PINHOLE camera model for COLMAP data. We provide a converter script ```convert.py```, to extract undistorted images and SfM information. Optionally, you can use ImageMagick to resize the undistorted images. This rescaling is similar to MipNeRF360, i.e., it creates images with 1/2, 1/4 and 1/8 the original resolution in corresponding folders. To use them, please first install a recent version of COLMAP (ideally CUDA-powered) and ImageMagick. Put the images you want to use in a directory ```/input```. +``` + +|---input +| |--- +| |--- +| |---... +``` + If you have COLMAP and ImageMagick on your system path, you can simply run ```shell python convert.py -s [--resize] #If not resizing, ImageMagick is not needed ``` -Alternatively, you can use the optional parameters ```--colmap_executable``` and ```--magick_executable``` to point to the respective paths. Please note that on Windows, the executable should point to the COLMAP ```.bat``` file that takes care of setting the execution environment. Once done, `````` will contain the expected COLMAP data set structure with undistorted, differently sized input images, in addition to your original images and temporary data in the directory ```distorted```. +Alternatively, you can use the optional parameters ```--colmap_executable``` and ```--magick_executable``` to point to the respective paths. Please note that on Windows, the executable should point to the COLMAP ```.bat``` file that takes care of setting the execution environment. Once done, `````` will contain the expected COLMAP data set structure with undistorted, resized input images, in addition to your original images and some temporary (distorted) data in the directory ```distorted```. + +If you have your own COLMAP dataset without undistortion (e.g., using ```OPENCV``` camera), you can try to just run the last part of the script: Put the images in ```input``` and the COLMAP info in a subdirectory ```distorted```: +``` + +|---input +| |--- +| |--- +| |---... +|---distorted +| |---sparse +| | |---0 +| |---database.db +``` +Then run +```shell +python convert.py -s --skip_matching [--resize] #If not resizing, ImageMagick is not needed +``` + +
+Command Line Arguments for convert.py + + #### --no_gpu + Flag to avoid using GPU in COLMAP. + #### --skip_matching + Flag to indicate that COLMAP info is available for images. + #### --source_path / -s + Location of the inputs. + #### --camera + Which camera model to use for the early matching steps, ```OPENCV``` by default. + #### --resize + Flag for creating resized versions of input images. + #### --colmap_executable + Path to the COLMAP executable (```.bat``` on Windows). + #### --magick_executable + Path to the ImageMagick executable. +
+
+ ## FAQ - *Where do I get data sets, e.g., those referenced in ```full_eval.py```?* The MipNeRF360 data set is provided by the authors of the original paper on the project site. Note that two of the data sets cannot be openly shared and require you to consult the authors directly. For Tanks&Temples and Deep Blending, please use the download links provided at the top of the page. diff --git a/convert.py b/convert.py index 0f386a4..333f875 100644 --- a/convert.py +++ b/convert.py @@ -16,6 +16,7 @@ import shutil # This Python script is based on the shell converter script provided in the MipNerF 360 repository. parser = ArgumentParser("Colmap converter") parser.add_argument("--no_gpu", action='store_true') +parser.add_argument("--skip_matching", action='store_true') parser.add_argument("--source_path", "-s", required=True, type=str) parser.add_argument("--camera", default="OPENCV", type=str) parser.add_argument("--colmap_executable", default="", type=str) @@ -26,29 +27,30 @@ colmap_command = '"{}"'.format(args.colmap_executable) if len(args.colmap_execut magick_command = '"{}"'.format(args.magick_executable) if len(args.magick_executable) > 0 else "magick" use_gpu = 1 if not args.no_gpu else 0 -os.makedirs(args.source_path + "/distorted/sparse", exist_ok=True) +if not args.skip_matching: + os.makedirs(args.source_path + "/distorted/sparse", exist_ok=True) -## Feature extraction -os.system(colmap_command + " feature_extractor "\ - "--database_path " + args.source_path + "/distorted/database.db \ - --image_path " + args.source_path + "/input \ - --ImageReader.single_camera 1 \ - --ImageReader.camera_model " + args.camera + " \ - --SiftExtraction.use_gpu " + str(use_gpu)) + ## Feature extraction + os.system(colmap_command + " feature_extractor "\ + "--database_path " + args.source_path + "/distorted/database.db \ + --image_path " + args.source_path + "/input \ + --ImageReader.single_camera 1 \ + --ImageReader.camera_model " + args.camera + " \ + --SiftExtraction.use_gpu " + str(use_gpu)) -## Feature matching -os.system(colmap_command + " exhaustive_matcher \ - --database_path " + args.source_path + "/distorted/database.db \ - --SiftMatching.use_gpu " + str(use_gpu)) + ## Feature matching + os.system(colmap_command + " exhaustive_matcher \ + --database_path " + args.source_path + "/distorted/database.db \ + --SiftMatching.use_gpu " + str(use_gpu)) -### Bundle adjustment -# The default Mapper tolerance is unnecessarily large, -# decreasing it speeds up bundle adjustment steps. -os.system(colmap_command + " mapper \ - --database_path " + args.source_path + "/distorted/database.db \ - --image_path " + args.source_path + "/input \ - --output_path " + args.source_path + "/distorted/sparse \ - --Mapper.ba_global_function_tolerance=0.000001") + ### Bundle adjustment + # The default Mapper tolerance is unnecessarily large, + # decreasing it speeds up bundle adjustment steps. + os.system(colmap_command + " mapper \ + --database_path " + args.source_path + "/distorted/database.db \ + --image_path " + args.source_path + "/input \ + --output_path " + args.source_path + "/distorted/sparse \ + --Mapper.ba_global_function_tolerance=0.000001") ### Image undistortion ## We need to undistort our images into ideal pinhole intrinsics. From 6073463220e7d38ecb704e1c07fa10cbabf1f728 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 18:11:03 +0200 Subject: [PATCH 12/18] Fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01e698d..b999243 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ In the current version, this process takes about 7h on our reference machine con python full_eval.py -o --skip_training -m360 -tat -db ``` -If you want to do compute the metrics on our paper's [evaluation images](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/evaluation/images.zip), you can also skip rendering. In this case it is not necessary to provide the source datasets. You can compute metrics for multiple image sets at a time. +If you want to compute the metrics on our paper's [evaluation images](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/evaluation/images.zip), you can also skip rendering. In this case it is not necessary to provide the source datasets. You can compute metrics for multiple image sets at a time. ```shell python full_eval.py -m /garden ... --skip_training --skip_rendering ``` From 0dfa61da0982e0d412b3ec9782406b9fbbe84120 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 18:12:34 +0200 Subject: [PATCH 13/18] Let's be exact here --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b999243..ea6b32c 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ The codebase has 4 main components: - An OpenGL-based real-time viewer to render trained models in real-time. - A script to help you turn your own images into optimization-ready SfM data sets -The components have different requirements w.r.t. both hardware and software. They have been tested on Windows 10 and Linux Ubuntu 22.04. Instructions for setting up and running each of them are found in the sections below. +The components have different requirements w.r.t. both hardware and software. They have been tested on Windows 10 and Ubuntu Linux 22.04. Instructions for setting up and running each of them are found in the sections below. ## Optimizer From bc8b8bfcb16b566e324cad6845e58ab47a473ef9 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 18:14:40 +0200 Subject: [PATCH 14/18] Clarification --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea6b32c..ebbe3d2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Bernhard Kerbl*, Georgios Kopanas*, Thomas Leimkühler, George Drettakis (* indi | [Webpage](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/) | [Full Paper](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/3d_gaussian_splatting_high.pdf) | [Video](https://youtu.be/T_kXY43VZnk) | [Other GRAPHDECO Publications](http://www-sop.inria.fr/reves/publis/gdindex.php) | [FUNGRAPH project page](https://fungraph.inria.fr) | -| [T&T+DB Datasets (650MB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/input/tandt_db.zip) | [Pre-trained Models (14 GB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/pretrained/models.zip) | [Viewers for Windows (60MB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/binaries/viewers.zip) | [Evaluation Images (7 GB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/evaluation/images.zip) |
+| [T&T+DB COLMAP (650MB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/input/tandt_db.zip) | [Pre-trained Models (14 GB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/pretrained/models.zip) | [Viewers for Windows (60MB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/binaries/viewers.zip) | [Evaluation Images (7 GB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/evaluation/images.zip) |
![Teaser image](assets/teaser.png) This repository contains the official authors implementation associated with the paper "3D Gaussian Splatting for Real-Time Radiance Field Rendering", which can be found [here](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/). We further provide the reference images used to create the error metrics reported in the paper, as well as recently created, pre-trained models. From d29178568e8d4eebbf1da40197ce3c34827fe7e9 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 20:10:42 +0200 Subject: [PATCH 15/18] Better readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ebbe3d2..7ec2b7f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ conda env create --file environment.yml --prefix //gaussian_spl conda activate //gaussian_splatting ``` -If you can afford the disk space, we recommend using our environment files for setting up a training environment identical to ours. If you want to make changes, please note that major version changes might affect the results of our method. However, our (limited) experiments suggest that the codebase works just fine inside a more up-to-date environment (Python 3.8, PyTorch 2.0.0, CUDA 11.8). +If you can afford the disk space, we recommend using our environment files for setting up a training environment identical to ours. If you want to make modifications, please note that major version changes might affect the results of our method. However, our (limited) experiments suggest that the codebase works just fine inside a more up-to-date environment (Python 3.8, PyTorch 2.0.0, CUDA 11.8). ### Running @@ -181,7 +181,7 @@ python render.py -m # Generate renderings python metrics.py -m # Compute error metrics on renderings ``` -If you want to evaluate our pre-trained models, you will have to download the corresponding source data sets and indicate their location to ```render.py``` with an additional ```--source_path/-s``` flag. Note: The pre-trained models were created with the release codebase. This code base has been cleaned up and includes bugfixes, hence the metrics you get from evaluating them will differ from those in the paper. +If you want to evaluate our [pre-trained models](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/pretrained/models.zip), you will have to download the corresponding source data sets and indicate their location to ```render.py``` with an additional ```--source_path/-s``` flag. Note: The pre-trained models were created with the release codebase. This code base has been cleaned up and includes bugfixes, hence the metrics you get from evaluating them will differ from those in the paper. ```shell python render.py -m -s python metrics.py -m @@ -400,6 +400,7 @@ If you have your own COLMAP dataset without undistortion (e.g., using ```OPENCV` |---distorted | |---sparse | | |---0 +| | | |---... | |---database.db ``` Then run From 62ec34c9fdeb451be810e5e484827a6ebed9a6db Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 20:18:51 +0200 Subject: [PATCH 16/18] Better tree --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7ec2b7f..88a4f48 100644 --- a/README.md +++ b/README.md @@ -380,9 +380,9 @@ Our rasterization requires a SIMPLE_PINHOLE or PINHOLE camera model for COLMAP d ``` |---input -| |--- -| |--- -| |---... + |--- + |--- + |---... ``` If you have COLMAP and ImageMagick on your system path, you can simply run ```shell @@ -398,10 +398,10 @@ If you have your own COLMAP dataset without undistortion (e.g., using ```OPENCV` | |--- | |---... |---distorted -| |---sparse -| | |---0 -| | | |---... -| |---database.db + |---database.db + |---sparse + |---0 + |---... ``` Then run ```shell From b89771f0ba472ebe88ad5d2b988fd99ab0706ed7 Mon Sep 17 00:00:00 2001 From: bkerbl Date: Mon, 10 Jul 2023 23:04:05 +0200 Subject: [PATCH 17/18] Bump simple knn --- submodules/simple-knn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/simple-knn b/submodules/simple-knn index 76eff30..44f7642 160000 --- a/submodules/simple-knn +++ b/submodules/simple-knn @@ -1 +1 @@ -Subproject commit 76eff30b1533718fe652799729203821b7f54d73 +Subproject commit 44f764299fa305faf6ec5ebd99939e0508331503 From 9490ef76125c3fbfee362039b816702f3241735d Mon Sep 17 00:00:00 2001 From: JonathonLuiten Date: Mon, 10 Jul 2023 18:45:55 -0400 Subject: [PATCH 18/18] fixed tensorboard update images by adding batch dimension (#9) --- train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 6192c8b..f919db2 100644 --- a/train.py +++ b/train.py @@ -162,9 +162,9 @@ def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_i images = torch.cat((images, image.unsqueeze(0)), dim=0) gts = torch.cat((gts, gt_image.unsqueeze(0)), dim=0) if tb_writer and (idx < 5): - tb_writer.add_images(config['name'] + "_view_{}/render".format(viewpoint.image_name), image, global_step=iteration) + tb_writer.add_images(config['name'] + "_view_{}/render".format(viewpoint.image_name), image[None], global_step=iteration) if iteration == testing_iterations[0]: - tb_writer.add_images(config['name'] + "_view_{}/ground_truth".format(viewpoint.image_name), gt_image, global_step=iteration) + tb_writer.add_images(config['name'] + "_view_{}/ground_truth".format(viewpoint.image_name), gt_image[None], global_step=iteration) l1_test = l1_loss(images, gts) psnr_test = psnr(images, gts).mean() @@ -204,4 +204,4 @@ if __name__ == "__main__": training(lp.extract(args), op.extract(args), pp.extract(args), args.test_iterations, args.save_iterations) # All done - print("\nTraining complete.") \ No newline at end of file + print("\nTraining complete.")