docs: update README

.gitignore

@@ -192,4 +192,3 @@ load/tets/256_tets.npz
 
 # dataset
 dataset/
-load/

README.md

@@ -79,7 +79,26 @@ pip install ninja
 ```sh
 pip install -r requirements.txt
 ```
+
+### Download pre-trained models
+- **Zero123**. We use the newest `stable-zero123.ckpt` by default. You can download it [here](https://huggingface.co/stabilityai/stable-zero123) into `load/zero123/`. In the paper we use `zero123-xl.ckpt` and you can download it by
+```sh
+cd load/zero123
+bash download.sh
+```
+
+- **Omnidata**. We use [Omnidata](https://github.com/EPFL-VILAB/omnidata/tree/main/omnidata_tools/torch) for depth and normal predition in `preprocess_image.py` (copyed from [stable-dreamfusion](https://github.com/ashawkey/stable-dreamfusion)).
+```sh
+cd load/omnidata
+gdown '1Jrh-bRnJEjyMCS7f-WsaFlccfPjJPPHI&confirm=t' # omnidata_dpt_depth_v2.ckpt
+gdown '1wNxVO4vVbDEMEpnAi_jwQObf2MFodcBR&confirm=t' # omnidata_dpt_normal_v2.ckpt
+```
+
 ## Quickstart
+Preprocess the input image to move background and obtain its depth and normal image.
+```sh
+python preprocess_image.py /path/to/image.png --recenter
+```
 Our model is trained in multiple stages. You can run it by
 ```sh
 prompt="a brightly colored mushroom growing on a log"
@@ -148,11 +167,12 @@ python launch.py --config configs/dreamcraft3d-coarse-nerf.yaml --train system.p
 ## Todo
 
 - [x] Release the reorganized code.
+- [ ] Realse the test image data.
 - [ ] Clean the original dreambooth training code.
 - [ ] Provide some running results and checkpoints.
 
 ## Credits
-This code is built on the amazing open-source [threestudio-project](https://github.com/threestudio-project/threestudio).
+This code is built on the amazing open-source projects [threestudio-project](https://github.com/threestudio-project/threestudio) and [stable-dreamfusion](https://github.com/ashawkey/stable-dreamfusion).
 
 ## Related links
 

load/images/groot_caption.txt

@@ -0,0 +1 @@
+cat head anthropomorphic humanoid body, movie poster, marvel little cute Groot character, high detail, hyper realistic, octane rendering.

load/images/jay-basket_caption.txt

@@ -0,0 +1 @@
+a DSLR photo of a blue jay standing on a large basket of rainbow macarons

load/images/mushroom_log_caption.txt

@@ -0,0 +1 @@
+a brightly colored mushroom growing on a log

load/lights/LICENSE.txt

@@ -0,0 +1,2 @@
+The mud_road_puresky.hdr HDR probe is from https://polyhaven.com/a/mud_road_puresky
+CC0 License.

load/make_prompt_library.py

@@ -0,0 +1,434 @@
+import json
+
+dreamfusion_gallery_video_names = [
+    "a_20-sided_die_made_out_of_glass.mp4",
+    "a_bald_eagle_carved_out_of_wood.mp4",
+    "a_banana_peeling_itself.mp4",
+    "a_beagle_in_a_detective's_outfit.mp4",
+    "a_beautiful_dress_made_out_of_fruit,_on_a_mannequin._Studio_lighting,_high_quality,_high_resolution.mp4",
+    "a_beautiful_dress_made_out_of_garbage_bags,_on_a_mannequin._Studio_lighting,_high_quality,_high_resolution.mp4",
+    "a_beautiful_rainbow_fish.mp4",
+    "a_bichon_frise_wearing_academic_regalia.mp4",
+    "a_blue_motorcycle.mp4",
+    "a_blue_poison-dart_frog_sitting_on_a_water_lily.mp4",
+    "a_brightly_colored_mushroom_growing_on_a_log.mp4",
+    "a_bumblebee_sitting_on_a_pink_flower.mp4",
+    "a_bunch_of_colorful_marbles_spilling_out_of_a_red_velvet_bag.mp4",
+    "a_capybara_wearing_a_top_hat,_low_poly.mp4",
+    "a_cat_with_a_mullet.mp4",
+    "a_ceramic_lion.mp4",
+    "a_ceramic_upside_down_yellow_octopus_holding_a_blue_green_ceramic_cup.mp4",
+    "a_chihuahua_wearing_a_tutu.mp4",
+    "a_chimpanzee_holding_a_peeled_banana.mp4",
+    "a_chimpanzee_looking_through_a_telescope.mp4",
+    "a_chimpanzee_stirring_a_bubbling_purple_potion_in_a_cauldron.mp4",
+    "a_chimpanzee_with_a_big_grin.mp4",
+    "a_completely_destroyed_car.mp4",
+    "a_confused_beagle_sitting_at_a_desk_working_on_homework.mp4",
+    "a_corgi_taking_a_selfie.mp4",
+    "a_crab,_low_poly.mp4",
+    "a_crocodile_playing_a_drum_set.mp4",
+    "a_cute_steampunk_elephant.mp4",
+    "a_dachsund_dressed_up_in_a_hotdog_costume.mp4",
+    "a_delicious_hamburger.mp4",
+    "a_dragon-cat_hybrid.mp4",
+    "a_DSLR_photo_of_a_baby_dragon_drinking_boba.mp4",
+    "a_DSLR_photo_of_a_baby_dragon_hatching_out_of_a_stone_egg.mp4",
+    "a_DSLR_photo_of_a_baby_grand_piano_viewed_from_far_away.mp4",
+    "a_DSLR_photo_of_a_bagel_filled_with_cream_cheese_and_lox.mp4",
+    "a_DSLR_photo_of_a_bald_eagle.mp4",
+    "a_DSLR_photo_of_a_barbecue_grill_cooking_sausages_and_burger_patties.mp4",
+    "a_DSLR_photo_of_a_basil_plant.mp4",
+    "a_DSLR_photo_of_a_bear_dancing_ballet.mp4",
+    "a_DSLR_photo_of_a_bear_dressed_as_a_lumberjack.mp4",
+    "a_DSLR_photo_of_a_bear_dressed_in_medieval_armor.mp4",
+    "a_DSLR_photo_of_a_beautiful_violin_sitting_flat_on_a_table.mp4",
+    "a_DSLR_photo_of_a_blue_jay_standing_on_a_large_basket_of_rainbow_macarons.mp4",
+    "a_DSLR_photo_of_a_bulldozer_clearing_away_a_pile_of_snow.mp4",
+    "a_DSLR_photo_of_a_bulldozer.mp4",
+    "a_DSLR_photo_of_a_cake_covered_in_colorful_frosting_with_a_slice_being_taken_out,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_candelabra_with_many_candles_on_a_red_velvet_tablecloth.mp4",
+    "a_DSLR_photo_of_a_car_made_out_of_cheese.mp4",
+    "a_DSLR_photo_of_A_car_made_out_of_sushi.mp4",
+    "a_DSLR_photo_of_a_car_made_out_pizza.mp4",
+    "a_DSLR_photo_of_a_cat_lying_on_its_side_batting_at_a_ball_of_yarn.mp4",
+    "a_DSLR_photo_of_a_cat_magician_making_a_white_dove_appear.mp4",
+    "a_DSLR_photo_of_a_cat_wearing_a_bee_costume.mp4",
+    "a_DSLR_photo_of_a_cat_wearing_a_lion_costume.mp4",
+    "a_DSLR_photo_of_a_cauldron_full_of_gold_coins.mp4",
+    "a_DSLR_photo_of_a_chimpanzee_dressed_like_Henry_VIII_king_of_England.mp4",
+    "a_DSLR_photo_of_a_chimpanzee_dressed_like_Napoleon_Bonaparte.mp4",
+    "a_DSLR_photo_of_a_chow_chow_puppy.mp4",
+    "a_DSLR_photo_of_a_Christmas_tree_with_donuts_as_decorations.mp4",
+    "a_DSLR_photo_of_a_chrome-plated_duck_with_a_golden_beak_arguing_with_an_angry_turtle_in_a_forest.mp4",
+    "a_DSLR_photo_of_a_classic_Packard_car.mp4",
+    "a_DSLR_photo_of_a_cocker_spaniel_wearing_a_crown.mp4",
+    "a_DSLR_photo_of_a_corgi_lying_on_its_back_with_its_tongue_lolling_out.mp4",
+    "a_DSLR_photo_of_a_corgi_puppy.mp4",
+    "a_DSLR_photo_of_a_corgi_sneezing.mp4",
+    "a_DSLR_photo_of_a_corgi_standing_up_drinking_boba.mp4",
+    "a_DSLR_photo_of_a_corgi_taking_a_selfie.mp4",
+    "a_DSLR_photo_of_a_corgi_wearing_a_beret_and_holding_a_baguette,_standing_up_on_two_hind_legs.mp4",
+    "a_DSLR_photo_of_a_covered_wagon.mp4",
+    "a_DSLR_photo_of_a_cracked_egg_with_the_yolk_spilling_out_on_a_wooden_table.mp4",
+    "a_DSLR_photo_of_a_cup_full_of_pens_and_pencils.mp4",
+    "a_DSLR_photo_of_a_dalmation_wearing_a_fireman's_hat.mp4",
+    "a_DSLR_photo_of_a_delicious_chocolate_brownie_dessert_with_ice_cream_on_the_side.mp4",
+    "a_DSLR_photo_of_a_delicious_croissant.mp4",
+    "a_DSLR_photo_of_A_DMC_Delorean_car.mp4",
+    "a_DSLR_photo_of_a_dog_made_out_of_salad.mp4",
+    "a_DSLR_photo_of_a_drum_set_made_of_cheese.mp4",
+    "a_DSLR_photo_of_a_drying_rack_covered_in_clothes.mp4",
+    "a_DSLR_photo_of_aerial_view_of_a_ruined_castle.mp4",
+    "a_DSLR_photo_of_a_football_helmet.mp4",
+    "a_DSLR_photo_of_a_fox_holding_a_videogame_controller.mp4",
+    "a_DSLR_photo_of_a_fox_taking_a_photograph_using_a_DSLR.mp4",
+    "a_DSLR_photo_of_a_frazer_nash_super_sport_car.mp4",
+    "a_DSLR_photo_of_a_frog_wearing_a_sweater.mp4",
+    "a_DSLR_photo_of_a_ghost_eating_a_hamburger.mp4",
+    "a_DSLR_photo_of_a_giant_worm_emerging_from_the_sand_in_the_middle_of_the_desert.mp4",
+    "a_DSLR_photo_of_a_goose_made_out_of_gold.mp4",
+    "a_DSLR_photo_of_a_green_monster_truck.mp4",
+    "a_DSLR_photo_of_a_group_of_dogs_eating_pizza.mp4",
+    "a_DSLR_photo_of_a_group_of_dogs_playing_poker.mp4",
+    "a_DSLR_photo_of_a_gummy_bear_playing_the_saxophone.mp4",
+    "a_DSLR_photo_of_a_hippo_wearing_a_sweater.mp4",
+    "a_DSLR_photo_of_a_humanoid_robot_holding_a_human_brain.mp4",
+    "a_DSLR_photo_of_a_humanoid_robot_playing_solitaire.mp4",
+    "a_DSLR_photo_of_a_humanoid_robot_playing_the_cello.mp4",
+    "a_DSLR_photo_of_a_humanoid_robot_using_a_laptop.mp4",
+    "a_DSLR_photo_of_a_humanoid_robot_using_a_rolling_pin_to_roll_out_dough.mp4",
+    "a_DSLR_photo_of_a_human_skull.mp4",
+    "a_DSLR_photo_of_a_kitten_standing_on_top_of_a_giant_tortoise.mp4",
+    "a_DSLR_photo_of_a_knight_chopping_wood.mp4",
+    "a_DSLR_photo_of_a_knight_holding_a_lance_and_sitting_on_an_armored_horse.mp4",
+    "a_DSLR_photo_of_a_koala_wearing_a_party_hat_and_blowing_out_birthday_candles_on_a_cake.mp4",
+    "a_DSLR_photo_of_a_lemur_taking_notes_in_a_journal.mp4",
+    "a_DSLR_photo_of_a_lion_reading_the_newspaper.mp4",
+    "a_DSLR_photo_of_a_mandarin_duck_swimming_in_a_pond.mp4",
+    "a_DSLR_photo_of_a_model_of_the_eiffel_tower_made_out_of_toothpicks.mp4",
+    "a_DSLR_photo_of_a_mouse_playing_the_tuba.mp4",
+    "a_DSLR_photo_of_a_mug_of_hot_chocolate_with_whipped_cream_and_marshmallows.mp4",
+    "a_DSLR_photo_of_an_adorable_piglet_in_a_field.mp4",
+    "a_DSLR_photo_of_an_airplane_taking_off_from_the_runway.mp4",
+    "a_DSLR_photo_of_an_astronaut_standing_on_the_surface_of_mars.mp4",
+    "a_DSLR_photo_of_an_eggshell_broken_in_two_with_an_adorable_chick_standing_next_to_it.mp4",
+    "a_DSLR_photo_of_an_elephant_skull.mp4",
+    "a_DSLR_photo_of_an_exercise_bike_in_a_well_lit_room.mp4",
+    "a_DSLR_photo_of_an_extravagant_mansion,_aerial_view.mp4",
+    "a_DSLR_photo_of_an_ice_cream_sundae.mp4",
+    "a_DSLR_photo_of_an_iguana_holding_a_balloon.mp4",
+    "a_DSLR_photo_of_an_intricate_and_complex_dish_from_a_michelin_star_restaurant.mp4",
+    "a_DSLR_photo_of_An_iridescent_steampunk_patterned_millipede_with_bison_horns.mp4",
+    "a_DSLR_photo_of_an_octopus_playing_the_piano.mp4",
+    "a_DSLR_photo_of_an_old_car_overgrown_by_vines_and_weeds.mp4",
+    "a_DSLR_photo_of_an_old_vintage_car.mp4",
+    "a_DSLR_photo_of_an_orangutan_making_a_clay_bowl_on_a_throwing_wheel.mp4",
+    "a_DSLR_photo_of_an_orc_forging_a_hammer_on_an_anvil.mp4",
+    "a_DSLR_photo_of_an_origami_motorcycle.mp4",
+    "a_DSLR_photo_of_an_ornate_silver_gravy_boat_sitting_on_a_patterned_tablecloth.mp4",
+    "a_DSLR_photo_of_an_overstuffed_pastrami_sandwich.mp4",
+    "a_DSLR_photo_of_an_unstable_rock_cairn_in_the_middle_of_a_stream.mp4",
+    "a_DSLR_photo_of_a_pair_of_headphones_sitting_on_a_desk.mp4",
+    "a_DSLR_photo_of_a_pair_of_tan_cowboy_boots,_studio_lighting,_product_photography.mp4",
+    "a_DSLR_photo_of_a_peacock_on_a_surfboard.mp4",
+    "a_DSLR_photo_of_a_pigeon_reading_a_book.mp4",
+    "a_DSLR_photo_of_a_piglet_sitting_in_a_teacup.mp4",
+    "a_DSLR_photo_of_a_pig_playing_a_drum_set.mp4",
+    "a_DSLR_photo_of_a_pile_of_dice_on_a_green_tabletop_next_to_some_playing_cards.mp4",
+    "a_DSLR_photo_of_a_pirate_collie_dog,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_plate_of_fried_chicken_and_waffles_with_maple_syrup_on_them.mp4",
+    "a_DSLR_photo_of_a_plate_piled_high_with_chocolate_chip_cookies.mp4",
+    "a_DSLR_photo_of_a_plush_t-rex_dinosaur_toy,_studio_lighting,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_plush_triceratops_toy,_studio_lighting,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_pomeranian_dog.mp4",
+    "a_DSLR_photo_of_a_porcelain_dragon.mp4",
+    "a_DSLR_photo_of_a_praying_mantis_wearing_roller_skates.mp4",
+    "a_DSLR_photo_of_a_puffin_standing_on_a_rock.mp4",
+    "a_DSLR_photo_of_a_pug_made_out_of_metal.mp4",
+    "a_DSLR_photo_of_a_pug_wearing_a_bee_costume.mp4",
+    "a_DSLR_photo_of_a_quill_and_ink_sitting_on_a_desk.mp4",
+    "a_DSLR_photo_of_a_raccoon_stealing_a_pie.mp4",
+    "a_DSLR_photo_of_a_red_cardinal_bird_singing.mp4",
+    "a_DSLR_photo_of_a_red_convertible_car_with_the_top_down.mp4",
+    "a_DSLR_photo_of_a_red-eyed_tree_frog.mp4",
+    "a_DSLR_photo_of_a_red_pickup_truck_driving_across_a_stream.mp4",
+    "a_DSLR_photo_of_a_red_wheelbarrow_with_a_shovel_in_it.mp4",
+    "a_DSLR_photo_of_a_roast_turkey_on_a_platter.mp4",
+    "a_DSLR_photo_of_a_robot_and_dinosaur_playing_chess,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_robot_arm_picking_up_a_colorful_block_from_a_table.mp4",
+    "a_DSLR_photo_of_a_robot_cat_knocking_over_a_chess_piece_on_a_board.mp4",
+    "a_DSLR_photo_of_a_robot_dinosaur.mp4",
+    "a_DSLR_photo_of_a_robot_made_out_of_vegetables.mp4",
+    "a_DSLR_photo_of_a_robot_stegosaurus.mp4",
+    "a_DSLR_photo_of_a_robot_tiger.mp4",
+    "a_DSLR_photo_of_a_rolling_pin_on_top_of_bread_dough.mp4",
+    "a_DSLR_photo_of_a_sheepdog_running.mp4",
+    "a_DSLR_photo_of_a_shiba_inu_playing_golf_wearing_tartan_golf_clothes_and_hat.mp4",
+    "a_DSLR_photo_of_a_shiny_silver_robot_cat.mp4",
+    "a_DSLR_photo_of_a_silverback_gorilla_holding_a_golden_trophy.mp4",
+    "a_DSLR_photo_of_a_silver_humanoid_robot_flipping_a_coin.mp4",
+    "a_DSLR_photo_of_a_small_cherry_tomato_plant_in_a_pot_with_a_few_red_tomatoes_growing_on_it.mp4",
+    "a_DSLR_photo_of_a_small_saguaro_cactus_planted_in_a_clay_pot.mp4",
+    "a_DSLR_photo_of_a_Space_Shuttle.mp4",
+    "a_DSLR_photo_of_a_squirrel_dressed_like_a_clown.mp4",
+    "a_DSLR_photo_of_a_squirrel_flying_a_biplane.mp4",
+    "a_DSLR_photo_of_a_squirrel_giving_a_lecture_writing_on_a_chalkboard.mp4",
+    "a_DSLR_photo_of_a_squirrel_holding_a_bowling_ball.mp4",
+    "a_DSLR_photo_of_a_squirrel-lizard_hybrid.mp4",
+    "a_DSLR_photo_of_a_squirrel_made_out_of_fruit.mp4",
+    "a_DSLR_photo_of_a_squirrel-octopus_hybrid.mp4",
+    "a_DSLR_photo_of_a_stack_of_pancakes_covered_in_maple_syrup.mp4",
+    "a_DSLR_photo_of_a_steam_engine_train,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_steaming_basket_full_of_dumplings.mp4",
+    "a_DSLR_photo_of_a_steaming_hot_plate_piled_high_with_spaghetti_and_meatballs.mp4",
+    "a_DSLR_photo_of_a_steampunk_space_ship_designed_in_the_18th_century.mp4",
+    "a_DSLR_photo_of_a_straw_basket_with_a_cobra_coming_out_of_it.mp4",
+    "a_DSLR_photo_of_a_swan_and_its_cygnets_swimming_in_a_pond.mp4",
+    "a_DSLR_photo_of_a_tarantula,_highly_detailed.mp4",
+    "a_DSLR_photo_of_a_teal_moped.mp4",
+    "a_DSLR_photo_of_a_teapot_shaped_like_an_elephant_head_where_its_snout_acts_as_the_spout.mp4",
+    "a_DSLR_photo_of_a_teddy_bear_taking_a_selfie.mp4",
+    "a_DSLR_photo_of_a_terracotta_bunny.mp4",
+    "a_DSLR_photo_of_a_tiger_dressed_as_a_doctor.mp4",
+    "a_DSLR_photo_of_a_tiger_made_out_of_yarn.mp4",
+    "a_DSLR_photo_of_a_toilet_made_out_of_gold.mp4",
+    "a_DSLR_photo_of_a_toy_robot.mp4",
+    "a_DSLR_photo_of_a_train_engine_made_out_of_clay.mp4",
+    "a_DSLR_photo_of_a_tray_of_Sushi_containing_pugs.mp4",
+    "a_DSLR_photo_of_a_tree_stump_with_an_axe_buried_in_it.mp4",
+    "a_DSLR_photo_of_a_turtle_standing_on_its_hind_legs,_wearing_a_top_hat_and_holding_a_cane.mp4",
+    "a_DSLR_photo_of_a_very_beautiful_small_organic_sculpture_made_of_fine_clockwork_and_gears_with_tiny_ruby_bearings,_very_intricate,_caved,_curved._Studio_lighting,_High_resolution,_white_background.mp4",
+    "a_DSLR_photo_of_A_very_beautiful_tiny_human_heart_organic_sculpture_made_of_copper_wire_and_threaded_pipes,_very_intricate,_curved,_Studio_lighting,_high_resolution.mp4",
+    "a_DSLR_photo_of_a_very_cool_and_trendy_pair_of_sneakers,_studio_lighting.mp4",
+    "a_DSLR_photo_of_a_vintage_record_player.mp4",
+    "a_DSLR_photo_of_a_wine_bottle_and_full_wine_glass_on_a_chessboard.mp4",
+    "a_DSLR_photo_of_a_wooden_desk_and_chair_from_an_elementary_school.mp4",
+    "a_DSLR_photo_of_a_yorkie_dog_eating_a_donut.mp4",
+    "a_DSLR_photo_of_a_yorkie_dog_wearing_extremely_cool_sneakers.mp4",
+    "a_DSLR_photo_of_baby_elephant_jumping_on_a_trampoline.mp4",
+    "a_DSLR_photo_of_cat_wearing_virtual_reality_headset_in_renaissance_oil_painting_high_detail_caravaggio.mp4",
+    "a_DSLR_photo_of_edible_typewriter_made_out_of_vegetables.mp4",
+    "a_DSLR_photo_of_Mont_Saint-Michel,_France,_aerial_view.mp4",
+    "a_DSLR_photo_of_Mount_Fuji,_aerial_view.mp4",
+    "a_DSLR_photo_of_Neuschwanstein_Castle,_aerial_view.mp4",
+    "A_DSLR_photo_of___pyramid_shaped_burrito_with_a_slice_cut_out_of_it.mp4",
+    "a_DSLR_photo_of_the_Imperial_State_Crown_of_England.mp4",
+    "a_DSLR_photo_of_the_leaning_tower_of_Pisa,_aerial_view.mp4",
+    "a_DSLR_photo_of_the_Statue_of_Liberty,_aerial_view.mp4",
+    "a_DSLR_photo_of_Two_locomotives_playing_tug_of_war.mp4",
+    "a_DSLR_photo_of_two_macaw_parrots_sharing_a_milkshake_with_two_straws.mp4",
+    "a_DSLR_photo_of_Westminster_Abbey,_aerial_view.mp4",
+    "a_ficus_planted_in_a_pot.mp4",
+    "a_flower_made_out_of_metal.mp4",
+    "a_fluffy_cat_lying_on_its_back_in_a_patch_of_sunlight.mp4",
+    "a_fox_and_a_hare_tangoing_together.mp4",
+    "a_fox_holding_a_videogame_controller.mp4",
+    "a_fox_playing_the_cello.mp4",
+    "a_frazer_nash_super_sport_car.mp4",
+    "a_freshly_baked_loaf_of_sourdough_bread_on_a_cutting_board.mp4",
+    "a_goat_drinking_beer.mp4",
+    "a_golden_goblet,_low_poly.mp4",
+    "a_green_dragon_breathing_fire.mp4",
+    "a_green_tractor_farming_corn_fields.mp4",
+    "a_highland_cow.mp4",
+    "a_hotdog_in_a_tutu_skirt.mp4",
+    "a_humanoid_robot_laying_on_the_couch_while_on_a_laptop.mp4",
+    "a_humanoid_robot_playing_the_violin.mp4",
+    "a_humanoid_robot_sitting_looking_at_a_Go_board_with_some_pieces_on_it.mp4",
+    "a_human_skeleton_drinking_a_glass_of_red_wine.mp4",
+    "a_human_skull_with_a_vine_growing_through_one_of_the_eye_sockets.mp4",
+    "a_kitten_looking_at_a_goldfish_in_a_bowl.mp4",
+    "a_lemur_drinking_boba.mp4",
+    "a_lemur_taking_notes_in_a_journal.mp4",
+    "a_lionfish.mp4",
+    "a_llama_wearing_a_suit.mp4",
+    "a_marble_bust_of_a_mouse.mp4",
+    "a_metal_sculpture_of_a_lion's_head,_highly_detailed.mp4",
+    "a_mojito_in_a_beach_chair.mp4",
+    "a_monkey-rabbit_hybrid.mp4",
+    "an_airplane_made_out_of_wood.mp4",
+    "an_amigurumi_bulldozer.mp4",
+    "An_anthropomorphic_tomato_eating_another_tomato.mp4",
+    "an_astronaut_playing_the_violin.mp4",
+    "an_astronaut_riding_a_kangaroo.mp4",
+    "an_English_castle,_aerial_view.mp4",
+    "an_erupting_volcano,_aerial_view.mp4",
+    "a_nest_with_a_few_white_eggs_and_one_golden_egg.mp4",
+    "an_exercise_bike.mp4",
+    "an_iridescent_metal_scorpion.mp4",
+    "An_octopus_and_a_giraffe_having_cheesecake.mp4",
+    "an_octopus_playing_the_harp.mp4",
+    "an_old_vintage_car.mp4",
+    "an_opulent_couch_from_the_palace_of_Versailles.mp4",
+    "an_orange_road_bike.mp4",
+    "an_orangutan_holding_a_paint_palette_in_one_hand_and_a_paintbrush_in_the_other.mp4",
+    "an_orangutan_playing_accordion_with_its_hands_spread_wide.mp4",
+    "an_orangutan_using_chopsticks_to_eat_ramen.mp4",
+    "an_orchid_flower_planted_in_a_clay_pot.mp4",
+    "a_palm_tree,_low_poly_3d_model.mp4",
+    "a_panda_rowing_a_boat_in_a_pond.mp4",
+    "a_panda_wearing_a_necktie_and_sitting_in_an_office_chair.mp4",
+    "A_Panther_De_Ville_car.mp4",
+    "a_pig_wearing_a_backpack.mp4",
+    "a_plate_of_delicious_tacos.mp4",
+    "a_plush_dragon_toy.mp4",
+    "a_plush_toy_of_a_corgi_nurse.mp4",
+    "a_rabbit,_animated_movie_character,_high_detail_3d_model.mp4",
+    "a_rabbit_cutting_grass_with_a_lawnmower.mp4",
+    "a_red_eyed_tree_frog,_low_poly.mp4",
+    "a_red_panda.mp4",
+    "a_ripe_strawberry.mp4",
+    "a_roulette_wheel.mp4",
+    "a_shiny_red_stand_mixer.mp4",
+    "a_silver_platter_piled_high_with_fruits.mp4",
+    "a_sliced_loaf_of_fresh_bread.mp4",
+    "a_snail_on_a_leaf.mp4",
+    "a_spanish_galleon_sailing_on_the_open_sea.mp4",
+    "a_squirrel_dressed_like_Henry_VIII_king_of_England.mp4",
+    "a_squirrel_gesturing_in_front_of_an_easel_showing_colorful_pie_charts.mp4",
+    "a_squirrel_wearing_a_tuxedo_and_holding_a_conductor's_baton.mp4",
+    "a_team_of_butterflies_playing_soccer_on_a_field.mp4",
+    "a_teddy_bear_pushing_a_shopping_cart_full_of_fruits_and_vegetables.mp4",
+    "a_tiger_dressed_as_a_military_general.mp4",
+    "a_tiger_karate_master.mp4",
+    "a_tiger_playing_the_violin.mp4",
+    "a_tiger_waiter_at_a_fancy_restaurant.mp4",
+    "a_tiger_wearing_a_tuxedo.mp4",
+    "a_t-rex_roaring_up_into_the_air.mp4",
+    "a_turtle_standing_on_its_hind_legs,_wearing_a_top_hat_and_holding_a_cane.mp4",
+    "a_typewriter.mp4",
+    "a_walrus_smoking_a_pipe.mp4",
+    "a_wedge_of_cheese_on_a_silver_platter.mp4",
+    "a_wide_angle_DSLR_photo_of_a_colorful_rooster.mp4",
+    "a_wide_angle_DSLR_photo_of_a_humanoid_banana_sitting_at_a_desk_doing_homework.mp4",
+    "a_wide_angle_DSLR_photo_of_a_mythical_troll_stirring_a_cauldron.mp4",
+    "a_wide_angle_DSLR_photo_of_a_squirrel_in_samurai_armor_wielding_a_katana.mp4",
+    "a_wide_angle_zoomed_out_DSLR_photo_of_A_red_dragon_dressed_in_a_tuxedo_and_playing_chess._The_chess_pieces_are_fashioned_after_robots.mp4",
+    "a_wide_angle_zoomed_out_DSLR_photo_of_a_skiing_penguin_wearing_a_puffy_jacket.mp4",
+    "a_wide_angle_zoomed_out_DSLR_photo_of_zoomed_out_view_of_Tower_Bridge_made_out_of_gingerbread_and_candy.mp4",
+    "a_woolly_mammoth_standing_on_ice.mp4",
+    "a_yellow_schoolbus.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_3d_model_of_an_adorable_cottage_with_a_thatched_roof.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_baby_bunny_sitting_on_top_of_a_stack_of_pancakes.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_baby_dragon.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_baby_monkey_riding_on_a_pig.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_badger_wearing_a_party_hat_and_blowing_out_birthday_candles_on_a_cake.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_beagle_eating_a_donut.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_bear_playing_electric_bass.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_beautifully_carved_wooden_knight_chess_piece.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_beautiful_suit_made_out_of_moss,_on_a_mannequin._Studio_lighting,_high_quality,_high_resolution.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_blue_lobster.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_blue_tulip.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_bowl_of_cereal_and_milk_with_a_spoon_in_it.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_brain_in_a_jar.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_bulldozer_made_out_of_toy_bricks.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_cake_in_the_shape_of_a_train.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_chihuahua_lying_in_a_pool_ring.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_chimpanzee_dressed_as_a_football_player.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_chimpanzee_holding_a_cup_of_hot_coffee.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_chimpanzee_wearing_headphones.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_colorful_camping_tent_in_a_patch_of_grass.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_complex_movement_from_an_expensive_watch_with_many_shiny_gears,_sitting_on_a_table.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_construction_excavator.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_corgi_wearing_a_top_hat.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_corn_cob_and_a_banana_playing_poker.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_dachsund_riding_a_unicycle.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_dachsund_wearing_a_boater_hat.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_few_pool_balls_sitting_on_a_pool_table.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_fox_working_on_a_jigsaw_puzzle.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_fresh_cinnamon_roll_covered_in_glaze.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_green_tractor.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_greyhound_dog_racing_down_the_track.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_group_of_squirrels_rowing_crew.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_gummy_bear_driving_a_convertible.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_hermit_crab_with_a_colorful_shell.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_hippo_biting_through_a_watermelon.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_hippo_made_out_of_chocolate.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_humanoid_robot_lying_on_a_couch_using_a_laptop.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_humanoid_robot_sitting_on_a_chair_drinking_a_cup_of_coffee.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_human_skeleton_relaxing_in_a_lounge_chair.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_kangaroo_sitting_on_a_bench_playing_the_accordion.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_kingfisher_bird.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_ladybug.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_lion's_mane_jellyfish.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_lobster_playing_the_saxophone.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_majestic_sailboat.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_marble_bust_of_a_cat,_a_real_mouse_is_sitting_on_its_head.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_marble_bust_of_a_fox_head.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_model_of_a_house_in_Tudor_style.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_monkey-rabbit_hybrid.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_monkey_riding_a_bike.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_mountain_goat_standing_on_a_boulder.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_mouse_holding_a_candlestick.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_adorable_kitten_lying_next_to_a_flower.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_all-utility_vehicle_driving_across_a_stream.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_amigurumi_motorcycle.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_astronaut_chopping_vegetables_in_a_sunlit_kitchen.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_egg_cracked_open_with_a_newborn_chick_hatching_out_of_it.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_expensive_office_chair.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_origami_bulldozer_sitting_on_the_ground.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_origami_crane.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_origami_hippo_in_a_river.mp4",
+    "a_zoomed_out_DSLR_photo_of_an_otter_lying_on_its_back_in_the_water_holding_a_flower.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_pair_of_floating_chopsticks_picking_up_noodles_out_of_a_bowl_of_ramen.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_panda_throwing_wads_of_cash_into_the_air.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_panda_wearing_a_chef's_hat_and_kneading_bread_dough_on_a_countertop.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_pigeon_standing_on_a_manhole_cover.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_pig_playing_the_saxophone.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_pile_of_dice_on_a_green_tabletop.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_pita_bread_full_of_hummus_and_falafel_and_vegetables.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_pug_made_out_of_modeling_clay.mp4",
+    "a_zoomed_out_DSLR_photo_of_A_punk_rock_squirrel_in_a_studded_leather_jacket_shouting_into_a_microphone_while_standing_on_a_stump_and_holding_a_beer.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_rabbit_cutting_grass_with_a_lawnmower.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_rabbit_digging_a_hole_with_a_shovel.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_raccoon_astronaut_holding_his_helmet.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_rainforest_bird_mating_ritual_dance.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_recliner_chair.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_red_rotary_telephone.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_robot_couple_fine_dining.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_rotary_telephone_carved_out_of_wood.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_shiny_beetle.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_silver_candelabra_sitting_on_a_red_velvet_tablecloth,_only_one_candle_is_lit.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_squirrel_DJing.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_squirrel_dressed_up_like_a_Victorian_woman.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_table_with_dim_sum_on_it.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_tiger_dressed_as_a_maid.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_tiger_dressed_as_a_military_general.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_tiger_eating_an_ice_cream_cone.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_tiger_wearing_sunglasses_and_a_leather_jacket,_riding_a_motorcycle.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_toad_catching_a_fly_with_its_tongue.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_wizard_raccoon_casting_a_spell.mp4",
+    "a_zoomed_out_DSLR_photo_of_a_yorkie_dog_dressed_as_a_maid.mp4",
+    "a_zoomed_out_DSLR_photo_of_cats_wearing_eyeglasses.mp4",
+    "a_zoomed_out_DSLR_photo_of_miniature_schnauzer_wooden_sculpture,_high_quality_studio_photo.mp4",
+    "A_zoomed_out_DSLR_photo_of___phoenix_made_of_splashing_water_.mp4",
+    "a_zoomed_out_DSLR_photo_of_Sydney_opera_house,_aerial_view.mp4",
+    "a_zoomed_out_DSLR_photo_of_two_foxes_tango_dancing.mp4",
+    "a_zoomed_out_DSLR_photo_of_two_raccoons_playing_poker.mp4",
+    "Chichen_Itza,_aerial_view.mp4",
+    "__Coffee_cup_with_many_holes.mp4",
+    "fries_and_a_hamburger.mp4",
+    "__Luminescent_wild_horses.mp4",
+    "Michelangelo_style_statue_of_an_astronaut.mp4",
+    "Michelangelo_style_statue_of_dog_reading_news_on_a_cellphone.mp4",
+    "the_titanic,_aerial_view.mp4",
+    "two_gummy_bears_playing_dominoes.mp4",
+    "two_macaw_parrots_playing_chess.mp4",
+    "Wedding_dress_made_of_tentacles.mp4",
+]
+
+
+def main():
+    prompt_library = {
+        "dreamfusion": [
+            p.replace(".mp4", "").replace("_", " ")
+            for p in dreamfusion_gallery_video_names
+        ]
+    }
+    with open("load/prompt_library.json", "w") as f:
+        json.dump(prompt_library, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

load/prompt_library.json

@@ -0,0 +1,419 @@
+{
+  "dreamfusion": [
+    "a 20-sided die made out of glass",
+    "a bald eagle carved out of wood",
+    "a banana peeling itself",
+    "a beagle in a detective's outfit",
+    "a beautiful dress made out of fruit, on a mannequin. Studio lighting, high quality, high resolution",
+    "a beautiful dress made out of garbage bags, on a mannequin. Studio lighting, high quality, high resolution",
+    "a beautiful rainbow fish",
+    "a bichon frise wearing academic regalia",
+    "a blue motorcycle",
+    "a blue poison-dart frog sitting on a water lily",
+    "a brightly colored mushroom growing on a log",
+    "a bumblebee sitting on a pink flower",
+    "a bunch of colorful marbles spilling out of a red velvet bag",
+    "a capybara wearing a top hat, low poly",
+    "a cat with a mullet",
+    "a ceramic lion",
+    "a ceramic upside down yellow octopus holding a blue green ceramic cup",
+    "a chihuahua wearing a tutu",
+    "a chimpanzee holding a peeled banana",
+    "a chimpanzee looking through a telescope",
+    "a chimpanzee stirring a bubbling purple potion in a cauldron",
+    "a chimpanzee with a big grin",
+    "a completely destroyed car",
+    "a confused beagle sitting at a desk working on homework",
+    "a corgi taking a selfie",
+    "a crab, low poly",
+    "a crocodile playing a drum set",
+    "a cute steampunk elephant",
+    "a dachsund dressed up in a hotdog costume",
+    "a delicious hamburger",
+    "a dragon-cat hybrid",
+    "a DSLR photo of a baby dragon drinking boba",
+    "a DSLR photo of a baby dragon hatching out of a stone egg",
+    "a DSLR photo of a baby grand piano viewed from far away",
+    "a DSLR photo of a bagel filled with cream cheese and lox",
+    "a DSLR photo of a bald eagle",
+    "a DSLR photo of a barbecue grill cooking sausages and burger patties",
+    "a DSLR photo of a basil plant",
+    "a DSLR photo of a bear dancing ballet",
+    "a DSLR photo of a bear dressed as a lumberjack",
+    "a DSLR photo of a bear dressed in medieval armor",
+    "a DSLR photo of a beautiful violin sitting flat on a table",
+    "a DSLR photo of a blue jay standing on a large basket of rainbow macarons",
+    "a DSLR photo of a bulldozer clearing away a pile of snow",
+    "a DSLR photo of a bulldozer",
+    "a DSLR photo of a cake covered in colorful frosting with a slice being taken out, high resolution",
+    "a DSLR photo of a candelabra with many candles on a red velvet tablecloth",
+    "a DSLR photo of a car made out of cheese",
+    "a DSLR photo of A car made out of sushi",
+    "a DSLR photo of a car made out pizza",
+    "a DSLR photo of a cat lying on its side batting at a ball of yarn",
+    "a DSLR photo of a cat magician making a white dove appear",
+    "a DSLR photo of a cat wearing a bee costume",
+    "a DSLR photo of a cat wearing a lion costume",
+    "a DSLR photo of a cauldron full of gold coins",
+    "a DSLR photo of a chimpanzee dressed like Henry VIII king of England",
+    "a DSLR photo of a chimpanzee dressed like Napoleon Bonaparte",
+    "a DSLR photo of a chow chow puppy",
+    "a DSLR photo of a Christmas tree with donuts as decorations",
+    "a DSLR photo of a chrome-plated duck with a golden beak arguing with an angry turtle in a forest",
+    "a DSLR photo of a classic Packard car",
+    "a DSLR photo of a cocker spaniel wearing a crown",
+    "a DSLR photo of a corgi lying on its back with its tongue lolling out",
+    "a DSLR photo of a corgi puppy",
+    "a DSLR photo of a corgi sneezing",
+    "a DSLR photo of a corgi standing up drinking boba",
+    "a DSLR photo of a corgi taking a selfie",
+    "a DSLR photo of a corgi wearing a beret and holding a baguette, standing up on two hind legs",
+    "a DSLR photo of a covered wagon",
+    "a DSLR photo of a cracked egg with the yolk spilling out on a wooden table",
+    "a DSLR photo of a cup full of pens and pencils",
+    "a DSLR photo of a dalmation wearing a fireman's hat",
+    "a DSLR photo of a delicious chocolate brownie dessert with ice cream on the side",
+    "a DSLR photo of a delicious croissant",
+    "a DSLR photo of A DMC Delorean car",
+    "a DSLR photo of a dog made out of salad",
+    "a DSLR photo of a drum set made of cheese",
+    "a DSLR photo of a drying rack covered in clothes",
+    "a DSLR photo of aerial view of a ruined castle",
+    "a DSLR photo of a football helmet",
+    "a DSLR photo of a fox holding a videogame controller",
+    "a DSLR photo of a fox taking a photograph using a DSLR",
+    "a DSLR photo of a frazer nash super sport car",
+    "a DSLR photo of a frog wearing a sweater",
+    "a DSLR photo of a ghost eating a hamburger",
+    "a DSLR photo of a giant worm emerging from the sand in the middle of the desert",
+    "a DSLR photo of a goose made out of gold",
+    "a DSLR photo of a green monster truck",
+    "a DSLR photo of a group of dogs eating pizza",
+    "a DSLR photo of a group of dogs playing poker",
+    "a DSLR photo of a gummy bear playing the saxophone",
+    "a DSLR photo of a hippo wearing a sweater",
+    "a DSLR photo of a humanoid robot holding a human brain",
+    "a DSLR photo of a humanoid robot playing solitaire",
+    "a DSLR photo of a humanoid robot playing the cello",
+    "a DSLR photo of a humanoid robot using a laptop",
+    "a DSLR photo of a humanoid robot using a rolling pin to roll out dough",
+    "a DSLR photo of a human skull",
+    "a DSLR photo of a kitten standing on top of a giant tortoise",
+    "a DSLR photo of a knight chopping wood",
+    "a DSLR photo of a knight holding a lance and sitting on an armored horse",
+    "a DSLR photo of a koala wearing a party hat and blowing out birthday candles on a cake",
+    "a DSLR photo of a lemur taking notes in a journal",
+    "a DSLR photo of a lion reading the newspaper",
+    "a DSLR photo of a mandarin duck swimming in a pond",
+    "a DSLR photo of a model of the eiffel tower made out of toothpicks",
+    "a DSLR photo of a mouse playing the tuba",
+    "a DSLR photo of a mug of hot chocolate with whipped cream and marshmallows",
+    "a DSLR photo of an adorable piglet in a field",
+    "a DSLR photo of an airplane taking off from the runway",
+    "a DSLR photo of an astronaut standing on the surface of mars",
+    "a DSLR photo of an eggshell broken in two with an adorable chick standing next to it",
+    "a DSLR photo of an elephant skull",
+    "a DSLR photo of an exercise bike in a well lit room",
+    "a DSLR photo of an extravagant mansion, aerial view",
+    "a DSLR photo of an ice cream sundae",
+    "a DSLR photo of an iguana holding a balloon",
+    "a DSLR photo of an intricate and complex dish from a michelin star restaurant",
+    "a DSLR photo of An iridescent steampunk patterned millipede with bison horns",
+    "a DSLR photo of an octopus playing the piano",
+    "a DSLR photo of an old car overgrown by vines and weeds",
+    "a DSLR photo of an old vintage car",
+    "a DSLR photo of an orangutan making a clay bowl on a throwing wheel",
+    "a DSLR photo of an orc forging a hammer on an anvil",
+    "a DSLR photo of an origami motorcycle",
+    "a DSLR photo of an ornate silver gravy boat sitting on a patterned tablecloth",
+    "a DSLR photo of an overstuffed pastrami sandwich",
+    "a DSLR photo of an unstable rock cairn in the middle of a stream",
+    "a DSLR photo of a pair of headphones sitting on a desk",
+    "a DSLR photo of a pair of tan cowboy boots, studio lighting, product photography",
+    "a DSLR photo of a peacock on a surfboard",
+    "a DSLR photo of a pigeon reading a book",
+    "a DSLR photo of a piglet sitting in a teacup",
+    "a DSLR photo of a pig playing a drum set",
+    "a DSLR photo of a pile of dice on a green tabletop next to some playing cards",
+    "a DSLR photo of a pirate collie dog, high resolution",
+    "a DSLR photo of a plate of fried chicken and waffles with maple syrup on them",
+    "a DSLR photo of a plate piled high with chocolate chip cookies",
+    "a DSLR photo of a plush t-rex dinosaur toy, studio lighting, high resolution",
+    "a DSLR photo of a plush triceratops toy, studio lighting, high resolution",
+    "a DSLR photo of a pomeranian dog",
+    "a DSLR photo of a porcelain dragon",
+    "a DSLR photo of a praying mantis wearing roller skates",
+    "a DSLR photo of a puffin standing on a rock",
+    "a DSLR photo of a pug made out of metal",
+    "a DSLR photo of a pug wearing a bee costume",
+    "a DSLR photo of a quill and ink sitting on a desk",
+    "a DSLR photo of a raccoon stealing a pie",
+    "a DSLR photo of a red cardinal bird singing",
+    "a DSLR photo of a red convertible car with the top down",
+    "a DSLR photo of a red-eyed tree frog",
+    "a DSLR photo of a red pickup truck driving across a stream",
+    "a DSLR photo of a red wheelbarrow with a shovel in it",
+    "a DSLR photo of a roast turkey on a platter",
+    "a DSLR photo of a robot and dinosaur playing chess, high resolution",
+    "a DSLR photo of a robot arm picking up a colorful block from a table",
+    "a DSLR photo of a robot cat knocking over a chess piece on a board",
+    "a DSLR photo of a robot dinosaur",
+    "a DSLR photo of a robot made out of vegetables",
+    "a DSLR photo of a robot stegosaurus",
+    "a DSLR photo of a robot tiger",
+    "a DSLR photo of a rolling pin on top of bread dough",
+    "a DSLR photo of a sheepdog running",
+    "a DSLR photo of a shiba inu playing golf wearing tartan golf clothes and hat",
+    "a DSLR photo of a shiny silver robot cat",
+    "a DSLR photo of a silverback gorilla holding a golden trophy",
+    "a DSLR photo of a silver humanoid robot flipping a coin",
+    "a DSLR photo of a small cherry tomato plant in a pot with a few red tomatoes growing on it",
+    "a DSLR photo of a small saguaro cactus planted in a clay pot",
+    "a DSLR photo of a Space Shuttle",
+    "a DSLR photo of a squirrel dressed like a clown",
+    "a DSLR photo of a squirrel flying a biplane",
+    "a DSLR photo of a squirrel giving a lecture writing on a chalkboard",
+    "a DSLR photo of a squirrel holding a bowling ball",
+    "a DSLR photo of a squirrel-lizard hybrid",
+    "a DSLR photo of a squirrel made out of fruit",
+    "a DSLR photo of a squirrel-octopus hybrid",
+    "a DSLR photo of a stack of pancakes covered in maple syrup",
+    "a DSLR photo of a steam engine train, high resolution",
+    "a DSLR photo of a steaming basket full of dumplings",
+    "a DSLR photo of a steaming hot plate piled high with spaghetti and meatballs",
+    "a DSLR photo of a steampunk space ship designed in the 18th century",
+    "a DSLR photo of a straw basket with a cobra coming out of it",
+    "a DSLR photo of a swan and its cygnets swimming in a pond",
+    "a DSLR photo of a tarantula, highly detailed",
+    "a DSLR photo of a teal moped",
+    "a DSLR photo of a teapot shaped like an elephant head where its snout acts as the spout",
+    "a DSLR photo of a teddy bear taking a selfie",
+    "a DSLR photo of a terracotta bunny",
+    "a DSLR photo of a tiger dressed as a doctor",
+    "a DSLR photo of a tiger made out of yarn",
+    "a DSLR photo of a toilet made out of gold",
+    "a DSLR photo of a toy robot",
+    "a DSLR photo of a train engine made out of clay",
+    "a DSLR photo of a tray of Sushi containing pugs",
+    "a DSLR photo of a tree stump with an axe buried in it",
+    "a DSLR photo of a turtle standing on its hind legs, wearing a top hat and holding a cane",
+    "a DSLR photo of a very beautiful small organic sculpture made of fine clockwork and gears with tiny ruby bearings, very intricate, caved, curved. Studio lighting, High resolution, white background",
+    "a DSLR photo of A very beautiful tiny human heart organic sculpture made of copper wire and threaded pipes, very intricate, curved, Studio lighting, high resolution",
+    "a DSLR photo of a very cool and trendy pair of sneakers, studio lighting",
+    "a DSLR photo of a vintage record player",
+    "a DSLR photo of a wine bottle and full wine glass on a chessboard",
+    "a DSLR photo of a wooden desk and chair from an elementary school",
+    "a DSLR photo of a yorkie dog eating a donut",
+    "a DSLR photo of a yorkie dog wearing extremely cool sneakers",
+    "a DSLR photo of baby elephant jumping on a trampoline",
+    "a DSLR photo of cat wearing virtual reality headset in renaissance oil painting high detail caravaggio",
+    "a DSLR photo of edible typewriter made out of vegetables",
+    "a DSLR photo of Mont Saint-Michel, France, aerial view",
+    "a DSLR photo of Mount Fuji, aerial view",
+    "a DSLR photo of Neuschwanstein Castle, aerial view",
+    "A DSLR photo of   pyramid shaped burrito with a slice cut out of it",
+    "a DSLR photo of the Imperial State Crown of England",
+    "a DSLR photo of the leaning tower of Pisa, aerial view",
+    "a DSLR photo of the Statue of Liberty, aerial view",
+    "a DSLR photo of Two locomotives playing tug of war",
+    "a DSLR photo of two macaw parrots sharing a milkshake with two straws",
+    "a DSLR photo of Westminster Abbey, aerial view",
+    "a ficus planted in a pot",
+    "a flower made out of metal",
+    "a fluffy cat lying on its back in a patch of sunlight",
+    "a fox and a hare tangoing together",
+    "a fox holding a videogame controller",
+    "a fox playing the cello",
+    "a frazer nash super sport car",
+    "a freshly baked loaf of sourdough bread on a cutting board",
+    "a goat drinking beer",
+    "a golden goblet, low poly",
+    "a green dragon breathing fire",
+    "a green tractor farming corn fields",
+    "a highland cow",
+    "a hotdog in a tutu skirt",
+    "a humanoid robot laying on the couch while on a laptop",
+    "a humanoid robot playing the violin",
+    "a humanoid robot sitting looking at a Go board with some pieces on it",
+    "a human skeleton drinking a glass of red wine",
+    "a human skull with a vine growing through one of the eye sockets",
+    "a kitten looking at a goldfish in a bowl",
+    "a lemur drinking boba",
+    "a lemur taking notes in a journal",
+    "a lionfish",
+    "a llama wearing a suit",
+    "a marble bust of a mouse",
+    "a metal sculpture of a lion's head, highly detailed",
+    "a mojito in a beach chair",
+    "a monkey-rabbit hybrid",
+    "an airplane made out of wood",
+    "an amigurumi bulldozer",
+    "An anthropomorphic tomato eating another tomato",
+    "an astronaut playing the violin",
+    "an astronaut riding a kangaroo",
+    "an English castle, aerial view",
+    "an erupting volcano, aerial view",
+    "a nest with a few white eggs and one golden egg",
+    "an exercise bike",
+    "an iridescent metal scorpion",
+    "An octopus and a giraffe having cheesecake",
+    "an octopus playing the harp",
+    "an old vintage car",
+    "an opulent couch from the palace of Versailles",
+    "an orange road bike",
+    "an orangutan holding a paint palette in one hand and a paintbrush in the other",
+    "an orangutan playing accordion with its hands spread wide",
+    "an orangutan using chopsticks to eat ramen",
+    "an orchid flower planted in a clay pot",
+    "a palm tree, low poly 3d model",
+    "a panda rowing a boat in a pond",
+    "a panda wearing a necktie and sitting in an office chair",
+    "A Panther De Ville car",
+    "a pig wearing a backpack",
+    "a plate of delicious tacos",
+    "a plush dragon toy",
+    "a plush toy of a corgi nurse",
+    "a rabbit, animated movie character, high detail 3d model",
+    "a rabbit cutting grass with a lawnmower",
+    "a red eyed tree frog, low poly",
+    "a red panda",
+    "a ripe strawberry",
+    "a roulette wheel",
+    "a shiny red stand mixer",
+    "a silver platter piled high with fruits",
+    "a sliced loaf of fresh bread",
+    "a snail on a leaf",
+    "a spanish galleon sailing on the open sea",
+    "a squirrel dressed like Henry VIII king of England",
+    "a squirrel gesturing in front of an easel showing colorful pie charts",
+    "a squirrel wearing a tuxedo and holding a conductor's baton",
+    "a team of butterflies playing soccer on a field",
+    "a teddy bear pushing a shopping cart full of fruits and vegetables",
+    "a tiger dressed as a military general",
+    "a tiger karate master",
+    "a tiger playing the violin",
+    "a tiger waiter at a fancy restaurant",
+    "a tiger wearing a tuxedo",
+    "a t-rex roaring up into the air",
+    "a turtle standing on its hind legs, wearing a top hat and holding a cane",
+    "a typewriter",
+    "a walrus smoking a pipe",
+    "a wedge of cheese on a silver platter",
+    "a wide angle DSLR photo of a colorful rooster",
+    "a wide angle DSLR photo of a humanoid banana sitting at a desk doing homework",
+    "a wide angle DSLR photo of a mythical troll stirring a cauldron",
+    "a wide angle DSLR photo of a squirrel in samurai armor wielding a katana",
+    "a wide angle zoomed out DSLR photo of A red dragon dressed in a tuxedo and playing chess. The chess pieces are fashioned after robots",
+    "a wide angle zoomed out DSLR photo of a skiing penguin wearing a puffy jacket",
+    "a wide angle zoomed out DSLR photo of zoomed out view of Tower Bridge made out of gingerbread and candy",
+    "a woolly mammoth standing on ice",
+    "a yellow schoolbus",
+    "a zoomed out DSLR photo of a 3d model of an adorable cottage with a thatched roof",
+    "a zoomed out DSLR photo of a baby bunny sitting on top of a stack of pancakes",
+    "a zoomed out DSLR photo of a baby dragon",
+    "a zoomed out DSLR photo of a baby monkey riding on a pig",
+    "a zoomed out DSLR photo of a badger wearing a party hat and blowing out birthday candles on a cake",
+    "a zoomed out DSLR photo of a beagle eating a donut",
+    "a zoomed out DSLR photo of a bear playing electric bass",
+    "a zoomed out DSLR photo of a beautifully carved wooden knight chess piece",
+    "a zoomed out DSLR photo of a beautiful suit made out of moss, on a mannequin. Studio lighting, high quality, high resolution",
+    "a zoomed out DSLR photo of a blue lobster",
+    "a zoomed out DSLR photo of a blue tulip",
+    "a zoomed out DSLR photo of a bowl of cereal and milk with a spoon in it",
+    "a zoomed out DSLR photo of a brain in a jar",
+    "a zoomed out DSLR photo of a bulldozer made out of toy bricks",
+    "a zoomed out DSLR photo of a cake in the shape of a train",
+    "a zoomed out DSLR photo of a chihuahua lying in a pool ring",
+    "a zoomed out DSLR photo of a chimpanzee dressed as a football player",
+    "a zoomed out DSLR photo of a chimpanzee holding a cup of hot coffee",
+    "a zoomed out DSLR photo of a chimpanzee wearing headphones",
+    "a zoomed out DSLR photo of a colorful camping tent in a patch of grass",
+    "a zoomed out DSLR photo of a complex movement from an expensive watch with many shiny gears, sitting on a table",
+    "a zoomed out DSLR photo of a construction excavator",
+    "a zoomed out DSLR photo of a corgi wearing a top hat",
+    "a zoomed out DSLR photo of a corn cob and a banana playing poker",
+    "a zoomed out DSLR photo of a dachsund riding a unicycle",
+    "a zoomed out DSLR photo of a dachsund wearing a boater hat",
+    "a zoomed out DSLR photo of a few pool balls sitting on a pool table",
+    "a zoomed out DSLR photo of a fox working on a jigsaw puzzle",
+    "a zoomed out DSLR photo of a fresh cinnamon roll covered in glaze",
+    "a zoomed out DSLR photo of a green tractor",
+    "a zoomed out DSLR photo of a greyhound dog racing down the track",
+    "a zoomed out DSLR photo of a group of squirrels rowing crew",
+    "a zoomed out DSLR photo of a gummy bear driving a convertible",
+    "a zoomed out DSLR photo of a hermit crab with a colorful shell",
+    "a zoomed out DSLR photo of a hippo biting through a watermelon",
+    "a zoomed out DSLR photo of a hippo made out of chocolate",
+    "a zoomed out DSLR photo of a humanoid robot lying on a couch using a laptop",
+    "a zoomed out DSLR photo of a humanoid robot sitting on a chair drinking a cup of coffee",
+    "a zoomed out DSLR photo of a human skeleton relaxing in a lounge chair",
+    "a zoomed out DSLR photo of a kangaroo sitting on a bench playing the accordion",
+    "a zoomed out DSLR photo of a kingfisher bird",
+    "a zoomed out DSLR photo of a ladybug",
+    "a zoomed out DSLR photo of a lion's mane jellyfish",
+    "a zoomed out DSLR photo of a lobster playing the saxophone",
+    "a zoomed out DSLR photo of a majestic sailboat",
+    "a zoomed out DSLR photo of a marble bust of a cat, a real mouse is sitting on its head",
+    "a zoomed out DSLR photo of a marble bust of a fox head",
+    "a zoomed out DSLR photo of a model of a house in Tudor style",
+    "a zoomed out DSLR photo of a monkey-rabbit hybrid",
+    "a zoomed out DSLR photo of a monkey riding a bike",
+    "a zoomed out DSLR photo of a mountain goat standing on a boulder",
+    "a zoomed out DSLR photo of a mouse holding a candlestick",
+    "a zoomed out DSLR photo of an adorable kitten lying next to a flower",
+    "a zoomed out DSLR photo of an all-utility vehicle driving across a stream",
+    "a zoomed out DSLR photo of an amigurumi motorcycle",
+    "a zoomed out DSLR photo of an astronaut chopping vegetables in a sunlit kitchen",
+    "a zoomed out DSLR photo of an egg cracked open with a newborn chick hatching out of it",
+    "a zoomed out DSLR photo of an expensive office chair",
+    "a zoomed out DSLR photo of an origami bulldozer sitting on the ground",
+    "a zoomed out DSLR photo of an origami crane",
+    "a zoomed out DSLR photo of an origami hippo in a river",
+    "a zoomed out DSLR photo of an otter lying on its back in the water holding a flower",
+    "a zoomed out DSLR photo of a pair of floating chopsticks picking up noodles out of a bowl of ramen",
+    "a zoomed out DSLR photo of a panda throwing wads of cash into the air",
+    "a zoomed out DSLR photo of a panda wearing a chef's hat and kneading bread dough on a countertop",
+    "a zoomed out DSLR photo of a pigeon standing on a manhole cover",
+    "a zoomed out DSLR photo of a pig playing the saxophone",
+    "a zoomed out DSLR photo of a pile of dice on a green tabletop",
+    "a zoomed out DSLR photo of a pita bread full of hummus and falafel and vegetables",
+    "a zoomed out DSLR photo of a pug made out of modeling clay",
+    "a zoomed out DSLR photo of A punk rock squirrel in a studded leather jacket shouting into a microphone while standing on a stump and holding a beer",
+    "a zoomed out DSLR photo of a rabbit cutting grass with a lawnmower",
+    "a zoomed out DSLR photo of a rabbit digging a hole with a shovel",
+    "a zoomed out DSLR photo of a raccoon astronaut holding his helmet",
+    "a zoomed out DSLR photo of a rainforest bird mating ritual dance",
+    "a zoomed out DSLR photo of a recliner chair",
+    "a zoomed out DSLR photo of a red rotary telephone",
+    "a zoomed out DSLR photo of a robot couple fine dining",
+    "a zoomed out DSLR photo of a rotary telephone carved out of wood",
+    "a zoomed out DSLR photo of a shiny beetle",
+    "a zoomed out DSLR photo of a silver candelabra sitting on a red velvet tablecloth, only one candle is lit",
+    "a zoomed out DSLR photo of a squirrel DJing",
+    "a zoomed out DSLR photo of a squirrel dressed up like a Victorian woman",
+    "a zoomed out DSLR photo of a table with dim sum on it",
+    "a zoomed out DSLR photo of a tiger dressed as a maid",
+    "a zoomed out DSLR photo of a tiger dressed as a military general",
+    "a zoomed out DSLR photo of a tiger eating an ice cream cone",
+    "a zoomed out DSLR photo of a tiger wearing sunglasses and a leather jacket, riding a motorcycle",
+    "a zoomed out DSLR photo of a toad catching a fly with its tongue",
+    "a zoomed out DSLR photo of a wizard raccoon casting a spell",
+    "a zoomed out DSLR photo of a yorkie dog dressed as a maid",
+    "a zoomed out DSLR photo of cats wearing eyeglasses",
+    "a zoomed out DSLR photo of miniature schnauzer wooden sculpture, high quality studio photo",
+    "A zoomed out DSLR photo of   phoenix made of splashing water ",
+    "a zoomed out DSLR photo of Sydney opera house, aerial view",
+    "a zoomed out DSLR photo of two foxes tango dancing",
+    "a zoomed out DSLR photo of two raccoons playing poker",
+    "Chichen Itza, aerial view",
+    "  Coffee cup with many holes",
+    "fries and a hamburger",
+    "  Luminescent wild horses",
+    "Michelangelo style statue of an astronaut",
+    "Michelangelo style statue of dog reading news on a cellphone",
+    "the titanic, aerial view",
+    "two gummy bears playing dominoes",
+    "two macaw parrots playing chess",
+    "Wedding dress made of tentacles"
+  ]
+}

load/tets/generate_tets.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
+# property and proprietary rights in and to this material, related
+# documentation and any modifications thereto. Any use, reproduction,
+# disclosure or distribution of this material and related documentation
+# without an express license agreement from NVIDIA CORPORATION or
+# its affiliates is strictly prohibited.
+
+import os
+
+import numpy as np
+
+"""
+This code segment shows how to use Quartet: https://github.com/crawforddoran/quartet,
+to generate a tet grid
+1) Download, compile and run Quartet as described in the link above. Example usage `quartet meshes/cube.obj 0.5 cube_5.tet`
+2) Run the function below to generate a file `cube_32_tet.tet`
+"""
+
+
+def generate_tetrahedron_grid_file(res=32, root=".."):
+    frac = 1.0 / res
+    command = f"cd {root}; ./quartet meshes/cube.obj {frac} meshes/cube_{res}_tet.tet -s meshes/cube_boundary_{res}.obj"
+    os.system(command)
+
+
+"""
+This code segment shows how to convert from a quartet .tet file to compressed npz file
+"""
+
+
+def convert_from_quartet_to_npz(quartetfile="cube_32_tet.tet", npzfile="32_tets"):
+    file1 = open(quartetfile, "r")
+    header = file1.readline()
+    numvertices = int(header.split(" ")[1])
+    numtets = int(header.split(" ")[2])
+    print(numvertices, numtets)
+
+    # load vertices
+    vertices = np.loadtxt(quartetfile, skiprows=1, max_rows=numvertices)
+    print(vertices.shape)
+
+    # load indices
+    indices = np.loadtxt(
+        quartetfile, dtype=int, skiprows=1 + numvertices, max_rows=numtets
+    )
+    print(indices.shape)
+
+    np.savez_compressed(npzfile, vertices=vertices, indices=indices)
+
+
+root = "/home/gyc/quartet"
+for res in [300, 350, 400]:
+    generate_tetrahedron_grid_file(res, root)
+    convert_from_quartet_to_npz(
+        os.path.join(root, f"meshes/cube_{res}_tet.tet"), npzfile=f"{res}_tets"
+    )

load/zero123/download.sh

@@ -0,0 +1,4 @@
+# wget https://huggingface.co/cvlab/zero123-weights/resolve/main/105000.ckpt
+# mv 105000.ckpt zero123-original.ckpt
+wget https://zero123.cs.columbia.edu/assets/zero123-xl.ckpt
+# Download stable_zero123.ckpt from https://huggingface.co/stabilityai/stable-zero123

load/zero123/sd-objaverse-finetune-c_concat-256.yaml

@@ -0,0 +1,117 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: extern.ldm_zero123.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "image_target"
+    cond_stage_key: "image_cond"
+    image_size: 32
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: hybrid
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+
+    scheduler_config: # 10000 warmup steps
+      target: extern.ldm_zero123.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 100 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: extern.ldm_zero123.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 8
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: extern.ldm_zero123.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: extern.ldm_zero123.modules.encoders.modules.FrozenCLIPImageEmbedder
+
+
+# data:
+#   target: extern.ldm_zero123.data.simple.ObjaverseDataModuleFromConfig
+#   params:
+#     root_dir: 'views_whole_sphere'
+#     batch_size: 192
+#     num_workers: 16
+#     total_view: 4
+#     train:
+#       validation: False
+#       image_transforms:
+#         size: 256
+
+#     validation:
+#       validation: True
+#       image_transforms:
+#         size: 256
+
+
+# lightning:
+#   find_unused_parameters: false
+#   metrics_over_trainsteps_checkpoint: True
+#   modelcheckpoint:
+#     params:
+#       every_n_train_steps: 5000
+#   callbacks:
+#     image_logger:
+#       target: main.ImageLogger
+#       params:
+#         batch_frequency: 500
+#         max_images: 32
+#         increase_log_steps: False
+#         log_first_step: True
+#         log_images_kwargs:
+#           use_ema_scope: False
+#           inpaint: False
+#           plot_progressive_rows: False
+#           plot_diffusion_rows: False
+#           N: 32
+#           unconditional_scale: 3.0
+#           unconditional_label: [""]
+
+#   trainer:
+#     benchmark: True
+#     val_check_interval: 5000000 # really sorry
+#     num_sanity_val_steps: 0
+#     accumulate_grad_batches: 1

preprocess_image.py

@@ -130,75 +130,70 @@ def preprocess_single_image(img_path, args):
     print(img_path)
     image = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
     carved_image = None
-    # debug
+
     if image.shape[-1] == 4:
-        if args.do_rm_bg_force:
+        carved_image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
-            image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
+        image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
-        else:
-            carved_image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
-            image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
 
     else:
         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 
-    if args.do_seg:
+    if carved_image is None:
-        if carved_image is None:
+        # carve background
-            # carve background
+        print(f'[INFO] background removal...')
-            print(f'[INFO] background removal...')
+        carved_image = BackgroundRemoval()(image) # [H, W, 4]
-            carved_image = BackgroundRemoval()(image) # [H, W, 4]
+    mask = carved_image[..., -1] > 0
-        mask = carved_image[..., -1] > 0
 
-        # predict depth
+    # predict depth
-        print(f'[INFO] depth estimation...')
+    print(f'[INFO] depth estimation...')
-        dpt_depth_model = DPT(task='depth')
+    dpt_depth_model = DPT(task='depth')
-        depth = dpt_depth_model(image)[0]
+    depth = dpt_depth_model(image)[0]
-        depth[mask] = (depth[mask] - depth[mask].min()) / (depth[mask].max() - depth[mask].min() + 1e-9)
+    depth[mask] = (depth[mask] - depth[mask].min()) / (depth[mask].max() - depth[mask].min() + 1e-9)
-        depth[~mask] = 0
+    depth[~mask] = 0
-        depth = (depth * 255).astype(np.uint8)
+    depth = (depth * 255).astype(np.uint8)
-        del dpt_depth_model
+    del dpt_depth_model
 
-        # predict normal
+    # predict normal
-        print(f'[INFO] normal estimation...')
+    print(f'[INFO] normal estimation...')
-        dpt_normal_model = DPT(task='normal')
+    dpt_normal_model = DPT(task='normal')
-        normal = dpt_normal_model(image)[0]
+    normal = dpt_normal_model(image)[0]
-        normal = (normal * 255).astype(np.uint8).transpose(1, 2, 0)
+    normal = (normal * 255).astype(np.uint8).transpose(1, 2, 0)
-        normal[~mask] = 0
+    normal[~mask] = 0
-        del dpt_normal_model
+    del dpt_normal_model
 
-        opt.recenter=False
+    # recenter
-        # recenter
+    if opt.recenter:
-        if opt.recenter:
+        print(f'[INFO] recenter...')
-            print(f'[INFO] recenter...')
+        final_rgba = np.zeros((opt.size, opt.size, 4), dtype=np.uint8)
-            final_rgba = np.zeros((opt.size, opt.size, 4), dtype=np.uint8)
+        final_depth = np.zeros((opt.size, opt.size), dtype=np.uint8)
-            final_depth = np.zeros((opt.size, opt.size), dtype=np.uint8)
+        final_normal = np.zeros((opt.size, opt.size, 3), dtype=np.uint8)
-            final_normal = np.zeros((opt.size, opt.size, 3), dtype=np.uint8)
 
-            coords = np.nonzero(mask)
+        coords = np.nonzero(mask)
-            x_min, x_max = coords[0].min(), coords[0].max()
+        x_min, x_max = coords[0].min(), coords[0].max()
-            y_min, y_max = coords[1].min(), coords[1].max()
+        y_min, y_max = coords[1].min(), coords[1].max()
-            h = x_max - x_min
+        h = x_max - x_min
-            w = y_max - y_min
+        w = y_max - y_min
-            desired_size = int(opt.size * (1 - opt.border_ratio))
+        desired_size = int(opt.size * (1 - opt.border_ratio))
-            scale = desired_size / max(h, w)
+        scale = desired_size / max(h, w)
-            h2 = int(h * scale)
+        h2 = int(h * scale)
-            w2 = int(w * scale)
+        w2 = int(w * scale)
-            x2_min = (opt.size - h2) // 2
+        x2_min = (opt.size - h2) // 2
-            x2_max = x2_min + h2
+        x2_max = x2_min + h2
-            y2_min = (opt.size - w2) // 2
+        y2_min = (opt.size - w2) // 2
-            y2_max = y2_min + w2
+        y2_max = y2_min + w2
-            final_rgba[x2_min:x2_max, y2_min:y2_max] = cv2.resize(carved_image[x_min:x_max, y_min:y_max], (w2, h2), interpolation=cv2.INTER_AREA)
+        final_rgba[x2_min:x2_max, y2_min:y2_max] = cv2.resize(carved_image[x_min:x_max, y_min:y_max], (w2, h2), interpolation=cv2.INTER_AREA)
-            final_depth[x2_min:x2_max, y2_min:y2_max] = cv2.resize(depth[x_min:x_max, y_min:y_max], (w2, h2), interpolation=cv2.INTER_AREA)
+        final_depth[x2_min:x2_max, y2_min:y2_max] = cv2.resize(depth[x_min:x_max, y_min:y_max], (w2, h2), interpolation=cv2.INTER_AREA)
-            final_normal[x2_min:x2_max, y2_min:y2_max] = cv2.resize(normal[x_min:x_max, y_min:y_max], (w2, h2), interpolation=cv2.INTER_AREA)
+        final_normal[x2_min:x2_max, y2_min:y2_max] = cv2.resize(normal[x_min:x_max, y_min:y_max], (w2, h2), interpolation=cv2.INTER_AREA)
 
-        else:
+    else:
-            final_rgba = carved_image
+        final_rgba = carved_image
-            final_depth = depth
+        final_depth = depth
-            final_normal = normal
+        final_normal = normal
 
-        # write output
+    # write output
-        cv2.imwrite(out_rgba, cv2.cvtColor(final_rgba, cv2.COLOR_RGBA2BGRA))
+    cv2.imwrite(out_rgba, cv2.cvtColor(final_rgba, cv2.COLOR_RGBA2BGRA))
-        cv2.imwrite(out_depth, final_depth)
+    cv2.imwrite(out_depth, final_depth)
-        cv2.imwrite(out_normal, final_normal)
+    cv2.imwrite(out_normal, final_normal)
 
     if opt.do_caption:
         # predict caption (it's too slow... use your brain instead)
@@ -215,11 +210,8 @@ if __name__ == '__main__':
     parser.add_argument('path', type=str, help="path to image (png, jpeg, etc.)")
     parser.add_argument('--size', default=1024, type=int, help="output resolution")
     parser.add_argument('--border_ratio', default=0.1, type=float, help="output border ratio")
-    parser.add_argument('--recenter', type=bool, default=False, help="recenter, potentially not helpful for multiview zero123")
+    parser.add_argument('--recenter', action='store_true', help="recenter, potentially not helpful for multiview zero123")
-    parser.add_argument('--dont_recenter', dest='recenter', action='store_false')
+    parser.add_argument('--do_caption', action='store_true', help="do text captioning")
-    parser.add_argument('--do_caption', type=bool, default=False, help="do text captioning")
-    parser.add_argument('--do_seg', type=bool, default=True)
-    parser.add_argument('--do_rm_bg_force', type=bool, default=False)
     
     opt = parser.parse_args()