DreamCraft3D/configs/dreamcraft3d-coarse-nerf.yaml

name: "dreamcraft3d-coarse-nerf"
tag: "${rmspace:${system.prompt_processor.prompt},_}"
exp_root_dir: "outputs"
seed: 0

data_type: "single-image-datamodule"
data:
  image_path: ./load/images/hamburger_rgba.png
  height: [128, 384]
  width: [128, 384]
  resolution_milestones: [3000]
  default_elevation_deg: 0.0
  default_azimuth_deg: 0.0
  default_camera_distance: 3.8
  default_fovy_deg: 20.0
  requires_depth: true
  requires_normal: ${cmaxgt0:${system.loss.lambda_normal}}
  random_camera:
    height: [128, 384]
    width: [128, 384]
    batch_size: [1, 1]
    resolution_milestones: [3000]
    eval_height: 512
    eval_width: 512
    eval_batch_size: 1
    elevation_range: [-10, 45]
    azimuth_range: [-180, 180]
    camera_distance_range: [3.8, 3.8]
    fovy_range: [20.0, 20.0] # Zero123 has fixed fovy
    progressive_until: 200
    camera_perturb: 0.0
    center_perturb: 0.0
    up_perturb: 0.0
    eval_elevation_deg: ${data.default_elevation_deg}
    eval_camera_distance: ${data.default_camera_distance}
    eval_fovy_deg: ${data.default_fovy_deg}
    batch_uniform_azimuth: false
    n_val_views: 40
    n_test_views: 120

system_type: "dreamcraft3d-system"
system:
  stage: coarse
  geometry_type: "implicit-volume"
  geometry:
    radius: 2.0
    normal_type: "finite_difference"

    # the density initialization proposed in the DreamFusion paper
    # does not work very well
    # density_bias: "blob_dreamfusion"
    # density_activation: exp
    # density_blob_scale: 5.
    # density_blob_std: 0.2

    # use Magic3D density initialization instead
    density_bias: "blob_magic3d"
    density_activation: softplus
    density_blob_scale: 10.
    density_blob_std: 0.5

    # coarse to fine hash grid encoding
    # to ensure smooth analytic normals
    pos_encoding_config:
      otype: ProgressiveBandHashGrid
      n_levels: 16
      n_features_per_level: 2
      log2_hashmap_size: 19
      base_resolution: 16
      per_level_scale: 1.447269237440378 # max resolution 4096
      start_level: 8 # resolution ~200
      start_step: 2000
      update_steps: 500

  material_type: "no-material"
  material:
    requires_normal: true

  background_type: "solid-color-background"

  renderer_type: "nerf-volume-renderer"
  renderer:
    radius: ${system.geometry.radius}
    num_samples_per_ray: 512
    return_normal_perturb: true
    return_comp_normal: ${cmaxgt0:${system.loss.lambda_normal_smooth}}

  prompt_processor_type: "deep-floyd-prompt-processor"
  prompt_processor:
    pretrained_model_name_or_path: "DeepFloyd/IF-I-XL-v1.0"
    prompt: ???
    use_perp_neg: true

  guidance_type: "deep-floyd-guidance"
  guidance:
    pretrained_model_name_or_path: "DeepFloyd/IF-I-XL-v1.0"
    guidance_scale: 20
    min_step_percent: [0, 0.7, 0.2, 200]
    max_step_percent: [0, 0.85, 0.5, 200]

  guidance_3d_type: "stable-zero123-guidance"
  guidance_3d:
    pretrained_model_name_or_path: "./load/zero123/stable_zero123.ckpt"
    pretrained_config: "./load/zero123/sd-objaverse-finetune-c_concat-256.yaml"
    cond_image_path: ${data.image_path}
    cond_elevation_deg: ${data.default_elevation_deg}
    cond_azimuth_deg: ${data.default_azimuth_deg}
    cond_camera_distance: ${data.default_camera_distance}
    guidance_scale: 5.0
    min_step_percent: [0, 0.7, 0.2, 200]  # (start_iter, start_val, end_val, end_iter)
    max_step_percent: [0, 0.85, 0.5, 200]

  freq:
    n_ref: 2
    ref_only_steps: 0
    ref_or_guidance: "alternate"
    no_diff_steps: 0
    guidance_eval: 0

  loggers:
    wandb:
      enable: false
      project: "threestudio"

  loss:
    lambda_sd: 0.1
    lambda_3d_sd: 0.1
    lambda_rgb: 1000.0
    lambda_mask: 100.0
    lambda_mask_binary: 0.0
    lambda_depth: 0.0
    lambda_depth_rel: 0.05
    lambda_normal: 0.0
    lambda_normal_smooth: 1.0
    lambda_3d_normal_smooth: [2000, 5., 1., 2001]
    lambda_orient: [2000, 1., 10., 2001]
    lambda_sparsity: [2000, 0.1, 10., 2001]
    lambda_opaque: [2000, 0.1, 10., 2001]
    lambda_clip: 0.0

  optimizer:
    name: Adam
    args:
      lr: 0.01
      betas: [0.9, 0.99]
      eps: 1.e-8

trainer:
  max_steps: 5000
  log_every_n_steps: 1
  num_sanity_val_steps: 0
  val_check_interval: 200
  enable_progress_bar: true
  precision: 16-mixed

checkpoint:
  save_last: true
  save_top_k: -1
  every_n_train_steps: ${trainer.max_steps}