diff --git a/internal/config/config.go b/internal/config/config.go
index c189ac54..576e16d7 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -151,6 +151,7 @@ func getDefault() (*Config, error) {
 				CDI: cdiModeConfig{
 					DefaultKind:        "nvidia.com/gpu",
 					AnnotationPrefixes: []string{cdi.AnnotationPrefix},
+					SpecDirs:           cdi.DefaultSpecDirs,
 				},
 			},
 		},
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 0e709f9b..029a0109 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -77,6 +77,7 @@ func TestGetConfig(t *testing.T) {
 						CDI: cdiModeConfig{
 							DefaultKind:        "nvidia.com/gpu",
 							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
 						},
 					},
 				},
@@ -102,6 +103,7 @@ func TestGetConfig(t *testing.T) {
 				"nvidia-container-runtime.mode = \"not-auto\"",
 				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
 				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"nvidia-container-runtime.modes.cdi.spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"nvidia-container-runtime-hook.path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
@@ -128,6 +130,10 @@ func TestGetConfig(t *testing.T) {
 								"cdi.k8s.io/",
 								"example.vendor.com/",
 							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
 						},
 					},
 				},
@@ -156,6 +162,7 @@ func TestGetConfig(t *testing.T) {
 				"[nvidia-container-runtime.modes.cdi]",
 				"default-kind = \"example.vendor.com/device\"",
 				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"[nvidia-container-runtime-hook]",
@@ -185,6 +192,10 @@ func TestGetConfig(t *testing.T) {
 								"cdi.k8s.io/",
 								"example.vendor.com/",
 							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
 						},
 					},
 				},
diff --git a/internal/modifier/cdi.go b/internal/modifier/cdi.go
index 4578534e..a5e0f3d0 100644
--- a/internal/modifier/cdi.go
+++ b/internal/modifier/cdi.go
@@ -48,14 +48,9 @@ func NewCDIModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spe
 	}
 	logger.Debugf("Creating CDI modifier for devices: %v", devices)
 
-	specDirs := cdi.DefaultSpecDirs
-	if len(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs) > 0 {
-		specDirs = cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs
-	}
-
 	m := cdiModifier{
 		logger:   logger,
-		specDirs: specDirs,
+		specDirs: cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs,
 		devices:  devices,
 	}