Merge branch 'skip-for-point-release' into 'main'

Skip components for patch releases

See merge request nvidia/container-toolkit/container-toolkit!374

.common-ci.yml

@@ -23,6 +23,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -34,14 +35,44 @@ stages:
   - scan
   - release
 
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
 .main-or-manual:
   rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+    - !reference [.pipeline-trigger-rules, rules]
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: manual
 
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
+  rules:
+    - !reference [.main-or-manual, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
+
 # Define the distribution targets
 .dist-amazonlinux2:
   rules:

.github/workflows/blossom-ci.yaml

@@ -0,0 +1,113 @@
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to trigger ci on hybrid infra (github + self hosted runner)
+name: Blossom-CI
+on:
+  issue_comment:
+    types: [created]
+  workflow_dispatch:
+      inputs:
+          platform:
+            description: 'runs-on argument'     
+            required: false
+          args:
+            description: 'argument'     
+            required: false
+jobs:
+  Authorization:
+    name: Authorization
+    runs-on: blossom 
+    outputs:
+      args: ${{ env.args }}
+     
+    # This job only runs for pull request comments
+    if: |
+         contains( '\
+         anstockatnv,\
+         rohitrajani2018,\
+         cdesiniotis,\
+         shivamerla,\
+         ArangoGutierrez,\
+         elezar,\
+         klueska,\
+         zvonkok,\
+         ', format('{0},', github.actor)) && 
+         github.event.comment.body == '/blossom-ci'  
+    steps:
+      - name: Check if comment is issued by authorized person
+        run: blossom-ci
+        env:
+          OPERATION: 'AUTH'
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+      
+  Vulnerability-scan:
+    name: Vulnerability scan
+    needs: [Authorization]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
+          ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
+          lfs: 'true'
+      
+      # repo specific steps 
+      #- name: Setup java
+      #  uses: actions/setup-java@v1
+      #  with:
+      #    java-version: 1.8
+      
+      # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file
+      #- name: Setup blackduck properties
+      #  run: |
+      #       PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g')
+      #       echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties
+      #       echo detect.maven.included.scopes=compile >> application.properties
+          
+      - name: Run blossom action
+        uses: NVIDIA/blossom-action@main
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+        with:
+          args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }}
+          args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }}
+          args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }}
+          
+  Job-trigger:
+    name: Start ci job
+    needs: [Vulnerability-scan]
+    runs-on: blossom
+    steps:
+      - name: Start ci job
+        run: blossom-ci
+        env:
+          OPERATION: 'START-CI-JOB'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              
+  Upload-Log:
+    name: Upload log
+    runs-on: blossom
+    if : github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here)
+        run: blossom-ci
+        env:
+          OPERATION: 'POST-PROCESSING'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,9 +1,11 @@
 dist
+artifacts
 *.swp
 *.swo
 /coverage.out*
 /test/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
 /nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk

.nvidia-ci.yml

@@ -36,8 +36,7 @@ variables:
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
   ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
-  # TODO: We should set the kitmaker release folder here once we have the end-to-end workflow set up
-  KITMAKER_RELEASE_FOLDER: "testing"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
 
 .image-pull:
   stage: image-build

CHANGELOG.md

@@ -1,17 +1,76 @@
 # NVIDIA Container Toolkit Changelog
 
-## v1.12.1
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
 
 * Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
 * Fix possible blank `nvidia-ctk` path in generated CDI specifications
 * Fix error in postun scriplet on RPM-based systems
-* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
-* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
-* Remove `fedora35` packaging targets
-* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
 * [toolkit-container] Install `nvidia-ctk` from toolkit container
 * [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
 * [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
 
 ## v1.12.0
 

build/container/Dockerfile.packaging

@@ -29,6 +29,7 @@ ARG PACKAGE_DIST
 ARG PACKAGE_VERSION
 ARG GIT_BRANCH
 ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
 ARG SOURCE_DATE_EPOCH
 ARG VERSION
 

build/container/Makefile

@@ -50,7 +50,7 @@ META_TARGETS := packaging
 
 BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
 PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
 
 .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -95,6 +95,7 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
 		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
 		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
 		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
 		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
@@ -105,24 +106,20 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
 build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
 
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
 build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-centos7: BASE_DIST = $(*)
 build-centos7: DOCKERFILE_SUFFIX := centos
 build-centos7: PACKAGE_DIST = $(BASE_DIST)
-build-centos7: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)

cmd/nvidia-container-runtime-hook/container_config.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
 )
 
@@ -130,7 +131,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -139,28 +140,31 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		for _, c := range caps {
+			if c == capSysAdmin {
+				return true
+			}
 		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
+		return false
 	}
 
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	process := specs.Process{
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
 func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -43,8 +43,9 @@ type HookConfig struct {
 	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
 	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
+	NvidiaContainerCLI         CLIConfig                `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime     config.RuntimeConfig     `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHook config.RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
 }
 
 func getDefaultHookConfig() HookConfig {
@@ -66,7 +67,8 @@ func getDefaultHookConfig() HookConfig {
 			User:        nil,
 			Ldconfig:    nil,
 		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntime:     *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntimeHook: *config.GetDefaultRuntimeHookConfig(),
 	}
 }
 

cmd/nvidia-container-runtime-hook/main.go

@@ -74,7 +74,7 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
+	if !hook.NVIDIAContainerRuntimeHook.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
 		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
 	}
 

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/main.go

@@ -1,89 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer func() {
-		if rerr != nil {
-			logger.Errorf("%v", rerr)
-		}
-		logger.Reset()
-	}()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,7 +42,7 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		logrus.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
@@ -53,11 +54,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		logrus.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		logrus.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,7 +78,7 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -91,9 +92,10 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -170,7 +172,7 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	m := modifier.NewStableRuntimeModifier(logrus.StandardLogger())
 	return m.Modify(spec)
 }
 

cmd/nvidia-ctk/cdi/cdi.go

@@ -18,6 +18,7 @@ package cdi
 
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -44,6 +45,7 @@ func (m command) build() *cli.Command {
 
 	hook.Subcommands = []*cli.Command{
 		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -18,25 +18,22 @@ package generate
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-	"sigs.k8s.io/yaml"
 )
 
 const (
-	formatJSON = "json"
-	formatYAML = "yaml"
+	allDeviceName = "all"
 )
 
 type command struct {
@@ -49,6 +46,9 @@ type config struct {
 	deviceNameStrategy string
 	driverRoot         string
 	nvidiaCTKPath      string
+	mode               string
+	vendor             string
+	class              string
 }
 
 // NewCommand constructs a generate-cdi command with the specified logger
@@ -84,13 +84,20 @@ func (m command) build() *cli.Command {
 		&cli.StringFlag{
 			Name:        "format",
 			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
-			Value:       formatYAML,
+			Value:       spec.FormatYAML,
 			Destination: &cfg.format,
 		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Aliases:     []string{"discovery-mode"},
+			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       nvcdi.ModeAuto,
+			Destination: &cfg.mode,
+		},
 		&cli.StringFlag{
 			Name:        "device-name-strategy",
 			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
-			Value:       deviceNameStrategyIndex,
+			Value:       nvcdi.DeviceNameStrategyIndex,
 			Destination: &cfg.deviceNameStrategy,
 		},
 		&cli.StringFlag{
@@ -103,61 +110,52 @@ func (m command) build() *cli.Command {
 			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
 			Destination: &cfg.nvidiaCTKPath,
 		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &cfg.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &cfg.class,
+		},
 	}
 
 	return &c
 }
 
-func (m command) validateFlags(r *cli.Context, cfg *config) error {
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+
 	cfg.format = strings.ToLower(cfg.format)
 	switch cfg.format {
-	case formatJSON:
-	case formatYAML:
+	case spec.FormatJSON:
+	case spec.FormatYAML:
 	default:
 		return fmt.Errorf("invalid output format: %v", cfg.format)
 	}
 
-	_, err := newDeviceNamer(cfg.deviceNameStrategy)
+	cfg.mode = strings.ToLower(cfg.mode)
+	switch cfg.mode {
+	case nvcdi.ModeAuto:
+	case nvcdi.ModeNvml:
+	case nvcdi.ModeWsl:
+	case nvcdi.ModeManagement:
+	default:
+		return fmt.Errorf("invalid discovery mode: %v", cfg.mode)
+	}
+
+	_, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
 	if err != nil {
 		return err
 	}
 
 	cfg.nvidiaCTKPath = discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath)
 
-	return nil
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	deviceNamer, err := newDeviceNamer(cfg.deviceNameStrategy)
-	if err != nil {
-		return fmt.Errorf("failed to create device namer: %v", err)
-	}
-
-	spec, err := m.generateSpec(
-		cfg.driverRoot,
-		discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath),
-		deviceNamer,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to generate CDI spec: %v", err)
-	}
-
-	var outputTo io.Writer
-	if cfg.output == "" {
-		outputTo = os.Stdout
-	} else {
-		err := createParentDirsIfRequired(cfg.output)
-		if err != nil {
-			return fmt.Errorf("failed to create parent folders for output file: %v", err)
-		}
-		outputFile, err := os.Create(cfg.output)
-		if err != nil {
-			return fmt.Errorf("failed to create output file: %v", err)
-		}
-		defer outputFile.Close()
-		outputTo = outputFile
-	}
-
 	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
 		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
 		if !c.IsSet("format") {
@@ -167,219 +165,119 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		}
 	}
 
-	data, err := yaml.Marshal(spec)
-	if err != nil {
-		return fmt.Errorf("failed to marshal CDI spec: %v", err)
+	if err := cdi.ValidateVendorName(cfg.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
 	}
-
-	if strings.ToLower(cfg.format) == formatJSON {
-		data, err = yaml.YAMLToJSONStrict(data)
-		if err != nil {
-			return fmt.Errorf("failed to convert CDI spec from YAML to JSON: %v", err)
-		}
+	if err := cdi.ValidateClassName(cfg.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
 	}
-
-	err = writeToOutput(cfg.format, data, outputTo)
-	if err != nil {
-		return fmt.Errorf("failed to write output: %v", err)
-	}
-
 	return nil
 }
 
+func (m command) run(c *cli.Context, cfg *config) error {
+	spec, err := m.generateSpec(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
+
+	if cfg.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return spec.Save(cfg.output)
+}
+
 func formatFromFilename(filename string) string {
 	ext := filepath.Ext(filename)
 	switch strings.ToLower(ext) {
 	case ".json":
-		return formatJSON
-	case ".yaml":
-		return formatYAML
-	case ".yml":
-		return formatYAML
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
 	}
 
 	return ""
 }
 
-func writeToOutput(format string, data []byte, output io.Writer) error {
-	if format == formatYAML {
-		_, err := output.Write([]byte("---\n"))
-		if err != nil {
-			return fmt.Errorf("failed to write YAML separator: %v", err)
-		}
-	}
-	_, err := output.Write(data)
+func (m command) generateSpec(cfg *config) (spec.Interface, error) {
+	deviceNamer, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
 	if err != nil {
-		return fmt.Errorf("failed to write data: %v", err)
+		return nil, fmt.Errorf("failed to create device namer: %v", err)
 	}
 
-	return nil
-}
-
-func (m command) generateSpec(driverRoot string, nvidiaCTKPath string, namer deviceNamer) (*specs.Spec, error) {
-	nvmllib := nvml.New()
-	if r := nvmllib.Init(); r != nvml.SUCCESS {
-		return nil, r
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(cfg.driverRoot),
+		nvcdi.WithNVIDIACTKPath(cfg.nvidiaCTKPath),
+		nvcdi.WithDeviceNamer(deviceNamer),
+		nvcdi.WithMode(string(cfg.mode)),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
 	}
-	defer nvmllib.Shutdown()
 
-	devicelib := device.New(device.WithNvml(nvmllib))
-
-	deviceSpecs, err := m.generateDeviceSpecs(devicelib, driverRoot, nvidiaCTKPath, namer)
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
 	}
+	var hasAll bool
+	for _, deviceSpec := range deviceSpecs {
+		if deviceSpec.Name == allDeviceName {
+			hasAll = true
+			break
+		}
+	}
+	if !hasAll {
+		allDevice, err := MergeDeviceSpecs(deviceSpecs, allDeviceName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create CDI specification for %q device: %v", allDeviceName, err)
+		}
+		deviceSpecs = append(deviceSpecs, allDevice)
+	}
 
-	allDevice := createAllDevice(deviceSpecs)
-
-	deviceSpecs = append(deviceSpecs, allDevice)
-
-	allEdits := edits.NewContainerEdits()
-
-	ipcs, err := NewIPCDiscoverer(m.logger, driverRoot)
+	commonEdits, err := cdilib.GetCommonEdits()
 	if err != nil {
-		return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
 	}
 
-	ipcEdits, err := edits.FromDiscoverer(ipcs)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create container edits for IPC sockets: %v", err)
-	}
-	// TODO: We should not have to update this after the fact
-	for _, s := range ipcEdits.Mounts {
-		s.Options = append(s.Options, "noexec")
-	}
-
-	allEdits.Append(ipcEdits)
-
-	common, err := NewCommonDiscoverer(m.logger, driverRoot, nvidiaCTKPath, nvmllib)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
-	}
-
-	deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, driverRoot, nvidiaCTKPath, deviceSpecs)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
-	}
-
-	commonEdits, err := edits.FromDiscoverer(discover.Merge(common, deviceFolderPermissionHooks))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create container edits for common entities: %v", err)
-	}
-
-	allEdits.Append(commonEdits)
-
-	// We construct the spec and determine the minimum required version based on the specification.
-	spec := specs.Spec{
-		Version:        "NOT_SET",
-		Kind:           "nvidia.com/gpu",
-		Devices:        deviceSpecs,
-		ContainerEdits: *allEdits.ContainerEdits,
-	}
-
-	minVersion, err := cdi.MinimumRequiredVersion(&spec)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get minumum required CDI spec version: %v", err)
-	}
-	m.logger.Infof("Using minimum required CDI spec version: %s", minVersion)
-
-	spec.Version = minVersion
-
-	return &spec, nil
+	return spec.New(
+		spec.WithVendor(cfg.vendor),
+		spec.WithClass(cfg.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(cfg.format),
+	)
 }
 
-func (m command) generateDeviceSpecs(devicelib device.Interface, driverRoot string, nvidiaCTKPath string, namer deviceNamer) ([]specs.Device, error) {
-	var deviceSpecs []specs.Device
-
-	err := devicelib.VisitDevices(func(i int, d device.Device) error {
-		isMigEnabled, err := d.IsMigEnabled()
-		if err != nil {
-			return fmt.Errorf("failed to check whether device is MIG device: %v", err)
+// MergeDeviceSpecs creates a device with the specified name which combines the edits from the previous devices.
+// If a device of the specified name already exists, an error is returned.
+func MergeDeviceSpecs(deviceSpecs []specs.Device, mergedDeviceName string) (specs.Device, error) {
+	if err := cdi.ValidateDeviceName(mergedDeviceName); err != nil {
+		return specs.Device{}, fmt.Errorf("invalid device name %q: %v", mergedDeviceName, err)
+	}
+	for _, d := range deviceSpecs {
+		if d.Name == mergedDeviceName {
+			return specs.Device{}, fmt.Errorf("device %q already exists", mergedDeviceName)
 		}
-		if isMigEnabled {
-			return nil
-		}
-		device, err := NewFullGPUDiscoverer(m.logger, driverRoot, nvidiaCTKPath, d)
-		if err != nil {
-			return fmt.Errorf("failed to create device: %v", err)
-		}
-
-		deviceEdits, err := edits.FromDiscoverer(device)
-		if err != nil {
-			return fmt.Errorf("failed to create container edits for device: %v", err)
-		}
-
-		deviceName, err := namer.GetDeviceName(i, d)
-		if err != nil {
-			return fmt.Errorf("failed to get device name: %v", err)
-		}
-		deviceSpec := specs.Device{
-			Name:           deviceName,
-			ContainerEdits: *deviceEdits.ContainerEdits,
-		}
-
-		deviceSpecs = append(deviceSpecs, deviceSpec)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate CDI spec for GPU devices: %v", err)
 	}
 
-	err = devicelib.VisitMigDevices(func(i int, d device.Device, j int, mig device.MigDevice) error {
-		device, err := NewMigDeviceDiscoverer(m.logger, "", d, mig)
-		if err != nil {
-			return fmt.Errorf("failed to create MIG device: %v", err)
-		}
-
-		deviceEdits, err := edits.FromDiscoverer(device)
-		if err != nil {
-			return fmt.Errorf("failed to create container edits for MIG device: %v", err)
-		}
-
-		deviceName, err := namer.GetMigDeviceName(i, j, mig)
-		if err != nil {
-			return fmt.Errorf("failed to get device name: %v", err)
-		}
-		deviceSpec := specs.Device{
-			Name:           deviceName,
-			ContainerEdits: *deviceEdits.ContainerEdits,
-		}
-
-		deviceSpecs = append(deviceSpecs, deviceSpec)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("falied to generate CDI spec for MIG devices: %v", err)
-	}
-
-	return deviceSpecs, nil
-}
-
-// createAllDevice creates an 'all' device which combines the edits from the previous devices
-func createAllDevice(deviceSpecs []specs.Device) specs.Device {
-	edits := edits.NewContainerEdits()
+	mergedEdits := edits.NewContainerEdits()
 
 	for _, d := range deviceSpecs {
 		edit := cdi.ContainerEdits{
 			ContainerEdits: &d.ContainerEdits,
 		}
-		edits.Append(&edit)
+		mergedEdits.Append(&edit)
 	}
 
-	all := specs.Device{
-		Name:           "all",
-		ContainerEdits: *edits.ContainerEdits,
+	merged := specs.Device{
+		Name:           mergedDeviceName,
+		ContainerEdits: *mergedEdits.ContainerEdits,
 	}
-	return all
-}
-
-// createParentDirsIfRequired creates the parent folders of the specified path if requried.
-// Note that MkdirAll does not specifically check whether the specified path is non-empty and raises an error if it is.
-// The path will be empty if filename in the current folder is specified, for example
-func createParentDirsIfRequired(filename string) error {
-	dir := filepath.Dir(filename)
-	if dir == "" {
-		return nil
-	}
-	return os.MkdirAll(dir, 0755)
+	return merged, nil
 }

cmd/nvidia-ctk/cdi/generate/generate_test.go

@@ -0,0 +1,117 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeDeviceSpecs(t *testing.T) {
+	testCases := []struct {
+		description      string
+		deviceSpecs      []specs.Device
+		mergedDeviceName string
+		expectedError    error
+		expected         specs.Device
+	}{
+		{
+			description:      "no devices",
+			mergedDeviceName: "all",
+			expected: specs.Device{
+				Name: "all",
+			},
+		},
+		{
+			description:      "one device",
+			mergedDeviceName: "all",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+			},
+			expected: specs.Device{
+				Name: "all",
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"GPU=0"},
+				},
+			},
+		},
+		{
+			description:      "two devices",
+			mergedDeviceName: "all",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+				{
+					Name: "gpu1",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=1"},
+					},
+				},
+			},
+			expected: specs.Device{
+				Name: "all",
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"GPU=0", "GPU=1"},
+				},
+			},
+		},
+		{
+			description:      "has merged device",
+			mergedDeviceName: "gpu0",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+			},
+			expectedError: fmt.Errorf("device %q already exists", "gpu0"),
+		},
+		{
+			description:      "invalid merged device name",
+			mergedDeviceName: ".-not-valid",
+			expectedError:    fmt.Errorf("invalid device name %q", ".-not-valid"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mergedDevice, err := MergeDeviceSpecs(tc.deviceSpecs, tc.mergedDeviceName)
+
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expected, mergedDevice)
+		})
+	}
+}

cmd/nvidia-ctk/cdi/generate/mig-device.go

@@ -1,75 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package generate
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
-	"github.com/sirupsen/logrus"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-)
-
-// NewMigDeviceDiscoverer creates a discoverer for the specified mig device and its parent.
-func NewMigDeviceDiscoverer(logger *logrus.Logger, driverRoot string, parent device.Device, d device.MigDevice) (discover.Discover, error) {
-	minor, ret := parent.GetMinorNumber()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
-	}
-	parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
-
-	migCaps, err := nvcaps.NewMigCaps()
-	if err != nil {
-		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
-	}
-
-	gi, ret := d.GetGpuInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
-	}
-
-	ci, ret := d.GetComputeInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
-	}
-
-	giCap := nvcaps.NewGPUInstanceCap(minor, gi)
-	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
-	}
-
-	ciCap := nvcaps.NewComputeInstanceCap(minor, gi, ci)
-	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
-	}
-
-	deviceNodes := discover.NewCharDeviceDiscoverer(
-		logger,
-		[]string{
-			parentPath,
-			giCapDevicePath,
-			ciCapDevicePath,
-		},
-		driverRoot,
-	)
-
-	return deviceNodes, nil
-}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,159 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type loadSaver interface {
+	Load() (spec.Interface, error)
+	Save(spec.Interface) error
+}
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from string
+	to   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transform.NewRootTransformer(
+		opts.from,
+		opts.to,
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -84,6 +84,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
+	_, err = os.Stat(filepath.Join(containerRoot, "/etc/ld.so.cache"))
+	if err != nil && os.IsNotExist(err) {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		return nil
+	}
+
 	err = m.createConfig(containerRoot, cfg.folders.Value())
 	if err != nil {
 		return fmt.Errorf("failed to update ld.so.conf: %v", err)
@@ -105,6 +111,10 @@ func (m command) createConfig(root string, folders []string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
 	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
 	if err != nil {
 		return fmt.Errorf("failed to create config file: %v", err)

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -22,8 +22,8 @@ import (
 	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -127,13 +127,14 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
 		configFilePath = defaultDockerConfigFilePath
 	}
 
-	cfg, err := docker.LoadConfig(configFilePath)
+	cfg, err := docker.New(
+		docker.WithPath(configFilePath),
+	)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
 	}
 
-	err = docker.UpdateConfig(
-		cfg,
+	err = cfg.AddRuntime(
 		config.nvidiaOptions.RuntimeName,
 		config.nvidiaOptions.RuntimePath,
 		config.nvidiaOptions.SetAsDefault,
@@ -150,12 +151,16 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
 		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
 		return nil
 	}
-	err = docker.FlushConfig(cfg, configFilePath)
+	n, err := cfg.Save(configFilePath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)
 	}
 
-	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	if n == 0 {
+		m.logger.Infof("Removed empty config from %v", configFilePath)
+	} else {
+		m.logger.Infof("Wrote updated config to %v", configFilePath)
+	}
 	m.logger.Infof("It is recommended that the docker daemon be restarted.")
 
 	return nil
@@ -168,13 +173,14 @@ func (m command) configureCrio(c *cli.Context, config *config) error {
 		configFilePath = defaultCrioConfigFilePath
 	}
 
-	cfg, err := crio.LoadConfig(configFilePath)
+	cfg, err := crio.New(
+		crio.WithPath(configFilePath),
+	)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
 	}
 
-	err = crio.UpdateConfig(
-		cfg,
+	err = cfg.AddRuntime(
 		config.nvidiaOptions.RuntimeName,
 		config.nvidiaOptions.RuntimePath,
 		config.nvidiaOptions.SetAsDefault,
@@ -191,12 +197,16 @@ func (m command) configureCrio(c *cli.Context, config *config) error {
 		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
 		return nil
 	}
-	err = crio.FlushConfig(configFilePath, cfg)
+	n, err := cfg.Save(configFilePath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)
 	}
 
-	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	if n == 0 {
+		m.logger.Infof("Removed empty config from %v", configFilePath)
+	} else {
+		m.logger.Infof("Wrote updated config to %v", configFilePath)
+	}
 	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
 
 	return nil

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,107 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type options struct {
+	driverRoot string
+
+	dryRun bool
+
+	control bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device ndoes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	s, err := system.New(
+		system.WithLogger(m.logger),
+		system.WithDryRun(opts.dryRun),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create library: %v", err)
+	}
+
+	if opts.control {
+		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
+		if err := s.CreateNVIDIAControlDeviceNodesAt(opts.driverRoot); err != nil {
+			return fmt.Errorf("failed to create control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -18,6 +18,7 @@ package system
 
 import (
 	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -43,6 +44,7 @@ func (m command) build() *cli.Command {
 
 	system.Subcommands = []*cli.Command{
 		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
 	}
 
 	return &system

docker/docker.mk

@@ -14,10 +14,10 @@
 
 # Supported OSs by architecture
 AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
-X86_64_TARGETS := fedora35 centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
+X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := fedora35 centos8 rhel8 amazonlinux2
+AARCH64_TARGETS := centos8 rhel8 amazonlinux2
 
 # Define top-level build targets
 docker%: SHELL:=/bin/bash
@@ -88,53 +88,31 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
 LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
 
+LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
---ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---ubuntu%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
---ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
---debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---debian%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
---debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
---centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --centos%: CONFIG_TOML_SUFFIX := rpm-yum
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
-# private fedora target
---fedora%: OS := fedora
---fedora%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
---fedora%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---fedora%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---fedora%: CONFIG_TOML_SUFFIX := rpm-yum
-# The fedora(35) base image has very slow performance when building aarch64 packages.
-# Since our primary concern here is glibc versions, we use the older glibc version available in centos8.
---fedora35%: BASEIMAGE = quay.io/centos/centos:stream8
-
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
 --rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
@@ -155,8 +133,8 @@ docker-build-%:
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 	    --build-arg PKG_NAME="$(LIB_NAME)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
+	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
 	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
 	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
 	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82
+	github.com/NVIDIA/go-nvml v0.12.0-0
 	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
 	github.com/fsnotify/fsnotify v1.5.4
 	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
@@ -12,7 +12,7 @@ require (
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
-	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438
 	golang.org/x/mod v0.5.0
 	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
 	sigs.k8s.io/yaml v1.3.0

go.sum

@@ -3,6 +3,8 @@ github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82 h1:x751Xx1tdxkiA/sdkv2J769n21UbYKzVOpe9S/h1M3k=
 github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
+github.com/NVIDIA/go-nvml v0.12.0-0 h1:eHYNHbzAsMgWYshf6dEmTY66/GCXnORJFnzm3TNH4mc=
+github.com/NVIDIA/go-nvml v0.12.0-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
@@ -88,6 +90,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342 h1:083n9fJt2dWOpJd/X/q9Xgl5XtQLL22uSFYbzVqJssg=
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438 h1:+qRai7XRl8omFQVCeHcaWzL542Yw64vfmuXG+79ZCIc=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
 golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=

internal/config/config.go

@@ -45,9 +45,12 @@ var (
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
-	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+	AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+
+	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file
@@ -91,6 +94,8 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 		return cfg, nil
 	}
 
+	cfg.AcceptEnvvarUnprivileged = toml.GetDefault("accept-nvidia-visible-devices-envvar-when-unprivileged", cfg.AcceptEnvvarUnprivileged).(bool)
+
 	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
 	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
 	runtimeConfig, err := getRuntimeConfigFrom(toml)
@@ -99,12 +104,19 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 	}
 	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
 
+	runtimeHookConfig, err := getRuntimeHookConfigFrom(toml)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load nvidia-container-runtime-hook config: %v", err)
+	}
+	cfg.NVIDIAContainerRuntimeHookConfig = *runtimeHookConfig
+
 	return cfg, nil
 }
 
 // getDefaultConfig defines the default values for the config
 func getDefaultConfig() *Config {
 	c := Config{
+		AcceptEnvvarUnprivileged:     true,
 		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
 		NVIDIACTKConfig:              *getDefaultCTKConfig(),
 		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),

internal/config/config_test.go

@@ -57,6 +57,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "empty config is default",
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: true,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "",
 				},
@@ -69,6 +70,10 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{
@@ -79,6 +84,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set inline",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"nvidia-container-cli.root = \"/bar/baz\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
 				"nvidia-container-runtime.experimental = true",
@@ -86,10 +92,13 @@ func TestGetConfig(t *testing.T) {
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
+				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "/bar/baz",
 				},
@@ -102,6 +111,13 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{
@@ -112,6 +128,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set in section",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
 				"[nvidia-container-runtime]",
@@ -121,12 +138,16 @@ func TestGetConfig(t *testing.T) {
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
 				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.cdi]",
+				"default-kind = \"example.vendor.com/device\"",
+				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "/bar/baz",
 				},
@@ -139,6 +160,13 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{

internal/config/crio/crio.go

@@ -1,125 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package crio
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pelletier/go-toml"
-	log "github.com/sirupsen/logrus"
-)
-
-// LoadConfig loads the cri-o config from disk
-func LoadConfig(config string) (*toml.Tree, error) {
-	log.Infof("Loading config: %v", config)
-
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	configFile := config
-	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		log.Infof("Config file does not exist, creating new one")
-	}
-
-	cfg, err := toml.LoadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-
-	return cfg, nil
-}
-
-// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
-func UpdateConfig(config *toml.Tree, runtimeClass string, runtimePath string, setAsDefault bool) error {
-	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
-	case *toml.Tree:
-		runc, _ = toml.Load(runc.String())
-		config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass}, runc)
-	}
-
-	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_path"}, runtimePath)
-	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_type"}, "oci")
-
-	if setAsDefault {
-		config.SetPath([]string{"crio", "runtime", "default_runtime"}, runtimeClass)
-	}
-
-	return nil
-}
-
-// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
-func RevertConfig(config *toml.Tree, runtimeClass string) error {
-	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
-		if runtimeClass == runtime {
-			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
-		}
-	}
-
-	runtimeClassPath := []string{"crio", "runtime", "runtimes", runtimeClass}
-	config.DeletePath(runtimeClassPath)
-	for i := 0; i < len(runtimeClassPath); i++ {
-		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
-		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
-			if len(entry.Keys()) != 0 {
-				break
-			}
-			config.DeletePath(remainingPath)
-		}
-	}
-
-	return nil
-}
-
-// FlushConfig flushes the updated/reverted config out to disk
-func FlushConfig(config string, cfg *toml.Tree) error {
-	log.Infof("Flushing config")
-
-	output, err := cfg.ToTomlString()
-	if err != nil {
-		return fmt.Errorf("unable to convert to TOML: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(config)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(config)
-		if err != nil {
-			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(output)
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
-}

internal/config/docker/docker.go

@@ -1,117 +0,0 @@
-/**
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package docker
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// LoadConfig loads the docker config from disk
-func LoadConfig(configFilePath string) (map[string]interface{}, error) {
-	log.Infof("Loading docker config from %v", configFilePath)
-
-	info, err := os.Stat(configFilePath)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	cfg := make(map[string]interface{})
-
-	if os.IsNotExist(err) {
-		log.Infof("Config file does not exist, creating new one")
-		return cfg, nil
-	}
-
-	readBytes, err := ioutil.ReadFile(configFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("unable to read config: %v", err)
-	}
-
-	reader := bytes.NewReader(readBytes)
-	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-	return cfg, nil
-}
-
-// UpdateConfig updates the docker config to include the nvidia runtimes
-func UpdateConfig(config map[string]interface{}, runtimeName string, runtimePath string, setAsDefault bool) error {
-	// Read the existing runtimes
-	runtimes := make(map[string]interface{})
-	if _, exists := config["runtimes"]; exists {
-		runtimes = config["runtimes"].(map[string]interface{})
-	}
-
-	// Add / update the runtime definitions
-	runtimes[runtimeName] = map[string]interface{}{
-		"path": runtimePath,
-		"args": []string{},
-	}
-
-	// Update the runtimes definition
-	if len(runtimes) > 0 {
-		config["runtimes"] = runtimes
-	}
-
-	if setAsDefault {
-		config["default-runtime"] = runtimeName
-	}
-
-	return nil
-}
-
-// FlushConfig flushes the updated/reverted config out to disk
-func FlushConfig(cfg map[string]interface{}, configFilePath string) error {
-	log.Infof("Flushing docker config to %v", configFilePath)
-
-	output, err := json.MarshalIndent(cfg, "", "    ")
-	if err != nil {
-		return fmt.Errorf("unable to convert to JSON: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(configFilePath)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(configFilePath)
-		if err != nil {
-			return fmt.Errorf("unable to open %v for writing: %v", configFilePath, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(string(output))
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
-}

internal/config/engine/api.go

@@ -0,0 +1,25 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package engine
+
+// Interface defines the API for a runtime config updater.
+type Interface interface {
+	DefaultRuntime() string
+	AddRuntime(string, string, bool) error
+	RemoveRuntime(string) error
+	Save(string) (int64, error)
+}

internal/config/engine/containerd/config_v1.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// ConfigV1 represents a version 1 containerd config
+type ConfigV1 Config
+
+var _ engine.Interface = (*ConfigV1)(nil)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := *c.Tree
+
+	config.Set("version", int64(1))
+
+	switch runc := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", "runc"}).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, runc)
+	}
+
+	if config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name}) == nil {
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := (*Config)(c).getRuntimeAnnotations([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
+	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}, path)
+
+	if setAsDefault && c.UseDefaultRuntimeName {
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
+	} else if setAsDefault {
+		// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
+		if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
+		}
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c ConfigV1) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *ConfigV1) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	// If the specified runtime was set as the default runtime we need to remove the default runtime too.
+	runtimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}).(string)
+	if !ok || runtimePath == "" {
+		runtimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}).(string)
+	}
+	defaultRuntimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}).(string)
+	if !ok || defaultRuntimePath == "" {
+		defaultRuntimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}).(string)
+	}
+	if runtimePath != "" && defaultRuntimePath != "" && runtimePath == defaultRuntimePath {
+		config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime"})
+	}
+
+	config.DeletePath([]string{"plugins", "cri", "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimeConfigPath := []string{"plugins", "cri", "containerd", "runtimes", name}
+	for i := 0; i < len(runtimeConfigPath); i++ {
+		if runtimes, ok := config.GetPath(runtimeConfigPath[:len(runtimeConfigPath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimeConfigPath[:len(runtimeConfigPath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// Save wrotes the config to a file
+func (c ConfigV1) Save(path string) (int64, error) {
+	return (Config)(c).Save(path)
+}

internal/config/engine/containerd/config_v2.go

@@ -0,0 +1,161 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+	config := *c.Tree
+
+	config.Set("version", int64(2))
+
+	switch runc := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc"}).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, runc)
+	}
+
+	if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := c.getRuntimeAnnotations([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
+
+	if setAsDefault {
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
+	if c == nil || c.Tree == nil {
+		return nil, nil
+	}
+
+	config := *c.Tree
+	if !config.HasPath(path) {
+		return nil, nil
+	}
+	annotationsI, ok := config.GetPath(path).([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("invalid annotations: %v", annotationsI)
+	}
+
+	var annotations []string
+	for _, annotation := range annotationsI {
+		a, ok := annotation.(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid annotation: %v", annotation)
+		}
+		annotations = append(annotations, a)
+	}
+
+	return annotations, nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
+	for i := 0; i < len(runtimePath); i++ {
+		if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimePath[:len(runtimePath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	config := c.Tree
+	output, err := config.ToTomlString()
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(output)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), err
+}

internal/config/engine/containerd/containerd.go

@@ -0,0 +1,40 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// Config represents the containerd config
+type Config struct {
+	*toml.Tree
+	RuntimeType           string
+	UseDefaultRuntimeName bool
+	ContainerAnnotations  []string
+}
+
+// New creates a containerd config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}

internal/config/engine/containerd/option.go

@@ -0,0 +1,149 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultRuntimeType = "io.containerd.runc.v2"
+)
+
+type builder struct {
+	path                 string
+	runtimeType          string
+	useLegacyConfig      bool
+	containerAnnotations []string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+// WithRuntimeType sets the runtime type for the config builder
+func WithRuntimeType(runtimeType string) Option {
+	return func(b *builder) {
+		b.runtimeType = runtimeType
+	}
+}
+
+// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
+func WithUseLegacyConfig(useLegacyConfig bool) Option {
+	return func(b *builder) {
+		b.useLegacyConfig = useLegacyConfig
+	}
+}
+
+// WithContainerAnnotations sets the container annotations for the config builder
+func WithContainerAnnotations(containerAnnotations ...string) Option {
+	return func(b *builder) {
+		b.containerAnnotations = containerAnnotations
+	}
+}
+
+func (b *builder) build() (engine.Interface, error) {
+	if b.path == "" {
+		return nil, fmt.Errorf("config path is empty")
+	}
+
+	if b.runtimeType == "" {
+		b.runtimeType = defaultRuntimeType
+	}
+
+	config, err := loadConfig(b.path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config.RuntimeType = b.runtimeType
+	config.UseDefaultRuntimeName = !b.useLegacyConfig
+	config.ContainerAnnotations = b.containerAnnotations
+
+	version, err := config.parseVersion(b.useLegacyConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config version: %v", err)
+	}
+	switch version {
+	case 1:
+		return (*ConfigV1)(config), nil
+	case 2:
+		return config, nil
+	}
+
+	return nil, fmt.Errorf("unsupported config version: %v", version)
+}
+
+// loadConfig loads the containerd config from disk
+func loadConfig(config string) (*Config, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	tomlConfig, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	cfg := Config{
+		Tree: tomlConfig,
+	}
+	return &cfg, nil
+}
+
+// parseVersion returns the version of the config
+func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
+	defaultVersion := 2
+	if useLegacyConfig {
+		defaultVersion = 1
+	}
+
+	switch v := c.Get("version").(type) {
+	case nil:
+		switch len(c.Keys()) {
+		case 0: // No config exists, or the config file is empty, use version inferred from containerd
+			return defaultVersion, nil
+		default: // A config file exists, has content, and no version is set
+			return 1, nil
+		}
+	case int64:
+		return int(v), nil
+	default:
+		return -1, fmt.Errorf("unsupported type for version field: %v", v)
+	}
+}

internal/config/engine/crio/crio.go

@@ -0,0 +1,131 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// Config represents the cri-o config
+type Config toml.Tree
+
+// New creates a cri-o config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}
+
+// AddRuntime adds a new runtime to the crio config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := (toml.Tree)(*c)
+
+	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"crio", "runtime", "runtimes", name}, runc)
+	}
+
+	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_path"}, path)
+	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_type"}, "oci")
+
+	if setAsDefault {
+		config.SetPath([]string{"crio", "runtime", "default_runtime"}, name)
+	}
+
+	*c = (Config)(config)
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	config := (toml.Tree)(c)
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the cri-o config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil {
+		return nil
+	}
+
+	config := (toml.Tree)(*c)
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
+		}
+	}
+
+	runtimeClassPath := []string{"crio", "runtime", "runtimes", name}
+	config.DeletePath(runtimeClassPath)
+	for i := 0; i < len(runtimeClassPath); i++ {
+		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
+		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
+			if len(entry.Keys()) != 0 {
+				break
+			}
+			config.DeletePath(remainingPath)
+		}
+	}
+
+	*c = (Config)(config)
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	config := (toml.Tree)(c)
+	output, err := config.ToTomlString()
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(output)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), err
+}

internal/config/engine/crio/option.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+type builder struct {
+	path string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+func (b *builder) build() (*Config, error) {
+	if b.path == "" {
+		empty := toml.Tree{}
+		return (*Config)(&empty), nil
+	}
+
+	return loadConfig(b.path)
+}
+
+// loadConfig loads the cri-o config from disk
+func loadConfig(config string) (*Config, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return (*Config)(cfg), nil
+}

internal/config/engine/docker/docker.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+)
+
+const (
+	defaultDockerRuntime = "runc"
+)
+
+// Config defines a docker config file.
+// TODO: This should not be public, but we need to access it from the tests in tools/container/docker
+type Config map[string]interface{}
+
+// New creates a docker config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}
+
+// AddRuntime adds a new runtime to the docker config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := *c
+
+	// Read the existing runtimes
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	// Add / update the runtime definitions
+	runtimes[name] = map[string]interface{}{
+		"path": path,
+		"args": []string{},
+	}
+
+	config["runtimes"] = runtimes
+
+	if setAsDefault {
+		config["default-runtime"] = name
+	}
+
+	*c = config
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the docker config
+func (c Config) DefaultRuntime() string {
+	r, ok := c["default-runtime"].(string)
+	if !ok {
+		return ""
+	}
+	return r
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil {
+		return nil
+	}
+	config := *c
+
+	if _, exists := config["default-runtime"]; exists {
+		defaultRuntime := config["default-runtime"].(string)
+		if defaultRuntime == name {
+			config["default-runtime"] = defaultDockerRuntime
+		}
+	}
+
+	if _, exists := config["runtimes"]; exists {
+		runtimes := config["runtimes"].(map[string]interface{})
+
+		delete(runtimes, name)
+
+		if len(runtimes) == 0 {
+			delete(config, "runtimes")
+		}
+	}
+
+	*c = config
+
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	output, err := json.MarshalIndent(c, "", "    ")
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(string(output))
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), nil
+}

internal/config/engine/docker/docker_test.go

@@ -26,7 +26,7 @@ import (
 
 func TestUpdateConfigDefaultRuntime(t *testing.T) {
 	testCases := []struct {
-		config                     map[string]interface{}
+		config                     Config
 		runtimeName                string
 		setAsDefault               bool
 		expectedDefaultRuntimeName interface{}
@@ -63,7 +63,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			if tc.config == nil {
 				tc.config = make(map[string]interface{})
 			}
-			err := UpdateConfig(tc.config, tc.runtimeName, "", tc.setAsDefault)
+			err := tc.config.AddRuntime(tc.runtimeName, "", tc.setAsDefault)
 			require.NoError(t, err)
 
 			defaultRuntimeName := tc.config["default-runtime"]
@@ -74,7 +74,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 
 func TestUpdateConfigRuntimes(t *testing.T) {
 	testCases := []struct {
-		config         map[string]interface{}
+		config         Config
 		runtimes       map[string]string
 		expectedConfig map[string]interface{}
 	}{
@@ -198,7 +198,7 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
 			for runtimeName, runtimePath := range tc.runtimes {
-				err := UpdateConfig(tc.config, runtimeName, runtimePath, false)
+				err := tc.config.AddRuntime(runtimeName, runtimePath, false)
 				require.NoError(t, err)
 			}
 

internal/config/engine/docker/option.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type builder struct {
+	path string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+func (b *builder) build() (*Config, error) {
+	if b.path == "" {
+		empty := make(Config)
+		return &empty, nil
+	}
+
+	return loadConfig(b.path)
+}
+
+// loadConfig loads the docker config from disk
+func loadConfig(configFilePath string) (*Config, error) {
+	log.Infof("Loading docker config from %v", configFilePath)
+
+	info, err := os.Stat(configFilePath)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(Config)
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return &cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return &cfg, nil
+}

internal/config/hook.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"fmt"
+
+	"github.com/pelletier/go-toml"
+)
+
+// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeHookConfig struct {
+	// SkipModeDetection disables the mode check for the runtime hook.
+	SkipModeDetection bool `toml:"skip-mode-detection"`
+}
+
+// dummyHookConfig allows us to unmarshal only a RuntimeHookConfig from a *toml.Tree
+type dummyHookConfig struct {
+	RuntimeHook RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
+}
+
+// getRuntimeHookConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getRuntimeHookConfigFrom(toml *toml.Tree) (*RuntimeHookConfig, error) {
+	cfg := GetDefaultRuntimeHookConfig()
+
+	if toml == nil {
+		return cfg, nil
+	}
+
+	d := dummyHookConfig{
+		RuntimeHook: *cfg,
+	}
+
+	if err := toml.Unmarshal(&d); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
+	}
+
+	return &d.RuntimeHook, nil
+}
+
+// GetDefaultRuntimeHookConfig defines the default values for the config
+func GetDefaultRuntimeHookConfig() *RuntimeHookConfig {
+	c := RuntimeHookConfig{
+		SkipModeDetection: false,
+	}
+
+	return &c
+}

internal/config/image/privileged.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged NVIDIA capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

@@ -19,6 +19,7 @@ package config
 import (
 	"fmt"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 )
@@ -50,6 +51,10 @@ type modesConfig struct {
 type cdiModeConfig struct {
 	// SpecDirs allows for the default spec dirs for CDI to be overridden
 	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
 }
 
 type csvModeConfig struct {
@@ -94,6 +99,12 @@ func GetDefaultRuntimeConfig() *RuntimeConfig {
 			CSV: csvModeConfig{
 				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 			},
+			CDI: cdiModeConfig{
+				DefaultKind: "nvidia.com/gpu",
+				AnnotationPrefixes: []string{
+					cdi.AnnotationPrefix,
+				},
+			},
 		},
 	}
 

internal/discover/char_devices.go

@@ -58,7 +58,11 @@ func (d *charDevices) Devices() ([]Device, error) {
 	}
 	var devices []Device
 	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
 	}
 
 	return devices, nil

internal/discover/discover.go

@@ -32,6 +32,7 @@ type Device struct {
 type Mount struct {
 	HostPath string
 	Path     string
+	Options  []string
 }
 
 // Hook represents a discovered hook.

internal/discover/graphics.go

@@ -20,11 +20,13 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
 	"github.com/sirupsen/logrus"
 )
 
@@ -44,9 +46,12 @@ func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices,
 
 	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
 
+	xorg := optionalXorgDiscoverer(logger, driverRoot, cfg.NvidiaCTKPath)
+
 	discover := Merge(
 		Merge(drmDeviceNodes, drmByPathSymlinks),
 		mounts,
+		xorg,
 	)
 
 	return discover, nil
@@ -243,6 +248,123 @@ func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, dri
 	return filter, nil
 }
 
+type xorgHooks struct {
+	libraries     Discover
+	driverVersion string
+	nvidiaCTKPath string
+}
+
+var _ Discover = (*xorgHooks)(nil)
+
+// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
+// If the creation of the discoverer fails, a None discoverer is returned.
+func optionalXorgDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		logger.Warnf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
+		return None{}
+	}
+	return xorg
+}
+
+func newXorgDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate(".*.*.*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+	libcudaPath := libCudaPaths[0]
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	libRoot := filepath.Dir(libcudaPath)
+	xorgLibs := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"nvidia/xorg/nvidia_drv.so",
+			fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
+		},
+	)
+	xorgHooks := xorgHooks{
+		libraries:     xorgLibs,
+		driverVersion: version,
+		nvidiaCTKPath: FindNvidiaCTK(logger, nvidiaCTKPath),
+	}
+
+	xorgConfg := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/usr/share"),
+		),
+		driverRoot,
+		[]string{"X11/xorg.conf.d/10-nvidia.conf"},
+	)
+
+	d := Merge(
+		xorgLibs,
+		xorgConfg,
+		xorgHooks,
+	)
+
+	return d, nil
+}
+
+// Devices returns no devices for Xorg
+func (m xorgHooks) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a hook to create symlinks for Xorg libraries
+func (m xorgHooks) Hooks() ([]Hook, error) {
+	mounts, err := m.libraries.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get mounts: %v", err)
+	}
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+
+	var target string
+	for _, mount := range mounts {
+		filename := filepath.Base(mount.HostPath)
+		if filename == "libglxserver_nvidia.so."+m.driverVersion {
+			target = mount.Path
+		}
+	}
+
+	if target == "" {
+		return nil, nil
+	}
+
+	link := strings.TrimSuffix(target, "."+m.driverVersion)
+	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
+	symlinkHook := CreateCreateSymlinkHook(
+		m.nvidiaCTKPath,
+		links,
+	)
+
+	return symlinkHook.Hooks()
+}
+
+// Mounts returns the libraries required for Xorg
+func (m xorgHooks) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
 // selectDeviceByPath is a filter that allows devices to be selected by the path
 type selectDeviceByPath map[string]bool
 

internal/discover/hooks.go

@@ -29,12 +29,47 @@ const (
 	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
 )
 
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+// CreateCreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateCreateSymlinkHook(nvidiaCTKPath string, links []string) Discover {
+	if len(links) == 0 {
+		return None{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateNvidiaCTKHook(
+		nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+}
+
 // CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
-func CreateNvidiaCTKHook(executable string, hookName string, additionalArgs ...string) Hook {
+func CreateNvidiaCTKHook(nvidiaCTKPath string, hookName string, additionalArgs ...string) Hook {
 	return Hook{
 		Lifecycle: cdi.CreateContainerHook,
-		Path:      executable,
-		Args:      append([]string{filepath.Base(executable), "hook", hookName}, additionalArgs...),
+		Path:      nvidiaCTKPath,
+		Args:      append([]string{filepath.Base(nvidiaCTKPath), "hook", hookName}, additionalArgs...),
 	}
 }
 

internal/discover/icp_test.go

@@ -0,0 +1,60 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIPCMounts(t *testing.T) {
+	l := ipcMounts(
+		mounts{
+			logger: logrus.New(),
+			lookup: &lookup.LocatorMock{
+				LocateFunc: func(path string) ([]string, error) {
+					return []string{"/host/path"}, nil
+				},
+			},
+			required: []string{"target"},
+		},
+	)
+
+	mounts, err := l.Mounts()
+	require.NoError(t, err)
+
+	require.EqualValues(
+		t,
+		[]Mount{
+			{
+				HostPath: "/host/path",
+				Path:     "/host/path",
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+					"noexec",
+				},
+			},
+		},
+		mounts,
+	)
+}

internal/discover/ipc.go

@@ -0,0 +1,78 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type ipcMounts mounts
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger *logrus.Logger, driverRoot string) (Discover, error) {
+	sockets := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/run", "/var/run"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/nvidia-persistenced/socket",
+			"/nvidia-fabricmanager/socket",
+		},
+	)
+
+	mps := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	d := Merge(
+		(*ipcMounts)(sockets),
+		(*ipcMounts)(mps),
+	)
+	return d, nil
+}
+
+// Mounts returns the discovered mounts with "noexec" added to the mount options.
+func (d *ipcMounts) Mounts() ([]Mount, error) {
+	mounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+
+	var modifiedMounts []Mount
+	for _, m := range mounts {
+		mount := m
+		mount.Options = append(m.Options, "noexec")
+		modifiedMounts = append(modifiedMounts, mount)
+	}
+
+	return modifiedMounts, nil
+}

internal/discover/mounts.go

@@ -43,6 +43,11 @@ var _ Discover = (*mounts)(nil)
 
 // NewMounts creates a discoverer for the required mounts using the specified locator.
 func NewMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) Discover {
+	return newMounts(logger, lookup, root, required)
+}
+
+// newMounts creates a discoverer for the required mounts using the specified locator.
+func newMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) *mounts {
 	return &mounts{
 		logger:   logger,
 		lookup:   lookup,
@@ -93,6 +98,12 @@ func (d *mounts) Mounts() ([]Mount, error) {
 			uniqueMounts[p] = Mount{
 				HostPath: p,
 				Path:     r,
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+				},
 			}
 		}
 	}

internal/discover/mounts_test.go

@@ -35,6 +35,14 @@ func TestMountsReturnsEmptyDevices(t *testing.T) {
 }
 
 func TestMounts(t *testing.T) {
+
+	mountOptions := []string{
+		"ro",
+		"nosuid",
+		"nodev",
+		"bind",
+	}
+
 	logger, logHook := testlog.NewNullLogger()
 
 	testCases := []struct {
@@ -70,7 +78,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required"},
 			},
-			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description:   "mounts removes located duplicates",
@@ -83,7 +91,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips located errors",
@@ -98,7 +106,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "error", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips unlocated",
@@ -113,7 +121,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "empty", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts adds multiple",
@@ -129,10 +137,10 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "required0", HostPath: "required0"},
-				{Path: "multiple0", HostPath: "multiple0"},
-				{Path: "multiple1", HostPath: "multiple1"},
-				{Path: "required1", HostPath: "required1"},
+				{Path: "required0", HostPath: "required0", Options: mountOptions},
+				{Path: "multiple0", HostPath: "multiple0", Options: mountOptions},
+				{Path: "multiple1", HostPath: "multiple1", Options: mountOptions},
+				{Path: "required1", HostPath: "required1", Options: mountOptions},
 			},
 		},
 		{
@@ -147,7 +155,7 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "/located", HostPath: "/some/root/located"},
+				{Path: "/located", HostPath: "/some/root/located", Options: mountOptions},
 			},
 		},
 	}

internal/dxcore/api.go

@@ -0,0 +1,64 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+import (
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+const (
+	libraryName      = "libdxcore.so"
+	libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+// dxcore stores a reference the dxcore dynamic library
+var dxcore *context
+
+// Init initializes the dxcore dynamic library
+func Init() error {
+	c, err := initContext()
+	if err != nil {
+		return err
+	}
+	dxcore = c
+	return nil
+}
+
+// Shutdown closes the dxcore dynamic library
+func Shutdown() error {
+	if dxcore != nil && dxcore.initialized != 0 {
+		dxcore.deinitContext()
+	}
+	return nil
+}
+
+// GetDriverStorePaths returns the list of driver store paths
+func GetDriverStorePaths() []string {
+	var paths []string
+	selected := make(map[string]bool)
+
+	for i := 0; i < dxcore.getAdapterCount(); i++ {
+		path := dxcore.getAdapter(i).getDriverStorePath()
+		if selected[path] {
+			continue
+		}
+		selected[path] = true
+		paths = append(paths, path)
+	}
+
+	return paths
+}

internal/dxcore/dxcore.c

@@ -0,0 +1,334 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include "dxcore.h"
+
+// We define log_write as an empty macro to allow dxcore to remain unchanged.
+#define log_write(...)
+
+// We define the following macros to allow dxcore to remain largely unchanged.
+#define log_info(msg) log_write('I', __FILE__, __LINE__, msg)
+#define log_warn(msg) log_write('W', __FILE__, __LINE__, msg)
+#define log_err(msg)  log_write('E', __FILE__, __LINE__, msg)
+#define log_infof(fmt, ...) log_write('I', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_warnf(fmt, ...) log_write('W', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_errf(fmt, ...)  log_write('E', __FILE__, __LINE__, fmt, __VA_ARGS__)
+
+
+#define DXCORE_MAX_PATH 260
+
+/*
+ * List of components we expect to find in the driver store that we need to mount
+ */
+static const char * const dxcore_nvidia_driver_store_components[] = {
+        "libcuda.so.1.1",                   /* Core library for cuda support */
+        "libcuda_loader.so",                /* Core library for cuda support on WSL */
+        "libnvidia-ptxjitcompiler.so.1",    /* Core library for PTX Jit support */
+        "libnvidia-ml.so.1",                /* Core library for nvml */
+        "libnvidia-ml_loader.so",           /* Core library for nvml on WSL */
+        "nvidia-smi",                       /* nvidia-smi binary*/
+        "nvcubins.bin",                     /* Binary containing GPU code for cuda */
+};
+
+
+/*
+ * List of functions and structures we need to communicate with libdxcore.
+ * Documentation on these functions can be found on docs.microsoft.com in d3dkmthk.
+ */
+
+struct dxcore_enumAdapters2;
+struct dxcore_queryAdapterInfo;
+
+typedef int(*pfnDxcoreEnumAdapters2)(struct dxcore_enumAdapters2* pParams);
+typedef int(*pfnDxcoreQueryAdapterInfo)(struct dxcore_queryAdapterInfo* pParams);
+
+struct dxcore_lib {
+        void* hDxcoreLib;
+        pfnDxcoreEnumAdapters2 pDxcoreEnumAdapters2;
+        pfnDxcoreQueryAdapterInfo pDxcoreQueryAdapterInfo;
+};
+
+struct dxcore_adapterInfo
+{
+        unsigned int              hAdapter;
+        struct dxcore_luid        AdapterLuid;
+        unsigned int              NumOfSources;
+        unsigned int              bPresentMoveRegionsPreferred;
+};
+
+struct dxcore_enumAdapters2
+{
+        unsigned int                   NumAdapters;
+        struct dxcore_adapterInfo     *pAdapters;
+};
+
+enum dxcore_kmtqueryAdapterInfoType
+{
+        DXCORE_QUERYDRIVERVERSION = 13,
+        DXCORE_QUERYREGISTRY = 48,
+};
+
+enum dxcore_queryregistry_type {
+        DXCORE_QUERYREGISTRY_DRIVERSTOREPATH = 2,
+        DXCORE_QUERYREGISTRY_DRIVERIMAGEPATH = 3,
+};
+
+enum dxcore_queryregistry_status {
+        DXCORE_QUERYREGISTRY_STATUS_SUCCESS = 0,
+        DXCORE_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW = 1,
+        DXCORE_QUERYREGISTRY_STATUS_FAIL = 2,
+};
+
+struct dxcore_queryregistry_info {
+        enum dxcore_queryregistry_type        QueryType;
+        unsigned int                          QueryFlags;
+        wchar_t                               ValueName[DXCORE_MAX_PATH];
+        unsigned int                          ValueType;
+        unsigned int                          PhysicalAdapterIndex;
+        unsigned int                          OutputValueSize;
+        enum dxcore_queryregistry_status      Status;
+        union {
+                unsigned long long                    OutputQword;
+                wchar_t                               Output;
+        };
+};
+
+struct dxcore_queryAdapterInfo
+{
+        unsigned int                           hAdapter;
+        enum dxcore_kmtqueryAdapterInfoType    Type;
+        void                                   *pPrivateDriverData;
+        unsigned int                           PrivateDriverDataSize;
+};
+
+static int dxcore_query_adapter_info_helper(struct dxcore_lib* pLib,
+                                            unsigned int hAdapter,
+                                            enum dxcore_kmtqueryAdapterInfoType type,
+                                            void* pPrivateDriverDate,
+                                            unsigned int privateDriverDataSize)
+{
+        struct dxcore_queryAdapterInfo queryAdapterInfo = { 0 };
+
+        queryAdapterInfo.hAdapter = hAdapter;
+        queryAdapterInfo.Type = type;
+        queryAdapterInfo.pPrivateDriverData = pPrivateDriverDate;
+        queryAdapterInfo.PrivateDriverDataSize = privateDriverDataSize;
+
+        return pLib->pDxcoreQueryAdapterInfo(&queryAdapterInfo);
+}
+
+static int dxcore_query_adapter_wddm_version(struct dxcore_lib* pLib, unsigned int hAdapter, unsigned int* version)
+{
+        return dxcore_query_adapter_info_helper(pLib,
+                                                hAdapter,
+                                                DXCORE_QUERYDRIVERVERSION,
+                                                (void*)version,
+                                                sizeof(*version));
+}
+
+static int dxcore_query_adapter_driverstore(struct dxcore_lib* pLib, unsigned int hAdapter, char** ppDriverStorePath)
+{
+        struct dxcore_queryregistry_info params = {0};
+        struct dxcore_queryregistry_info* pValue = NULL;
+        wchar_t* pOutput;
+        size_t outputSizeInBytes;
+        size_t outputSize;
+
+        params.QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                             hAdapter,
+                                             DXCORE_QUERYREGISTRY,
+                                             (void*)&params,
+                                             sizeof(params)))
+        {
+                log_err("Failed to query driver store path size for the WDDM Adapter");
+                return (-1);
+        }
+
+        if (params.OutputValueSize > DXCORE_MAX_PATH * sizeof(wchar_t)) {
+                log_err("The driver store path size returned by dxcore is not valid");
+                return (-1);
+        }
+
+        outputSizeInBytes = (size_t)params.OutputValueSize;
+        outputSize = outputSizeInBytes / sizeof(wchar_t);
+
+        pValue = calloc(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes + sizeof(wchar_t), 1);
+        if (!pValue) {
+                log_err("Out of memory while allocating temp buffer to query adapter info");
+                return (-1);
+        }
+
+        pValue->QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+        pValue->OutputValueSize = (unsigned int)outputSizeInBytes;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                            hAdapter,
+                                            DXCORE_QUERYREGISTRY,
+                                            (void*)pValue,
+                                            (unsigned int)(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes)))
+        {
+                log_err("Failed to query driver store path data for the WDDM Adapter");
+                free(pValue);
+                return (-1);
+        }
+        pOutput = (wchar_t*)(&pValue->Output);
+
+        // Make sure no matter what happened the wchar_t string is null terminated
+        pOutput[outputSize] = L'\0';
+
+        // Convert the output into a regular c string
+        *ppDriverStorePath = (char*)calloc(outputSize + 1, sizeof(char));
+        if (!*ppDriverStorePath) {
+                log_err("Out of memory while allocating the buffer for the driver store path");
+                free(pValue);
+                return (-1);
+        }
+        wcstombs(*ppDriverStorePath, pOutput, outputSize);
+
+        free(pValue);
+
+    return 0;
+}
+
+static void dxcore_add_adapter(struct dxcore_context* pCtx, struct dxcore_lib* pLib, struct dxcore_adapterInfo *pAdapterInfo)
+{
+        unsigned int wddmVersion = 0;
+        char* driverStorePath = NULL;
+
+        log_infof("Creating a new WDDM Adapter for hAdapter:%x luid:%llx", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid));
+
+        if (dxcore_query_adapter_wddm_version(pLib, pAdapterInfo->hAdapter, &wddmVersion)) {
+                log_err("Failed to query the WDDM version for the specified adapter. Skipping it.");
+                return;
+        }
+
+        if (wddmVersion < 2700) {
+                log_err("Found a WDDM adapter running a driver with pre-WDDM 2.7 . Skipping it.");
+                return;
+        }
+
+        if (dxcore_query_adapter_driverstore(pLib, pAdapterInfo->hAdapter, &driverStorePath)) {
+                log_err("Failed to query driver store path for the WDDM Adapter . Skipping it.");
+                return;
+        }
+
+        // We got all the info we needed. Adding it to the tracking structure.
+        {
+                struct dxcore_adapter* newList;
+                newList = realloc(pCtx->adapterList, sizeof(struct dxcore_adapter) * (pCtx->adapterCount + 1));
+                if (!newList) {
+                        log_err("Out of memory when trying to add a new WDDM Adapter to the list of valid adapters");
+                        free(driverStorePath);
+                        return;
+                }
+
+                pCtx->adapterList = newList;
+
+                pCtx->adapterList[pCtx->adapterCount].hAdapter = pAdapterInfo->hAdapter;
+                pCtx->adapterList[pCtx->adapterCount].pDriverStorePath = driverStorePath;
+                pCtx->adapterList[pCtx->adapterCount].wddmVersion = wddmVersion;
+                pCtx->adapterCount++;
+        }
+
+        log_infof("Adding new adapter via dxcore hAdapter:%x luid:%llx wddm version:%d", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid), wddmVersion);
+}
+
+static void dxcore_enum_adapters(struct dxcore_context* pCtx, struct dxcore_lib* pLib)
+{
+        struct dxcore_enumAdapters2 params = {0};
+        unsigned int adapterIndex = 0;
+
+        params.NumAdapters = 0;
+        params.pAdapters = NULL;
+
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        params.pAdapters = malloc(sizeof(struct dxcore_adapterInfo) * params.NumAdapters);
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                free(params.pAdapters);
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        for (adapterIndex = 0; adapterIndex < params.NumAdapters; adapterIndex++) {
+                dxcore_add_adapter(pCtx, pLib, &params.pAdapters[adapterIndex]);
+        }
+
+        free(params.pAdapters);
+}
+
+int dxcore_init_context(struct dxcore_context* pCtx)
+{
+        struct dxcore_lib lib = {0};
+
+        pCtx->initialized = 0;
+        pCtx->adapterCount = 0;
+        pCtx->adapterList = NULL;
+
+        lib.hDxcoreLib = dlopen("libdxcore.so", RTLD_LAZY);
+        if (!lib.hDxcoreLib) {
+                goto error;
+        }
+
+        lib.pDxcoreEnumAdapters2 = (pfnDxcoreEnumAdapters2)dlsym(lib.hDxcoreLib, "D3DKMTEnumAdapters2");
+        if (!lib.pDxcoreEnumAdapters2) {
+                log_err("dxcore library is present but the symbol D3DKMTEnumAdapters2 is missing");
+                goto error;
+        }
+
+        lib.pDxcoreQueryAdapterInfo = (pfnDxcoreQueryAdapterInfo)dlsym(lib.hDxcoreLib, "D3DKMTQueryAdapterInfo");
+        if (!lib.pDxcoreQueryAdapterInfo) {
+                log_err("dxcore library is present but the symbol D3DKMTQueryAdapterInfo is missing");
+                goto error;
+        }
+
+        dxcore_enum_adapters(pCtx, &lib);
+
+        log_info("dxcore layer initialized successfully");
+        pCtx->initialized = 1;
+
+        dlclose(lib.hDxcoreLib);
+
+        return 0;
+
+error:
+        dxcore_deinit_context(pCtx);
+
+        if (lib.hDxcoreLib)
+                dlclose(lib.hDxcoreLib);
+
+        return (-1);
+}
+
+static void dxcore_deinit_adapter(struct dxcore_adapter* pAdapter)
+{
+        if (!pAdapter)
+            return;
+
+        free(pAdapter->pDriverStorePath);
+}
+
+void dxcore_deinit_context(struct dxcore_context* pCtx)
+{
+        unsigned int adapterIndex = 0;
+
+        if (!pCtx)
+                return;
+
+        for (adapterIndex = 0; adapterIndex < pCtx->adapterCount; adapterIndex++) {
+                dxcore_deinit_adapter(&pCtx->adapterList[adapterIndex]);
+        }
+
+        free(pCtx->adapterList);
+
+        pCtx->initialized = 0;
+}

internal/dxcore/dxcore.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+/*
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
+#include <dxcore.h>
+*/
+import "C"
+import (
+	"fmt"
+	"unsafe"
+)
+
+type context C.struct_dxcore_context
+type adapter C.struct_dxcore_adapter
+
+// initContext initializes the dxcore context and populates the list of adapters.
+func initContext() (*context, error) {
+	cContext := C.struct_dxcore_context{}
+	if C.dxcore_init_context(&cContext) != 0 {
+		return nil, fmt.Errorf("failed to initialize dxcore context")
+	}
+	c := (*context)(&cContext)
+	return c, nil
+}
+
+// deinitContext deinitializes the dxcore context and frees the list of adapters.
+func (c context) deinitContext() {
+	cContext := C.struct_dxcore_context(c)
+	C.dxcore_deinit_context(&cContext)
+}
+
+func (c context) getAdapterCount() int {
+	return int(c.adapterCount)
+}
+
+func (c context) getAdapter(index int) adapter {
+	arrayPointer := (*[1 << 30]C.struct_dxcore_adapter)(unsafe.Pointer(c.adapterList))
+	return adapter(arrayPointer[index])
+}
+
+func (a adapter) getDriverStorePath() string {
+	return C.GoString(a.pDriverStorePath)
+}

internal/dxcore/dxcore.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#ifndef HEADER_DXCORE_H_
+#define HEADER_DXCORE_H_
+
+#define MAX_DXCORE_DRIVERSTORE_LIBRAIRIES (16)
+
+struct dxcore_luid
+{
+        unsigned int lowPart;
+        int highPart;
+};
+
+struct dxcore_adapter
+{
+        unsigned int             hAdapter;
+        unsigned int             wddmVersion;
+        char*                    pDriverStorePath;
+        unsigned int             driverStoreComponentCount;
+        const char*              pDriverStoreComponents[MAX_DXCORE_DRIVERSTORE_LIBRAIRIES];
+        struct dxcore_context    *pContext;
+};
+
+struct dxcore_context
+{
+        unsigned int adapterCount;
+        struct dxcore_adapter *adapterList;
+
+        int initialized;
+};
+
+
+
+int dxcore_init_context(struct dxcore_context* pDxcore_context);
+void dxcore_deinit_context(struct dxcore_context* pDxcore_context);
+
+#endif // HEADER_DXCORE_H_

internal/edits/mount.go

@@ -40,12 +40,7 @@ func (d mount) toSpec() *specs.Mount {
 	s := specs.Mount{
 		HostPath:      d.HostPath,
 		ContainerPath: d.Path,
-		Options: []string{
-			"ro",
-			"nosuid",
-			"nodev",
-			"bind",
-		},
+		Options:       d.Options,
 	}
 
 	return &s

internal/ldcache/ldcache.go

@@ -307,11 +307,7 @@ func (c *ldcache) resolve(target string) (string, error) {
 		link = filepath.Join(filepath.Dir(target), link)
 	}
 
-	// Ensure that the returned path is relative to the root.
-	link = filepath.Join(c.root, link)
-
-	c.logger.Debugf("Resolved link: '%v' => '%v'", name, link)
-	return link, nil
+	return c.resolve(link)
 }
 
 // bytesToString converts a byte slice to a string.

internal/lookup/cuda/cuda.go

@@ -0,0 +1,104 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cuda
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type cudaLocator struct {
+	logger     *logrus.Logger
+	driverRoot string
+}
+
+// Options is a function that configures a cudaLocator.
+type Options func(*cudaLocator)
+
+// WithLogger is an option that configures the logger used by the locator.
+func WithLogger(logger *logrus.Logger) Options {
+	return func(c *cudaLocator) {
+		c.logger = logger
+	}
+}
+
+// WithDriverRoot is an option that configures the driver root used by the locator.
+func WithDriverRoot(driverRoot string) Options {
+	return func(c *cudaLocator) {
+		c.driverRoot = driverRoot
+	}
+}
+
+// New creates a new CUDA library locator.
+func New(opts ...Options) lookup.Locator {
+	c := &cudaLocator{}
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	if c.logger == nil {
+		c.logger = logrus.StandardLogger()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+
+	return c
+}
+
+// Locate returns the path to the libcuda.so.RMVERSION file.
+// libcuda.so is prefixed to the specified pattern.
+func (l *cudaLocator) Locate(pattern string) ([]string, error) {
+	ldcacheLocator, err := lookup.NewLibraryLocator(
+		l.logger,
+		l.driverRoot,
+	)
+	if err != nil {
+		l.logger.Debugf("Failed to create LDCache locator: %v", err)
+	}
+
+	fullPattern := "libcuda.so" + pattern
+
+	candidates, err := ldcacheLocator.Locate("libcuda.so")
+	if err == nil {
+		for _, c := range candidates {
+			if match, err := filepath.Match(fullPattern, filepath.Base(c)); err != nil || !match {
+				l.logger.Debugf("Skipping non-matching candidate %v: %v", c, err)
+				continue
+			}
+			return []string{c}, nil
+		}
+	}
+	l.logger.Debugf("Could not locate %q in LDCache: Checking predefined library paths.", pattern)
+
+	pathLocator := lookup.NewFileLocator(
+		lookup.WithLogger(l.logger),
+		lookup.WithRoot(l.driverRoot),
+		lookup.WithSearchPaths(
+			"/usr/lib64",
+			"/usr/lib/x86_64-linux-gnu",
+			"/usr/lib/aarch64-linux-gnu",
+			"/usr/lib/x86_64-linux-gnu/nvidia/current",
+			"/usr/lib/aarch64-linux-gnu/nvidia/current",
+		),
+		lookup.WithCount(1),
+	)
+
+	return pathLocator.Locate(fullPattern)
+}

internal/lookup/library.go

@@ -40,6 +40,7 @@ func NewLibraryLocator(logger *log.Logger, root string) (Locator, error) {
 	}
 
 	l := library{
+		logger:  logger,
 		symlink: NewSymlinkLocator(logger, root),
 		cache:   cache,
 	}

internal/modifier/cdi.go

@@ -38,7 +38,7 @@ type cdiModifier struct {
 // CDI specifications available on the system. The NVIDIA_VISIBLE_DEVICES enviroment variable is
 // used to select the devices to include.
 func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	devices, err := getDevicesFromSpec(ociSpec)
+	devices, err := getDevicesFromSpec(logger, ociSpec, cfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get required devices from OCI specification: %v", err)
 	}
@@ -46,6 +46,7 @@ func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		logger.Debugf("No devices requested; no modification required.")
 		return nil, nil
 	}
+	logger.Debugf("Creating CDI modifier for devices: %v", devices)
 
 	specDirs := cdi.DefaultSpecDirs
 	if len(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs) > 0 {
@@ -61,38 +62,82 @@ func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 	return m, nil
 }
 
-func getDevicesFromSpec(ociSpec oci.Spec) ([]string, error) {
+func getDevicesFromSpec(logger *logrus.Logger, ociSpec oci.Spec, cfg *config.Config) ([]string, error) {
 	rawSpec, err := ociSpec.Load()
 	if err != nil {
 		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
 	}
 
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	envDevices := image.DevicesFromEnvvars(visibleDevicesEnvvar)
-
-	_, annotationDevices, err := cdi.ParseAnnotations(rawSpec.Annotations)
+	annotationDevices, err := getAnnotationDevices(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.AnnotationPrefixes, rawSpec.Annotations)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse container annotations: %v", err)
 	}
-
-	uniqueDevices := make(map[string]struct{})
-	for _, name := range append(envDevices.List(), annotationDevices...) {
-		if !cdi.IsQualifiedName(name) {
-			name = cdi.QualifiedName("nvidia.com", "gpu", name)
-		}
-		uniqueDevices[name] = struct{}{}
+	if len(annotationDevices) > 0 {
+		return annotationDevices, nil
 	}
 
+	container, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+	envDevices := container.DevicesFromEnvvars(visibleDevicesEnvvar)
+
 	var devices []string
-	for name := range uniqueDevices {
+	seen := make(map[string]bool)
+	for _, name := range envDevices.List() {
+		if !cdi.IsQualifiedName(name) {
+			name = fmt.Sprintf("%s=%s", cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.DefaultKind, name)
+		}
+		if seen[name] {
+			logger.Debugf("Ignoring duplicate device %q", name)
+			continue
+		}
 		devices = append(devices, name)
 	}
 
-	return devices, nil
+	if len(devices) == 0 {
+		return nil, nil
+	}
+
+	if cfg.AcceptEnvvarUnprivileged || image.IsPrivileged(rawSpec) {
+		return devices, nil
+	}
+
+	logger.Warningf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES: %v", devices)
+
+	return nil, nil
+}
+
+// getAnnotationDevices returns a list of devices specified in the annotations.
+// Keys starting with the specified prefixes are considered and expected to contain a comma-separated list of
+// fully-qualified CDI devices names. If any device name is not fully-quality an error is returned.
+// The list of returned devices is deduplicated.
+func getAnnotationDevices(prefixes []string, annotations map[string]string) ([]string, error) {
+	devicesByKey := make(map[string][]string)
+	for key, value := range annotations {
+		for _, prefix := range prefixes {
+			if strings.HasPrefix(key, prefix) {
+				devicesByKey[key] = strings.Split(value, ",")
+			}
+		}
+	}
+
+	seen := make(map[string]bool)
+	var annotationDevices []string
+	for key, devices := range devicesByKey {
+		for _, device := range devices {
+			if !cdi.IsQualifiedName(device) {
+				return nil, fmt.Errorf("invalid device name %q in annotation %q", device, key)
+			}
+			if seen[device] {
+				continue
+			}
+			annotationDevices = append(annotationDevices, device)
+			seen[device] = true
+		}
+	}
+
+	return annotationDevices, nil
 }
 
 // Modify loads the CDI registry and injects the specified CDI devices into the OCI runtime specification.
@@ -105,21 +150,8 @@ func (m cdiModifier) Modify(spec *specs.Spec) error {
 		m.logger.Debugf("The following error was triggered when refreshing the CDI registry: %v", err)
 	}
 
-	devices := m.devices
-	for _, d := range devices {
-		if d == "nvidia.com/gpu=all" {
-			devices = []string{}
-			for _, candidate := range registry.DeviceDB().ListDevices() {
-				if strings.HasPrefix(candidate, "nvidia.com/gpu=") {
-					devices = append(devices, candidate)
-				}
-			}
-			break
-		}
-	}
-
-	m.logger.Debugf("Injecting devices using CDI: %v", devices)
-	_, err := registry.InjectDevices(spec, devices...)
+	m.logger.Debugf("Injecting devices using CDI: %v", m.devices)
+	_, err := registry.InjectDevices(spec, m.devices...)
 	if err != nil {
 		return fmt.Errorf("failed to inject CDI devices: %v", err)
 	}

internal/modifier/cdi_test.go

@@ -0,0 +1,92 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetAnnotationDevices(t *testing.T) {
+	testCases := []struct {
+		description     string
+		prefixes        []string
+		annotations     map[string]string
+		expectedDevices []string
+		expectedError   error
+	}{
+		{
+			description: "no annotations",
+		},
+		{
+			description: "no matching annotations",
+			prefixes:    []string{"not-prefix/"},
+			annotations: map[string]string{
+				"prefix/foo": "example.com/device=bar",
+			},
+		},
+		{
+			description: "single matching annotation",
+			prefixes:    []string{"prefix/"},
+			annotations: map[string]string{
+				"prefix/foo": "example.com/device=bar",
+			},
+			expectedDevices: []string{"example.com/device=bar"},
+		},
+		{
+			description: "multiple matching annotations",
+			prefixes:    []string{"prefix/", "another-prefix/"},
+			annotations: map[string]string{
+				"prefix/foo":         "example.com/device=bar",
+				"another-prefix/bar": "example.com/device=baz",
+			},
+			expectedDevices: []string{"example.com/device=bar", "example.com/device=baz"},
+		},
+		{
+			description: "multiple matching annotations with duplicate devices",
+			prefixes:    []string{"prefix/", "another-prefix/"},
+			annotations: map[string]string{
+				"prefix/foo":         "example.com/device=bar",
+				"another-prefix/bar": "example.com/device=bar",
+			},
+			expectedDevices: []string{"example.com/device=bar"},
+		},
+		{
+			description: "invalid devices",
+			prefixes:    []string{"prefix/"},
+			annotations: map[string]string{
+				"prefix/foo": "example.com/device",
+			},
+			expectedError: fmt.Errorf("invalid device %q", "example.com/device"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			devices, err := getAnnotationDevices(tc.prefixes, tc.annotations)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.ElementsMatch(t, tc.expectedDevices, devices)
+		})
+	}
+}

internal/oci/runtime_modifier.go

@@ -14,27 +14,26 @@
 # limitations under the License.
 */
 
-package runtime
+package oci
 
 import (
 	"fmt"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	log "github.com/sirupsen/logrus"
 )
 
 type modifyingRuntimeWrapper struct {
 	logger   *log.Logger
-	runtime  oci.Runtime
-	ociSpec  oci.Spec
-	modifier oci.SpecModifier
+	runtime  Runtime
+	ociSpec  Spec
+	modifier SpecModifier
 }
 
-var _ oci.Runtime = (*modifyingRuntimeWrapper)(nil)
+var _ Runtime = (*modifyingRuntimeWrapper)(nil)
 
 // NewModifyingRuntimeWrapper creates a runtime wrapper that applies the specified modifier to the OCI specification
 // before invoking the wrapped runtime. If the modifier is nil, the input runtime is returned.
-func NewModifyingRuntimeWrapper(logger *log.Logger, runtime oci.Runtime, spec oci.Spec, modifier oci.SpecModifier) oci.Runtime {
+func NewModifyingRuntimeWrapper(logger *log.Logger, runtime Runtime, spec Spec, modifier SpecModifier) Runtime {
 	if modifier == nil {
 		logger.Infof("Using low-level runtime with no modification")
 		return runtime
@@ -52,7 +51,7 @@ func NewModifyingRuntimeWrapper(logger *log.Logger, runtime oci.Runtime, spec oc
 // Exec checks whether a modification of the OCI specification is required and modifies it accordingly before exec-ing
 // into the wrapped runtime.
 func (r *modifyingRuntimeWrapper) Exec(args []string) error {
-	if oci.HasCreateSubcommand(args) {
+	if HasCreateSubcommand(args) {
 		err := r.modify()
 		if err != nil {
 			return fmt.Errorf("could not apply required modification to OCI specification: %v", err)

internal/oci/runtime_modifier_test.go

@@ -14,13 +14,12 @@
 # limitations under the License.
 */
 
-package runtime
+package oci
 
 import (
 	"fmt"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
@@ -38,7 +37,7 @@ func TestExec(t *testing.T) {
 		args          []string
 		modifyError   error
 		writeError    error
-		modifer       oci.SpecModifier
+		modifer       SpecModifier
 	}{
 		{
 			description:   "no args forwards",
@@ -92,9 +91,9 @@ func TestExec(t *testing.T) {
 		hook.Reset()
 
 		t.Run(tc.description, func(t *testing.T) {
-			runtimeMock := &oci.RuntimeMock{}
-			specMock := &oci.SpecMock{
-				ModifyFunc: func(specModifier oci.SpecModifier) error {
+			runtimeMock := &RuntimeMock{}
+			specMock := &SpecMock{
+				ModifyFunc: func(specModifier SpecModifier) error {
 					return tc.modifyError
 				},
 				FlushFunc: func() error {
@@ -144,8 +143,8 @@ func TestExec(t *testing.T) {
 func TestNilModiferReturnsRuntime(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
-	runtimeMock := &oci.RuntimeMock{}
-	specMock := &oci.SpecMock{}
+	runtimeMock := &RuntimeMock{}
+	specMock := &SpecMock{}
 
 	shim := NewModifyingRuntimeWrapper(
 		logger,

internal/runtime/api.go

@@ -0,0 +1,33 @@
+package runtime
+
+type rt struct {
+	logger       *Logger
+	modeOverride string
+}
+
+// Interface is the interface for the runtime library.
+type Interface interface {
+	Run([]string) error
+}
+
+// Option is a function that configures the runtime.
+type Option func(*rt)
+
+// New creates a runtime with the specified options.
+func New(opts ...Option) Interface {
+	r := rt{}
+	for _, opt := range opts {
+		opt(&r)
+	}
+	if r.logger == nil {
+		r.logger = NewLogger()
+	}
+	return &r
+}
+
+// WithModeOverride allows for overriding the mode specified in the config.
+func WithModeOverride(mode string) Option {
+	return func(r *rt) {
+		r.modeOverride = mode
+	}
+}

internal/runtime/logger.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 */
 
-package main
+package runtime
 
 import (
 	"fmt"
@@ -42,11 +42,17 @@ func NewLogger() *Logger {
 	}
 }
 
-// UpdateLogger constructs a Logger with a preddefined formatter
-func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, error) {
+// Update constructs a Logger with a preddefined formatter
+func (l *Logger) Update(filename string, logLevel string, argv []string) error {
+
 	configFromArgs := parseArgs(argv)
 
 	level, logLevelError := configFromArgs.getLevel(logLevel)
+	defer func() {
+		if logLevelError != nil {
+			l.Warn(logLevelError)
+		}
+	}()
 
 	var logFiles []*os.File
 	var argLogFileError error
@@ -55,7 +61,7 @@ func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, err
 	if !configFromArgs.version {
 		configLogFile, err := createLogFile(filename)
 		if err != nil {
-			return logger, fmt.Errorf("error opening debug log file: %v", err)
+			return fmt.Errorf("error opening debug log file: %v", err)
 		}
 		if configLogFile != nil {
 			logFiles = append(logFiles, configLogFile)
@@ -67,14 +73,15 @@ func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, err
 		}
 		argLogFileError = err
 	}
+	defer func() {
+		if argLogFileError != nil {
+			l.Warnf("Failed to open log file: %v", argLogFileError)
+		}
+	}()
 
-	l := &Logger{
-		Logger:         logrus.New(),
-		previousLogger: logger.Logger,
-		logFiles:       logFiles,
-	}
+	newLogger := logrus.New()
 
-	l.SetLevel(level)
+	newLogger.SetLevel(level)
 	if level == logrus.DebugLevel {
 		logrus.SetReportCaller(true)
 		// Shorten function and file names reported by the logger, by
@@ -92,30 +99,28 @@ func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, err
 	}
 
 	if configFromArgs.format == "json" {
-		l.SetFormatter(new(logrus.JSONFormatter))
+		newLogger.SetFormatter(new(logrus.JSONFormatter))
 	}
 
 	if len(logFiles) == 0 {
-		l.SetOutput(io.Discard)
+		newLogger.SetOutput(io.Discard)
 	} else if len(logFiles) == 1 {
-		l.SetOutput(logFiles[0])
+		newLogger.SetOutput(logFiles[0])
 	} else if len(logFiles) > 1 {
 		var writers []io.Writer
 		for _, f := range logFiles {
 			writers = append(writers, f)
 		}
-		l.SetOutput(io.MultiWriter(writers...))
+		newLogger.SetOutput(io.MultiWriter(writers...))
 	}
 
-	if logLevelError != nil {
-		l.Warn(logLevelError)
+	*l = Logger{
+		Logger:         newLogger,
+		previousLogger: l.Logger,
+		logFiles:       logFiles,
 	}
 
-	if argLogFileError != nil {
-		l.Warnf("Failed to open log file: %v", argLogFileError)
-	}
-
-	return l, nil
+	return nil
 }
 
 // Reset closes the log file (if any) and resets the logger output to what it
@@ -126,7 +131,9 @@ func (l *Logger) Reset() error {
 		if previous == nil {
 			previous = logrus.New()
 		}
-		logger = &Logger{Logger: previous}
+		l.Logger = previous
+		l.previousLogger = nil
+		l.logFiles = nil
 	}()
 
 	var errs []error

internal/runtime/logger_test.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLogger(t *testing.T) {
+	l := NewLogger()
+
+	l.Update("", "debug", nil)
+
+	require.Equal(t, logrus.DebugLevel, l.Logger.Level)
+	require.Equal(t, logrus.InfoLevel, l.previousLogger.Level)
+
+}

internal/runtime/runtime.go

@@ -0,0 +1,109 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// Run is an entry point that allows for idiomatic handling of errors
+// when calling from the main function.
+func (r rt) Run(argv []string) (rerr error) {
+	defer func() {
+		if rerr != nil {
+			r.logger.Errorf("%v", rerr)
+		}
+	}()
+
+	printVersion := hasVersionFlag(argv)
+	if printVersion {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
+	}
+
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return fmt.Errorf("error loading config: %v", err)
+	}
+	if r.modeOverride != "" {
+		cfg.NVIDIAContainerRuntimeConfig.Mode = r.modeOverride
+	}
+
+	err = r.logger.Update(
+		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
+		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
+		argv,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to set up logger: %v", err)
+	}
+	defer func() {
+		if rerr != nil {
+			r.logger.Errorf("%v", rerr)
+		}
+		r.logger.Reset()
+	}()
+
+	// Print the config to the output.
+	configJSON, err := json.MarshalIndent(cfg, "", "  ")
+	if err == nil {
+		r.logger.Infof("Running with config:\n%v", string(configJSON))
+	} else {
+		r.logger.Infof("Running with config:\n%+v", cfg)
+	}
+
+	r.logger.Debugf("Command line arguments: %v", argv)
+	runtime, err := newNVIDIAContainerRuntime(r.logger.Logger, cfg, argv)
+	if err != nil {
+		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
+	}
+
+	if printVersion {
+		fmt.Print("\n")
+	}
+	return runtime.Exec(argv)
+}
+
+func (r rt) Errorf(format string, args ...interface{}) {
+	r.logger.Errorf(format, args...)
+}
+
+// TODO: This should be refactored / combined with parseArgs in logger.
+func hasVersionFlag(args []string) bool {
+	for i := 0; i < len(args); i++ {
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		trimmed := strings.TrimLeft(parts[0], "-")
+		// If this is not a flag we continue
+		if parts[0] == trimmed {
+			continue
+		}
+
+		// Check the version flag
+		if trimmed == "version" {
+			return true
+		}
+	}
+
+	return false
+}

internal/runtime/runtime_factory.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 */
 
-package main
+package runtime
 
 import (
 	"fmt"
@@ -23,7 +23,6 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 	"github.com/sirupsen/logrus"
 )
 
@@ -50,7 +49,7 @@ func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv [
 	}
 
 	// Create the wrapping runtime with the specified modifier
-	r := runtime.NewModifyingRuntimeWrapper(
+	r := oci.NewModifyingRuntimeWrapper(
 		logger,
 		lowLevelRuntime,
 		ociSpec,

internal/runtime/runtime_factory_test.go

@@ -14,20 +14,52 @@
 # limitations under the License.
 */
 
-package main
+package runtime
 
 import (
 	"encoding/json"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
+const (
+	runcExecutableName = "runc"
+)
+
+func TestMain(m *testing.M) {
+	// TEST SETUP
+	// Determine the module root and the test binary path
+	var err error
+	moduleRoot, err := test.GetModuleRoot()
+	if err != nil {
+		logrus.Fatalf("error in test setup: could not get module root: %v", err)
+	}
+	testBinPath := filepath.Join(moduleRoot, "test", "bin")
+
+	// Set the environment variables for the test
+	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
+
+	// Confirm that the environment is configured correctly
+	runcPath, err := exec.LookPath(runcExecutableName)
+	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
+		logrus.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+	}
+
+	// RUN TESTS
+	exitCode := m.Run()
+
+	os.Exit(exitCode)
+}
+
 func TestFactoryMethod(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 

internal/system/options.go

@@ -0,0 +1,36 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import "github.com/sirupsen/logrus"
+
+// Option is a functional option for the system command
+type Option func(*Interface)
+
+// WithLogger sets the logger for the system command
+func WithLogger(logger *logrus.Logger) Option {
+	return func(i *Interface) {
+		i.logger = logger
+	}
+}
+
+// WithDryRun sets the dry run flag
+func WithDryRun(dryRun bool) Option {
+	return func(i *Interface) {
+		i.dryRun = dryRun
+	}
+}

internal/system/system.go

@@ -0,0 +1,149 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+// Interface is the interface for the system command
+type Interface struct {
+	logger *logrus.Logger
+	dryRun bool
+
+	nvidiaDevices nvidiaDevices
+}
+
+// New constructs a system command with the specified options
+func New(opts ...Option) (*Interface, error) {
+	i := &Interface{
+		logger: logrus.StandardLogger(),
+	}
+	for _, opt := range opts {
+		opt(i)
+	}
+
+	devices, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create devices info: %v", err)
+	}
+	i.nvidiaDevices = nvidiaDevices{devices}
+
+	return i, nil
+}
+
+// CreateNVIDIAControlDeviceNodesAt creates the NVIDIA control device nodes associated with the NVIDIA driver at the specified root.
+func (m *Interface) CreateNVIDIAControlDeviceNodesAt(root string) error {
+	controlNodes := []string{"/dev/nvidiactl", "/dev/nvidia-modeset", "/dev/nvidia-uvm", "/dev/nvidia-uvm-tools"}
+
+	for _, node := range controlNodes {
+		path := filepath.Join(root, node)
+		err := m.CreateNVIDIADeviceNode(path)
+		if err != nil {
+			return fmt.Errorf("failed to create device node %s: %v", path, err)
+		}
+	}
+
+	return nil
+}
+
+// CreateNVIDIADeviceNode creates a specified device node associated with the NVIDIA driver.
+func (m *Interface) CreateNVIDIADeviceNode(path string) error {
+	node := filepath.Base(path)
+	if !strings.HasPrefix(node, "nvidia") {
+		return fmt.Errorf("invalid device node %q", node)
+	}
+
+	major, err := m.nvidiaDevices.Major(node)
+	if err != nil {
+		return fmt.Errorf("failed to determine major: %v", err)
+	}
+
+	minor, err := m.nvidiaDevices.Minor(node)
+	if err != nil {
+		return fmt.Errorf("failed to determine minor: %v", err)
+	}
+
+	return m.createDeviceNode(path, int(major), int(minor))
+}
+
+func (m *Interface) createDeviceNode(path string, major int, minor int) error {
+	if m.dryRun {
+		m.logger.Infof("Running: mknod --mode=0666 %s c %d %d", path, major, minor)
+		return nil
+	}
+
+	if _, err := os.Stat(path); err == nil {
+		m.logger.Infof("Skipping: %s already exists", path)
+		return nil
+	} else if !os.IsNotExist(err) {
+		return fmt.Errorf("failed to stat %s: %v", path, err)
+	}
+
+	err := unix.Mknod(path, unix.S_IFCHR, int(unix.Mkdev(uint32(major), uint32(minor))))
+	if err != nil {
+		return err
+	}
+	return unix.Chmod(path, 0666)
+}
+
+type nvidiaDevices struct {
+	devices.Devices
+}
+
+// Major returns the major number for the specified NVIDIA device node.
+// If the device node is not supported, an error is returned.
+func (n *nvidiaDevices) Major(node string) (int64, error) {
+	var valid bool
+	var major devices.Major
+	switch node {
+	case "nvidia-uvm", "nvidia-uvm-tools":
+		major, valid = n.Get(devices.NVIDIAUVM)
+	case "nvidia-modeset", "nvidiactl":
+		major, valid = n.Get(devices.NVIDIAGPU)
+	}
+
+	if !valid {
+		return 0, fmt.Errorf("invalid device node %q", node)
+	}
+
+	return int64(major), nil
+}
+
+// Minor returns the minor number for the specified NVIDIA device node.
+// If the device node is not supported, an error is returned.
+func (n *nvidiaDevices) Minor(node string) (int64, error) {
+	switch node {
+	case "nvidia-modeset":
+		return devices.NVIDIAModesetMinor, nil
+	case "nvidia-uvm-tools":
+		return devices.NVIDIAUVMToolsMinor, nil
+	case "nvidia-uvm":
+		return devices.NVIDIAUVMMinor, nil
+	case "nvidiactl":
+		return devices.NVIDIACTLMinor, nil
+	}
+
+	return 0, fmt.Errorf("invalid device node %q", node)
+}

packaging/debian/control

@@ -23,3 +23,9 @@ Breaks: nvidia-container-runtime (<= 3.5.0-1), nvidia-container-runtime-hook, nv
 Replaces: nvidia-container-runtime (<= 3.5.0-1), nvidia-container-runtime-hook
 Description: NVIDIA Container Toolkit Base
  Provides tools such as the NVIDIA Container Runtime and NVIDIA Container Toolkit CLI to enable GPU support in containers.
+
+Package: nvidia-container-toolkit-operator-extensions
+Architecture: any
+Depends: ${misc:Depends}, nvidia-container-toolkit-base (= @VERSION@)
+Description: NVIDIA Container Toolkit Operator Extensions
+ Provides tools for using the NVIDIA Container Toolkit with the GPU Operator

packaging/debian/nvidia-container-toolkit-operator-extensions.install

@@ -0,0 +1,2 @@
+nvidia-container-runtime.cdi /usr/bin
+nvidia-container-runtime.legacy /usr/bin

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -11,12 +11,14 @@ URL: https://github.com/NVIDIA/nvidia-container-toolkit
 License: Apache-2.0
 
 Source0: nvidia-container-runtime-hook
-Source1: nvidia-container-runtime
-Source2: nvidia-ctk
-Source3: config.toml
-Source4: oci-nvidia-hook
-Source5: oci-nvidia-hook.json
-Source6: LICENSE
+Source1: nvidia-ctk
+Source2: config.toml
+Source3: oci-nvidia-hook
+Source4: oci-nvidia-hook.json
+Source5: LICENSE
+Source6: nvidia-container-runtime
+Source7: nvidia-container-runtime.cdi
+Source8: nvidia-container-runtime.legacy
 
 Obsoletes: nvidia-container-runtime <= 3.5.0-1, nvidia-container-runtime-hook <= 1.4.0-2
 Provides: nvidia-container-runtime
@@ -35,12 +37,14 @@ Requires: libseccomp
 Provides tools and utilities to enable GPU support in containers.
 
 %prep
-cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} .
+cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} %{SOURCE7} %{SOURCE8} .
 
 %install
 mkdir -p %{buildroot}%{_bindir}
 install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime-hook
 install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime
+install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime.cdi
+install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime.legacy
 install -m 755 -t %{buildroot}%{_bindir} nvidia-ctk
 
 mkdir -p %{buildroot}/etc/nvidia-container-runtime
@@ -57,10 +61,10 @@ mkdir -p %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
 cp -af %{_bindir}/nvidia-container-runtime-hook %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
 
 %posttrans
-if [ ! -e %{_bindir}/nvidia-container-runtime-hook ]; then 
+if [ ! -e %{_bindir}/nvidia-container-runtime-hook ]; then
   # reparing lost file nvidia-container-runtime-hook
-  cp -avf %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit/nvidia-container-runtime-hook %{_bindir} 
-fi 
+  cp -avf %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit/nvidia-container-runtime-hook %{_bindir}
+fi
 rm -rf %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
 ln -sf %{_bindir}/nvidia-container-runtime-hook %{_bindir}/nvidia-container-toolkit
 
@@ -99,3 +103,17 @@ Provides tools such as the NVIDIA Container Runtime and NVIDIA Container Toolkit
 %config /etc/nvidia-container-runtime/config.toml
 %{_bindir}/nvidia-container-runtime
 %{_bindir}/nvidia-ctk
+
+# The OPERATOR EXTENSIONS package consists of components that are required to enable GPU support in Kubernetes.
+# This package is not distributed as part of the NVIDIA Container Toolkit RPMs.
+%package operator-extensions
+Summary: NVIDIA Container Toolkit Operator Extensions
+Requires: nvidia-container-toolkit-base == %{version}-%{release}
+
+%description operator-extensions
+Provides tools for using the NVIDIA Container Toolkit with the GPU Operator
+
+%files operator-extensions
+%license LICENSE
+%{_bindir}/nvidia-container-runtime.cdi
+%{_bindir}/nvidia-container-runtime.legacy

pkg/nvcdi/api.go

@@ -0,0 +1,50 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+)
+
+const (
+	// ModeAuto configures the CDI spec generator to automatically detect the system configuration
+	ModeAuto = "auto"
+	// ModeNvml configures the CDI spec generator to use the NVML library.
+	ModeNvml = "nvml"
+	// ModeWsl configures the CDI spec generator to generate a WSL spec.
+	ModeWsl = "wsl"
+	// ModeManagement configures the CDI spec generator to generate a management spec.
+	ModeManagement = "management"
+	// ModeGds configures the CDI spec generator to generate a GDS spec.
+	ModeGds = "gds"
+	// ModeMofed configures the CDI spec generator to generate a MOFED spec.
+	ModeMofed = "mofed"
+)
+
+// Interface defines the API for the nvcdi package
+type Interface interface {
+	GetSpec() (spec.Interface, error)
+	GetCommonEdits() (*cdi.ContainerEdits, error)
+	GetAllDeviceSpecs() ([]specs.Device, error)
+	GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error)
+	GetGPUDeviceSpecs(int, device.Device) (*specs.Device, error)
+	GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error)
+	GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error)
+}

pkg/nvcdi/common-nvml.go

@@ -14,20 +14,21 @@
 # limitations under the License.
 **/
 
-package generate
+package nvcdi
 
 import (
 	"fmt"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+
 	"github.com/sirupsen/logrus"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
 )
 
-// NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
+// newCommonNVMLDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
 // This includes driver libraries and meta devices, for example.
-func NewCommonDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
+func newCommonNVMLDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
 	metaDevices := discover.NewDeviceDiscoverer(
 		logger,
 		lookup.NewCharDeviceLocator(

pkg/nvcdi/device-wsl.go

@@ -14,29 +14,24 @@
 # limitations under the License.
 **/
 
-package generate
+package nvcdi
 
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/sirupsen/logrus"
 )
 
-// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
-func NewIPCDiscoverer(logger *logrus.Logger, driverRoot string) (discover.Discover, error) {
-	d := discover.NewMounts(
+const (
+	dxgDeviceNode = "/dev/dxg"
+)
+
+// newDXGDeviceDiscoverer returns a Discoverer for DXG devices under WSL2.
+func newDXGDeviceDiscoverer(logger *logrus.Logger, driverRoot string) discover.Discover {
+	deviceNodes := discover.NewCharDeviceDiscoverer(
 		logger,
-		lookup.NewFileLocator(
-			lookup.WithLogger(logger),
-			lookup.WithRoot(driverRoot),
-		),
+		[]string{dxgDeviceNode},
 		driverRoot,
-		[]string{
-			"/var/run/nvidia-persistenced/socket",
-			"/var/run/nvidia-fabricmanager/socket",
-			"/tmp/nvidia-mps",
-		},
 	)
 
-	return d, nil
+	return deviceNodes
 }

pkg/nvcdi/driver-nvml.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 **/
 
-package generate
+package nvcdi
 
 import (
 	"fmt"
@@ -22,8 +22,8 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
 )
@@ -31,22 +31,37 @@ import (
 // NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
 // The supplied NVML Library is used to query the expected driver version.
 func NewDriverDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
+	if r := nvmllib.Init(); r != nvml.SUCCESS {
+		return nil, fmt.Errorf("failed to initalize NVML: %v", r)
+	}
+	defer nvmllib.Shutdown()
+
 	version, r := nvmllib.SystemGetDriverVersion()
 	if r != nvml.SUCCESS {
 		return nil, fmt.Errorf("failed to determine driver version: %v", r)
 	}
 
+	return newDriverVersionDiscoverer(logger, driverRoot, nvidiaCTKPath, version)
+}
+
+func newDriverVersionDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, version string) (discover.Discover, error) {
 	libraries, err := NewDriverLibraryDiscoverer(logger, driverRoot, nvidiaCTKPath, version)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
 	}
 
+	ipcs, err := discover.NewIPCDiscoverer(logger, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
+	}
+
 	firmwares := NewDriverFirmwareDiscoverer(logger, driverRoot, version)
 
 	binaries := NewDriverBinariesDiscoverer(logger, driverRoot)
 
 	d := discover.Merge(
 		libraries,
+		ipcs,
 		firmwares,
 		binaries,
 	)
@@ -121,26 +136,24 @@ func NewDriverBinariesDiscoverer(logger *logrus.Logger, driverRoot string) disco
 func getVersionLibs(logger *logrus.Logger, driverRoot string, version string) ([]string, error) {
 	logger.Infof("Using driver version %v", version)
 
-	cache, err := ldcache.New(logger, driverRoot)
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate("." + version)
 	if err != nil {
-		return nil, fmt.Errorf("failed to load ldcache: %v", err)
+		return nil, fmt.Errorf("failed to locate libcuda.so.%v: %v", version, err)
 	}
+	libRoot := filepath.Dir(libCudaPaths[0])
 
-	libs32, libs64 := cache.List()
+	libraries := lookup.NewFileLocator(
+		lookup.WithLogger(logger),
+		lookup.WithSearchPaths(libRoot),
+		lookup.WithOptional(true),
+	)
 
-	var libs []string
-	for _, l := range libs64 {
-		if strings.HasSuffix(l, version) {
-			logger.Infof("found 64-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
-	}
-
-	for _, l := range libs32 {
-		if strings.HasSuffix(l, version) {
-			logger.Infof("found 32-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
+	libs, err := libraries.Locate("*.so." + version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libraries for driver version %v: %v", version, err)
 	}
 
 	if driverRoot == "/" || driverRoot == "" {

pkg/nvcdi/driver-wsl.go

@@ -0,0 +1,106 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/dxcore"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+var requiredDriverStoreFiles = []string{
+	"libcuda.so.1.1",                /* Core library for cuda support */
+	"libcuda_loader.so",             /* Core library for cuda support on WSL */
+	"libnvidia-ptxjitcompiler.so.1", /* Core library for PTX Jit support */
+	"libnvidia-ml.so.1",             /* Core library for nvml */
+	"libnvidia-ml_loader.so",        /* Core library for nvml on WSL */
+	"libdxcore.so",                  /* Core library for dxcore support */
+	"nvcubins.bin",                  /* Binary containing GPU code for cuda */
+	"nvidia-smi",                    /* nvidia-smi binary*/
+}
+
+// newWSLDriverDiscoverer returns a Discoverer for WSL2 drivers.
+func newWSLDriverDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) (discover.Discover, error) {
+	err := dxcore.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize dxcore: %v", err)
+	}
+	defer dxcore.Shutdown()
+
+	driverStorePaths := dxcore.GetDriverStorePaths()
+	if len(driverStorePaths) == 0 {
+		return nil, fmt.Errorf("no driver store paths found")
+	}
+	logger.Infof("Using WSL driver store paths: %v", driverStorePaths)
+
+	return newWSLDriverStoreDiscoverer(logger, driverRoot, nvidiaCTKPath, driverStorePaths)
+}
+
+// newWSLDriverStoreDiscoverer returns a Discoverer for WSL2 drivers in the driver store associated with a dxcore adapter.
+func newWSLDriverStoreDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, driverStorePaths []string) (discover.Discover, error) {
+	var searchPaths []string
+	seen := make(map[string]bool)
+	for _, path := range driverStorePaths {
+		if seen[path] {
+			continue
+		}
+		searchPaths = append(searchPaths, path)
+	}
+	if len(searchPaths) > 1 {
+		logger.Warnf("Found multiple driver store paths: %v", searchPaths)
+	}
+	driverStorePath := searchPaths[0]
+	searchPaths = append(searchPaths, "/usr/lib/wsl/lib")
+
+	libraries := discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithSearchPaths(
+				searchPaths...,
+			),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		requiredDriverStoreFiles,
+	)
+
+	// On WSL2 the driver store location is used unchanged.
+	// For this reason we need to create a symlink from /usr/bin/nvidia-smi to the nvidia-smi binary in the driver store.
+	target := filepath.Join(driverStorePath, "nvidia-smi")
+	link := "/usr/bin/nvidia-smi"
+	links := []string{fmt.Sprintf("%s::%s", target, link)}
+	symlinkHook := discover.CreateCreateSymlinkHook(nvidiaCTKPath, links)
+
+	cfg := &discover.Config{
+		DriverRoot:    driverRoot,
+		NvidiaCTKPath: nvidiaCTKPath,
+	}
+	ldcacheHook, _ := discover.NewLDCacheUpdateHook(logger, libraries, cfg)
+
+	d := discover.Merge(
+		libraries,
+		symlinkHook,
+		ldcacheHook,
+	)
+
+	return d, nil
+}

pkg/nvcdi/full-gpu-nvml.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 **/
 
-package generate
+package nvcdi
 
 import (
 	"fmt"
@@ -23,24 +23,63 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
 )
 
+// GetGPUDeviceSpecs returns the CDI device specs for the full GPU represented by 'device'.
+func (l *nvmllib) GetGPUDeviceSpecs(i int, d device.Device) (*specs.Device, error) {
+	edits, err := l.GetGPUDeviceEdits(d)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get edits for device: %v", err)
+	}
+
+	name, err := l.deviceNamer.GetDeviceName(i, d)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get device name: %v", err)
+	}
+
+	spec := specs.Device{
+		Name:           name,
+		ContainerEdits: *edits.ContainerEdits,
+	}
+
+	return &spec, nil
+}
+
+// GetGPUDeviceEdits returns the CDI edits for the full GPU represented by 'device'.
+func (l *nvmllib) GetGPUDeviceEdits(d device.Device) (*cdi.ContainerEdits, error) {
+	device, err := newFullGPUDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath, d)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device discoverer: %v", err)
+	}
+
+	editsForDevice, err := edits.FromDiscoverer(device)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for device: %v", err)
+	}
+
+	return editsForDevice, nil
+}
+
 // byPathHookDiscoverer discovers the entities required for injecting by-path DRM device links
 type byPathHookDiscoverer struct {
 	logger        *logrus.Logger
 	driverRoot    string
 	nvidiaCTKPath string
 	pciBusID      string
+	deviceNodes   discover.Discover
 }
 
 var _ discover.Discover = (*byPathHookDiscoverer)(nil)
 
-// NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device.
-func NewFullGPUDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, d device.Device) (discover.Discover, error) {
+// newFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device.
+func newFullGPUDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, d device.Device) (discover.Discover, error) {
 	// TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface.
 	// This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin.
 	minor, ret := d.GetMinorNumber()
@@ -73,11 +112,20 @@ func NewFullGPUDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPat
 		driverRoot:    driverRoot,
 		nvidiaCTKPath: nvidiaCTKPath,
 		pciBusID:      pciBusID,
+		deviceNodes:   deviceNodes,
 	}
 
+	deviceFolderPermissionHooks := newDeviceFolderPermissionHookDiscoverer(
+		logger,
+		driverRoot,
+		nvidiaCTKPath,
+		deviceNodes,
+	)
+
 	dd := discover.Merge(
 		deviceNodes,
 		byPathHooks,
+		deviceFolderPermissionHooks,
 	)
 
 	return dd, nil
@@ -120,6 +168,20 @@ func (d *byPathHookDiscoverer) Mounts() ([]discover.Mount, error) {
 }
 
 func (d *byPathHookDiscoverer) deviceNodeLinks() ([]string, error) {
+	devices, err := d.deviceNodes.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover device nodes: %v", err)
+	}
+
+	if len(devices) == 0 {
+		return nil, nil
+	}
+
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[d.HostPath] = true
+	}
+
 	candidates := []string{
 		fmt.Sprintf("/dev/dri/by-path/pci-%s-card", d.pciBusID),
 		fmt.Sprintf("/dev/dri/by-path/pci-%s-render", d.pciBusID),
@@ -134,6 +196,14 @@ func (d *byPathHookDiscoverer) deviceNodeLinks() ([]string, error) {
 			continue
 		}
 
+		deviceNode := device
+		if !filepath.IsAbs(device) {
+			deviceNode = filepath.Join(filepath.Dir(linkPath), device)
+		}
+		if !selectedDevices[deviceNode] {
+			d.logger.Debugf("ignoring device symlink %v -> %v since %v is not mounted", linkPath, device, deviceNode)
+			continue
+		}
 		d.logger.Debugf("adding device symlink %v -> %v", linkPath, device)
 		links = append(links, fmt.Sprintf("%v::%v", device, linkPath))
 	}

pkg/nvcdi/gds.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+)
+
+type gdslib nvcdilib
+
+var _ Interface = (*gdslib)(nil)
+
+// GetAllDeviceSpecs returns the device specs for all available devices.
+func (l *gdslib) GetAllDeviceSpecs() ([]specs.Device, error) {
+	discoverer, err := discover.NewGDSDiscoverer(l.logger, l.driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create GPUDirect Storage discoverer: %v", err)
+	}
+	edits, err := edits.FromDiscoverer(discoverer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for GPUDirect Storage: %v", err)
+	}
+
+	deviceSpec := specs.Device{
+		Name:           "all",
+		ContainerEdits: *edits.ContainerEdits,
+	}
+
+	return []specs.Device{deviceSpec}, nil
+}
+
+// GetCommonEdits generates a CDI specification that can be used for ANY devices
+func (l *gdslib) GetCommonEdits() (*cdi.ContainerEdits, error) {
+	return edits.FromDiscoverer(discover.None{})
+}
+
+// GetSpec is unsppported for the gdslib specs.
+// gdslib is typically wrapped by a spec that implements GetSpec.
+func (l *gdslib) GetSpec() (spec.Interface, error) {
+	return nil, fmt.Errorf("GetSpec is not supported")
+}
+
+// GetGPUDeviceEdits is unsupported for the gdslib specs
+func (l *gdslib) GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetGPUDeviceEdits is not supported")
+}
+
+// GetGPUDeviceSpecs is unsupported for the gdslib specs
+func (l *gdslib) GetGPUDeviceSpecs(int, device.Device) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetGPUDeviceSpecs is not supported")
+}
+
+// GetMIGDeviceEdits is unsupported for the gdslib specs
+func (l *gdslib) GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetMIGDeviceEdits is not supported")
+}
+
+// GetMIGDeviceSpecs is unsupported for the gdslib specs
+func (l *gdslib) GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetMIGDeviceSpecs is not supported")
+}

pkg/nvcdi/lib-nvml.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+type nvmllib nvcdilib
+
+var _ Interface = (*nvmllib)(nil)
+
+// GetSpec should not be called for nvmllib
+func (l *nvmllib) GetSpec() (spec.Interface, error) {
+	return nil, fmt.Errorf("Unexpected call to nvmllib.GetSpec()")
+}
+
+// GetAllDeviceSpecs returns the device specs for all available devices.
+func (l *nvmllib) GetAllDeviceSpecs() ([]specs.Device, error) {
+	var deviceSpecs []specs.Device
+
+	if r := l.nvmllib.Init(); r != nvml.SUCCESS {
+		return nil, fmt.Errorf("failed to initalize NVML: %v", r)
+	}
+	defer l.nvmllib.Shutdown()
+
+	gpuDeviceSpecs, err := l.getGPUDeviceSpecs()
+	if err != nil {
+		return nil, err
+	}
+	deviceSpecs = append(deviceSpecs, gpuDeviceSpecs...)
+
+	migDeviceSpecs, err := l.getMigDeviceSpecs()
+	if err != nil {
+		return nil, err
+	}
+	deviceSpecs = append(deviceSpecs, migDeviceSpecs...)
+
+	return deviceSpecs, nil
+}
+
+// GetCommonEdits generates a CDI specification that can be used for ANY devices
+func (l *nvmllib) GetCommonEdits() (*cdi.ContainerEdits, error) {
+	common, err := newCommonNVMLDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath, l.nvmllib)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
+	}
+
+	return edits.FromDiscoverer(common)
+}
+
+func (l *nvmllib) getGPUDeviceSpecs() ([]specs.Device, error) {
+	var deviceSpecs []specs.Device
+	err := l.devicelib.VisitDevices(func(i int, d device.Device) error {
+		deviceSpec, err := l.GetGPUDeviceSpecs(i, d)
+		if err != nil {
+			return err
+		}
+		deviceSpecs = append(deviceSpecs, *deviceSpec)
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate CDI edits for GPU devices: %v", err)
+	}
+	return deviceSpecs, err
+}
+
+func (l *nvmllib) getMigDeviceSpecs() ([]specs.Device, error) {
+	var deviceSpecs []specs.Device
+	err := l.devicelib.VisitMigDevices(func(i int, d device.Device, j int, mig device.MigDevice) error {
+		deviceSpec, err := l.GetMIGDeviceSpecs(i, d, j, mig)
+		if err != nil {
+			return err
+		}
+		deviceSpecs = append(deviceSpecs, *deviceSpec)
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate CDI edits for GPU devices: %v", err)
+	}
+	return deviceSpecs, err
+}

pkg/nvcdi/lib-wsl.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+)
+
+type wsllib nvcdilib
+
+var _ Interface = (*wsllib)(nil)
+
+// GetSpec should not be called for wsllib
+func (l *wsllib) GetSpec() (spec.Interface, error) {
+	return nil, fmt.Errorf("Unexpected call to wsllib.GetSpec()")
+}
+
+// GetAllDeviceSpecs returns the device specs for all available devices.
+func (l *wsllib) GetAllDeviceSpecs() ([]specs.Device, error) {
+	device := newDXGDeviceDiscoverer(l.logger, l.driverRoot)
+	deviceEdits, err := edits.FromDiscoverer(device)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for DXG device: %v", err)
+	}
+
+	deviceSpec := specs.Device{
+		Name:           "all",
+		ContainerEdits: *deviceEdits.ContainerEdits,
+	}
+
+	return []specs.Device{deviceSpec}, nil
+}
+
+// GetCommonEdits generates a CDI specification that can be used for ANY devices
+func (l *wsllib) GetCommonEdits() (*cdi.ContainerEdits, error) {
+	driver, err := newWSLDriverDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for WSL driver: %v", err)
+	}
+
+	return edits.FromDiscoverer(driver)
+}
+
+// GetGPUDeviceEdits generates a CDI specification that can be used for GPU devices
+func (l *wsllib) GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetGPUDeviceEdits is not supported on WSL")
+}
+
+// GetGPUDeviceSpecs returns the CDI device specs for the full GPU represented by 'device'.
+func (l *wsllib) GetGPUDeviceSpecs(i int, d device.Device) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetGPUDeviceSpecs is not supported on WSL")
+}
+
+// GetMIGDeviceEdits generates a CDI specification that can be used for MIG devices
+func (l *wsllib) GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetMIGDeviceEdits is not supported on WSL")
+}
+
+// GetMIGDeviceSpecs returns the CDI device specs for the full MIG represented by 'device'.
+func (l *wsllib) GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetMIGDeviceSpecs is not supported on WSL")
+}

pkg/nvcdi/lib.go

@@ -0,0 +1,175 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/info"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+type wrapper struct {
+	Interface
+
+	vendor string
+	class  string
+}
+
+type nvcdilib struct {
+	logger        *logrus.Logger
+	nvmllib       nvml.Interface
+	mode          string
+	devicelib     device.Interface
+	deviceNamer   DeviceNamer
+	driverRoot    string
+	nvidiaCTKPath string
+
+	vendor string
+	class  string
+
+	infolib info.Interface
+}
+
+// New creates a new nvcdi library
+func New(opts ...Option) (Interface, error) {
+	l := &nvcdilib{}
+	for _, opt := range opts {
+		opt(l)
+	}
+	if l.mode == "" {
+		l.mode = ModeAuto
+	}
+	if l.logger == nil {
+		l.logger = logrus.StandardLogger()
+	}
+	if l.deviceNamer == nil {
+		l.deviceNamer, _ = NewDeviceNamer(DeviceNameStrategyIndex)
+	}
+	if l.driverRoot == "" {
+		l.driverRoot = "/"
+	}
+	if l.nvidiaCTKPath == "" {
+		l.nvidiaCTKPath = "/usr/bin/nvidia-ctk"
+	}
+	if l.infolib == nil {
+		l.infolib = info.New()
+	}
+
+	var lib Interface
+	switch l.resolveMode() {
+	case ModeManagement:
+		if l.vendor == "" {
+			l.vendor = "management.nvidia.com"
+		}
+		lib = (*managementlib)(l)
+	case ModeNvml:
+		if l.nvmllib == nil {
+			l.nvmllib = nvml.New()
+		}
+		if l.devicelib == nil {
+			l.devicelib = device.New(device.WithNvml(l.nvmllib))
+		}
+
+		lib = (*nvmllib)(l)
+	case ModeWsl:
+		lib = (*wsllib)(l)
+	case ModeGds:
+		if l.class == "" {
+			l.class = "gds"
+		}
+		lib = (*gdslib)(l)
+	case ModeMofed:
+		if l.class == "" {
+			l.class = "mofed"
+		}
+		lib = (*mofedlib)(l)
+	default:
+		return nil, fmt.Errorf("unknown mode %q", l.mode)
+	}
+
+	w := wrapper{
+		Interface: lib,
+		vendor:    l.vendor,
+		class:     l.class,
+	}
+	return &w, nil
+}
+
+// GetSpec combines the device specs and common edits from the wrapped Interface to a single spec.Interface.
+func (l *wrapper) GetSpec() (spec.Interface, error) {
+	deviceSpecs, err := l.GetAllDeviceSpecs()
+	if err != nil {
+		return nil, err
+	}
+
+	edits, err := l.GetCommonEdits()
+	if err != nil {
+		return nil, err
+	}
+
+	return spec.New(
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*edits.ContainerEdits),
+		spec.WithVendor(l.vendor),
+		spec.WithClass(l.class),
+	)
+
+}
+
+// resolveMode resolves the mode for CDI spec generation based on the current system.
+func (l *nvcdilib) resolveMode() (rmode string) {
+	if l.mode != ModeAuto {
+		return l.mode
+	}
+	defer func() {
+		l.logger.Infof("Auto-detected mode as %q", rmode)
+	}()
+
+	isWSL, reason := l.infolib.HasDXCore()
+	l.logger.Debugf("Is WSL-based system? %v: %v", isWSL, reason)
+
+	if isWSL {
+		return ModeWsl
+	}
+
+	return ModeNvml
+}
+
+// getCudaVersion returns the CUDA version of the current system.
+func (l *nvcdilib) getCudaVersion() (string, error) {
+	if hasNVML, reason := l.infolib.HasNvml(); !hasNVML {
+		return "", fmt.Errorf("nvml not detected: %v", reason)
+	}
+	if l.nvmllib == nil {
+		return "", fmt.Errorf("nvml library not initialized")
+	}
+	r := l.nvmllib.Init()
+	if r != nvml.SUCCESS {
+		return "", fmt.Errorf("failed to initialize nvml: %v", r)
+	}
+	defer l.nvmllib.Shutdown()
+
+	version, r := l.nvmllib.SystemGetDriverVersion()
+	if r != nvml.SUCCESS {
+		return "", fmt.Errorf("failed to get driver version: %v", r)
+	}
+	return version, nil
+}

pkg/nvcdi/lib_test.go

@@ -0,0 +1,88 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveMode(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		mode string
+		// TODO: This should be a proper mock
+		hasDXCore bool
+		expected  string
+	}{
+		{
+			mode:      "auto",
+			hasDXCore: true,
+			expected:  "wsl",
+		},
+		{
+			mode:      "auto",
+			hasDXCore: false,
+			expected:  "nvml",
+		},
+		{
+			mode:      "nvml",
+			hasDXCore: true,
+			expected:  "nvml",
+		},
+		{
+			mode:      "wsl",
+			hasDXCore: false,
+			expected:  "wsl",
+		},
+		{
+			mode:      "not-auto",
+			hasDXCore: true,
+			expected:  "not-auto",
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			l := nvcdilib{
+				logger:  logger,
+				mode:    tc.mode,
+				infolib: infoMock(tc.hasDXCore),
+			}
+
+			require.Equal(t, tc.expected, l.resolveMode())
+		})
+	}
+}
+
+type infoMock bool
+
+func (i infoMock) HasDXCore() (bool, string) {
+	return bool(i), ""
+}
+
+func (i infoMock) HasNvml() (bool, string) {
+	panic("should not be called")
+}
+
+func (i infoMock) IsTegraSystem() (bool, string) {
+	panic("should not be called")
+}

pkg/nvcdi/management.go

@@ -0,0 +1,190 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+)
+
+type managementlib nvcdilib
+
+var _ Interface = (*managementlib)(nil)
+
+// GetAllDeviceSpecs returns all device specs for use in managemnt containers.
+// A single device with the name `all` is returned.
+func (m *managementlib) GetAllDeviceSpecs() ([]specs.Device, error) {
+	devices, err := m.newManagementDeviceDiscoverer()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device discoverer: %v", err)
+	}
+
+	edits, err := edits.FromDiscoverer(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits from discoverer: %v", err)
+	}
+
+	if len(edits.DeviceNodes) == 0 {
+		return nil, fmt.Errorf("no NVIDIA device nodes found")
+	}
+
+	device := specs.Device{
+		Name:           "all",
+		ContainerEdits: *edits.ContainerEdits,
+	}
+	return []specs.Device{device}, nil
+}
+
+// GetCommonEdits returns the common edits for use in managementlib containers.
+func (m *managementlib) GetCommonEdits() (*cdi.ContainerEdits, error) {
+	version, err := m.getCudaVersion()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CUDA version: %v", err)
+	}
+
+	driver, err := newDriverVersionDiscoverer(m.logger, m.driverRoot, m.nvidiaCTKPath, version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create driver library discoverer: %v", err)
+	}
+
+	edits, err := edits.FromDiscoverer(driver)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits from discoverer: %v", err)
+	}
+
+	return edits, nil
+}
+
+// getCudaVersion returns the CUDA version for use in managementlib containers.
+func (m *managementlib) getCudaVersion() (string, error) {
+	version, err := (*nvcdilib)(m).getCudaVersion()
+	if err == nil {
+		return version, nil
+	}
+
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(m.logger),
+		cuda.WithDriverRoot(m.driverRoot),
+	).Locate(".*.*.*")
+	if err != nil {
+		return "", fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+
+	libCudaPath := libCudaPaths[0]
+
+	version = strings.TrimPrefix(filepath.Base(libCudaPath), "libcuda.so.")
+
+	return version, nil
+}
+
+type managementDiscoverer struct {
+	discover.Discover
+}
+
+// newManagementDeviceDiscoverer returns a discover.Discover that discovers device nodes for use in managementlib containers.
+// NVML is not used to query devices and all device nodes are returned.
+func (m *managementlib) newManagementDeviceDiscoverer() (discover.Discover, error) {
+	deviceNodes := discover.NewCharDeviceDiscoverer(
+		m.logger,
+		[]string{
+			"/dev/nvidia*",
+			"/dev/nvidia-caps/nvidia-cap*",
+			"/dev/nvidia-modeset",
+			"/dev/nvidia-uvm-tools",
+			"/dev/nvidia-uvm",
+			"/dev/nvidiactl",
+		},
+		m.driverRoot,
+	)
+
+	deviceFolderPermissionHooks := newDeviceFolderPermissionHookDiscoverer(
+		m.logger,
+		m.driverRoot,
+		m.nvidiaCTKPath,
+		deviceNodes,
+	)
+
+	d := discover.Merge(
+		&managementDiscoverer{deviceNodes},
+		deviceFolderPermissionHooks,
+	)
+	return d, nil
+}
+
+func (m *managementDiscoverer) Devices() ([]discover.Device, error) {
+	devices, err := m.Discover.Devices()
+	if err != nil {
+		return devices, err
+	}
+
+	var filteredDevices []discover.Device
+	for _, device := range devices {
+		if m.nodeIsBlocked(device.HostPath) {
+			continue
+		}
+		filteredDevices = append(filteredDevices, device)
+	}
+
+	return filteredDevices, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m managementDiscoverer) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// GetSpec is unsppported for the managementlib specs.
+// managementlib is typically wrapped by a spec that implements GetSpec.
+func (m *managementlib) GetSpec() (spec.Interface, error) {
+	return nil, fmt.Errorf("GetSpec is not supported")
+}
+
+// GetGPUDeviceEdits is unsupported for the managementlib specs
+func (m *managementlib) GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetGPUDeviceEdits is not supported")
+}
+
+// GetGPUDeviceSpecs is unsupported for the managementlib specs
+func (m *managementlib) GetGPUDeviceSpecs(int, device.Device) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetGPUDeviceSpecs is not supported")
+}
+
+// GetMIGDeviceEdits is unsupported for the managementlib specs
+func (m *managementlib) GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetMIGDeviceEdits is not supported")
+}
+
+// GetMIGDeviceSpecs is unsupported for the managementlib specs
+func (m *managementlib) GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetMIGDeviceSpecs is not supported")
+}

pkg/nvcdi/mig-device-nvml.go

@@ -0,0 +1,124 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// GetMIGDeviceSpecs returns the CDI device specs for the full GPU represented by 'device'.
+func (l *nvmllib) GetMIGDeviceSpecs(i int, d device.Device, j int, mig device.MigDevice) (*specs.Device, error) {
+	edits, err := l.GetMIGDeviceEdits(d, mig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get edits for device: %v", err)
+	}
+
+	name, err := l.deviceNamer.GetMigDeviceName(i, d, j, mig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get device name: %v", err)
+	}
+
+	spec := specs.Device{
+		Name:           name,
+		ContainerEdits: *edits.ContainerEdits,
+	}
+
+	return &spec, nil
+}
+
+// GetMIGDeviceEdits returns the CDI edits for the MIG device represented by 'mig' on 'parent'.
+func (l *nvmllib) GetMIGDeviceEdits(parent device.Device, mig device.MigDevice) (*cdi.ContainerEdits, error) {
+	gpu, ret := parent.GetMinorNumber()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU minor: %v", ret)
+	}
+
+	gi, ret := mig.GetGpuInstanceId()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
+	}
+
+	ci, ret := mig.GetComputeInstanceId()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
+	}
+
+	editsForDevice, err := GetEditsForComputeInstance(l.logger, l.driverRoot, gpu, gi, ci)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for MIG device: %v", err)
+	}
+
+	return editsForDevice, nil
+}
+
+// GetEditsForComputeInstance returns the CDI edits for a particular compute instance defined by the (gpu, gi, ci) tuple
+func GetEditsForComputeInstance(logger *logrus.Logger, driverRoot string, gpu int, gi int, ci int) (*cdi.ContainerEdits, error) {
+	computeInstance, err := newComputeInstanceDiscoverer(logger, driverRoot, gpu, gi, ci)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for Compute Instance: %v", err)
+	}
+
+	editsForDevice, err := edits.FromDiscoverer(computeInstance)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for Compute Instance: %v", err)
+	}
+
+	return editsForDevice, nil
+}
+
+// newComputeInstanceDiscoverer returns a discoverer for the specified compute instance
+func newComputeInstanceDiscoverer(logger *logrus.Logger, driverRoot string, gpu int, gi int, ci int) (discover.Discover, error) {
+	parentPath := fmt.Sprintf("/dev/nvidia%d", gpu)
+
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
+	}
+
+	giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
+	}
+
+	ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
+	}
+
+	deviceNodes := discover.NewCharDeviceDiscoverer(
+		logger,
+		[]string{
+			parentPath,
+			giCapDevicePath,
+			ciCapDevicePath,
+		},
+		driverRoot,
+	)
+
+	return deviceNodes, nil
+}

pkg/nvcdi/mofed.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+)
+
+type mofedlib nvcdilib
+
+var _ Interface = (*mofedlib)(nil)
+
+// GetAllDeviceSpecs returns the device specs for all available devices.
+func (l *mofedlib) GetAllDeviceSpecs() ([]specs.Device, error) {
+	discoverer, err := discover.NewMOFEDDiscoverer(l.logger, l.driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create MOFED discoverer: %v", err)
+	}
+	edits, err := edits.FromDiscoverer(discoverer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for MOFED devices: %v", err)
+	}
+
+	deviceSpec := specs.Device{
+		Name:           "all",
+		ContainerEdits: *edits.ContainerEdits,
+	}
+
+	return []specs.Device{deviceSpec}, nil
+}
+
+// GetCommonEdits generates a CDI specification that can be used for ANY devices
+func (l *mofedlib) GetCommonEdits() (*cdi.ContainerEdits, error) {
+	return edits.FromDiscoverer(discover.None{})
+}
+
+// GetSpec is unsppported for the mofedlib specs.
+// mofedlib is typically wrapped by a spec that implements GetSpec.
+func (l *mofedlib) GetSpec() (spec.Interface, error) {
+	return nil, fmt.Errorf("GetSpec is not supported")
+}
+
+// GetGPUDeviceEdits is unsupported for the mofedlib specs
+func (l *mofedlib) GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetGPUDeviceEdits is not supported")
+}
+
+// GetGPUDeviceSpecs is unsupported for the mofedlib specs
+func (l *mofedlib) GetGPUDeviceSpecs(int, device.Device) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetGPUDeviceSpecs is not supported")
+}
+
+// GetMIGDeviceEdits is unsupported for the mofedlib specs
+func (l *mofedlib) GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetMIGDeviceEdits is not supported")
+}
+
+// GetMIGDeviceSpecs is unsupported for the mofedlib specs
+func (l *mofedlib) GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetMIGDeviceSpecs is not supported")
+}

pkg/nvcdi/namer.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 **/
 
-package generate
+package nvcdi
 
 import (
 	"fmt"
@@ -23,15 +23,20 @@ import (
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
 )
 
-type deviceNamer interface {
+// DeviceNamer is an interface for getting device names
+type DeviceNamer interface {
 	GetDeviceName(int, device.Device) (string, error)
-	GetMigDeviceName(int, int, device.MigDevice) (string, error)
+	GetMigDeviceName(int, device.Device, int, device.MigDevice) (string, error)
 }
 
+// Supported device naming strategies
 const (
-	deviceNameStrategyIndex     = "index"
-	deviceNameStrategyTypeIndex = "type-index"
-	deviceNameStrategyUUID      = "uuid"
+	// DeviceNameStrategyIndex generates devices names such as 0 or 1:0
+	DeviceNameStrategyIndex = "index"
+	// DeviceNameStrategyTypeIndex generates devices names such as gpu0 or mig1:0
+	DeviceNameStrategyTypeIndex = "type-index"
+	// DeviceNameStrategyUUID uses the device UUID as the name
+	DeviceNameStrategyUUID = "uuid"
 )
 
 type deviceNameIndex struct {
@@ -40,15 +45,15 @@ type deviceNameIndex struct {
 }
 type deviceNameUUID struct{}
 
-// newDeviceNamer creates a Device Namer based on the supplied strategy.
+// NewDeviceNamer creates a Device Namer based on the supplied strategy.
 // This namer can be used to construct the names for MIG and GPU devices when generating the CDI spec.
-func newDeviceNamer(strategy string) (deviceNamer, error) {
+func NewDeviceNamer(strategy string) (DeviceNamer, error) {
 	switch strategy {
-	case deviceNameStrategyIndex:
+	case DeviceNameStrategyIndex:
 		return deviceNameIndex{}, nil
-	case deviceNameStrategyTypeIndex:
+	case DeviceNameStrategyTypeIndex:
 		return deviceNameIndex{gpuPrefix: "gpu", migPrefix: "mig"}, nil
-	case deviceNameStrategyUUID:
+	case DeviceNameStrategyUUID:
 		return deviceNameUUID{}, nil
 	}
 
@@ -61,7 +66,7 @@ func (s deviceNameIndex) GetDeviceName(i int, d device.Device) (string, error) {
 }
 
 // GetMigDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameIndex) GetMigDeviceName(i int, j int, d device.MigDevice) (string, error) {
+func (s deviceNameIndex) GetMigDeviceName(i int, d device.Device, j int, mig device.MigDevice) (string, error) {
 	return fmt.Sprintf("%s%d:%d", s.migPrefix, i, j), nil
 }
 
@@ -75,8 +80,8 @@ func (s deviceNameUUID) GetDeviceName(i int, d device.Device) (string, error) {
 }
 
 // GetMigDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameUUID) GetMigDeviceName(i int, j int, d device.MigDevice) (string, error) {
-	uuid, ret := d.GetUUID()
+func (s deviceNameUUID) GetMigDeviceName(i int, d device.Device, j int, mig device.MigDevice) (string, error) {
+	uuid, ret := mig.GetUUID()
 	if ret != nvml.SUCCESS {
 		return "", fmt.Errorf("failed to get device UUID: %v", ret)
 	}

pkg/nvcdi/options.go

@@ -0,0 +1,89 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// Option is a function that configures the nvcdilib
+type Option func(*nvcdilib)
+
+// WithDeviceLib sets the device library for the library
+func WithDeviceLib(devicelib device.Interface) Option {
+	return func(l *nvcdilib) {
+		l.devicelib = devicelib
+	}
+}
+
+// WithDeviceNamer sets the device namer for the library
+func WithDeviceNamer(namer DeviceNamer) Option {
+	return func(l *nvcdilib) {
+		l.deviceNamer = namer
+	}
+}
+
+// WithDriverRoot sets the driver root for the library
+func WithDriverRoot(root string) Option {
+	return func(l *nvcdilib) {
+		l.driverRoot = root
+	}
+}
+
+// WithLogger sets the logger for the library
+func WithLogger(logger *logrus.Logger) Option {
+	return func(l *nvcdilib) {
+		l.logger = logger
+	}
+}
+
+// WithNVIDIACTKPath sets the path to the NVIDIA Container Toolkit CLI path for the library
+func WithNVIDIACTKPath(path string) Option {
+	return func(l *nvcdilib) {
+		l.nvidiaCTKPath = path
+	}
+}
+
+// WithNvmlLib sets the nvml library for the library
+func WithNvmlLib(nvmllib nvml.Interface) Option {
+	return func(l *nvcdilib) {
+		l.nvmllib = nvmllib
+	}
+}
+
+// WithMode sets the discovery mode for the library
+func WithMode(mode string) Option {
+	return func(l *nvcdilib) {
+		l.mode = mode
+	}
+}
+
+// WithVendor sets the vendor for the library
+func WithVendor(vendor string) Option {
+	return func(o *nvcdilib) {
+		o.vendor = vendor
+	}
+}
+
+// WithClass sets the class for the library
+func WithClass(class string) Option {
+	return func(o *nvcdilib) {
+		o.class = class
+	}
+}

pkg/nvcdi/spec/api.go

@@ -0,0 +1,40 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package spec
+
+import (
+	"io"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+const (
+	// DetectMinimumVersion is a constant that triggers a spec to detect the minimum required version.
+	DetectMinimumVersion = "DETECT_MINIMUM_VERSION"
+
+	// FormatJSON indicates a JSON output format
+	FormatJSON = "json"
+	// FormatYAML indicates a YAML output format
+	FormatYAML = "yaml"
+)
+
+// Interface is the interface for the spec API
+type Interface interface {
+	io.WriterTo
+	Save(string) error
+	Raw() *specs.Spec
+}

pkg/nvcdi/spec/builder.go

@@ -0,0 +1,159 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package spec
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type builder struct {
+	raw         *specs.Spec
+	version     string
+	vendor      string
+	class       string
+	deviceSpecs []specs.Device
+	edits       specs.ContainerEdits
+	format      string
+	noSimplify  bool
+}
+
+// newBuilder creates a new spec builder with the supplied options
+func newBuilder(opts ...Option) *builder {
+	s := &builder{}
+	for _, opt := range opts {
+		opt(s)
+	}
+	if s.raw != nil {
+		s.noSimplify = true
+		vendor, class := cdi.ParseQualifier(s.raw.Kind)
+		s.vendor = vendor
+		s.class = class
+	}
+
+	if s.version == "" {
+		s.version = DetectMinimumVersion
+	}
+	if s.vendor == "" {
+		s.vendor = "nvidia.com"
+	}
+	if s.class == "" {
+		s.class = "gpu"
+	}
+	if s.format == "" {
+		s.format = FormatYAML
+	}
+
+	return s
+}
+
+// Build builds a CDI spec form the spec builder.
+func (o *builder) Build() (*spec, error) {
+	raw := o.raw
+	if raw == nil {
+		raw = &specs.Spec{
+			Version:        o.version,
+			Kind:           fmt.Sprintf("%s/%s", o.vendor, o.class),
+			Devices:        o.deviceSpecs,
+			ContainerEdits: o.edits,
+		}
+	}
+
+	if raw.Version == DetectMinimumVersion {
+		minVersion, err := cdi.MinimumRequiredVersion(raw)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get minumum required CDI spec version: %v", err)
+		}
+		raw.Version = minVersion
+	}
+
+	if !o.noSimplify {
+		err := transform.NewSimplifier().Transform(raw)
+		if err != nil {
+			return nil, fmt.Errorf("failed to simplify spec: %v", err)
+		}
+	}
+
+	s := spec{
+		Spec:   raw,
+		format: o.format,
+	}
+
+	return &s, nil
+}
+
+// Option defines a function that can be used to configure the spec builder.
+type Option func(*builder)
+
+// WithDeviceSpecs sets the device specs for the spec builder
+func WithDeviceSpecs(deviceSpecs []specs.Device) Option {
+	return func(o *builder) {
+		o.deviceSpecs = deviceSpecs
+	}
+}
+
+// WithEdits sets the container edits for the spec builder
+func WithEdits(edits specs.ContainerEdits) Option {
+	return func(o *builder) {
+		o.edits = edits
+	}
+}
+
+// WithVersion sets the version for the spec builder
+func WithVersion(version string) Option {
+	return func(o *builder) {
+		o.version = version
+	}
+}
+
+// WithVendor sets the vendor for the spec builder
+func WithVendor(vendor string) Option {
+	return func(o *builder) {
+		o.vendor = vendor
+	}
+}
+
+// WithClass sets the class for the spec builder
+func WithClass(class string) Option {
+	return func(o *builder) {
+		o.class = class
+	}
+}
+
+// WithFormat sets the output file format
+func WithFormat(format string) Option {
+	return func(o *builder) {
+		o.format = format
+	}
+}
+
+// WithNoSimplify sets whether the spec must be simplified
+func WithNoSimplify(noSimplify bool) Option {
+	return func(o *builder) {
+		o.noSimplify = noSimplify
+	}
+}
+
+// WithRawSpec sets the raw spec for the spec builder
+func WithRawSpec(raw *specs.Spec) Option {
+	return func(o *builder) {
+		o.raw = raw
+	}
+}

pkg/nvcdi/spec/spec.go

@@ -0,0 +1,120 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package spec
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type spec struct {
+	*specs.Spec
+	format string
+}
+
+var _ Interface = (*spec)(nil)
+
+// New creates a new spec with the specified options.
+func New(opts ...Option) (Interface, error) {
+	return newBuilder(opts...).Build()
+}
+
+// Save writes the spec to the specified path and overwrites the file if it exists.
+func (s *spec) Save(path string) error {
+	path, err := s.normalizePath(path)
+	if err != nil {
+		return fmt.Errorf("failed to normalize path: %w", err)
+	}
+
+	specDir := filepath.Dir(path)
+	registry := cdi.GetRegistry(
+		cdi.WithAutoRefresh(false),
+		cdi.WithSpecDirs(specDir),
+	)
+
+	return registry.SpecDB().WriteSpec(s.Raw(), filepath.Base(path))
+}
+
+// WriteTo writes the spec to the specified writer.
+func (s *spec) WriteTo(w io.Writer) (int64, error) {
+	name, err := cdi.GenerateNameForSpec(s.Raw())
+	if err != nil {
+		return 0, err
+	}
+
+	path, _ := s.normalizePath(name)
+	tmpFile, err := os.CreateTemp("", "*"+filepath.Base(path))
+	if err != nil {
+		return 0, err
+	}
+	defer os.Remove(tmpFile.Name())
+
+	if err := s.Save(tmpFile.Name()); err != nil {
+		return 0, err
+	}
+
+	err = tmpFile.Close()
+	if err != nil {
+		return 0, fmt.Errorf("failed to close temporary file: %w", err)
+	}
+
+	r, err := os.Open(tmpFile.Name())
+	if err != nil {
+		return 0, fmt.Errorf("failed to open temporary file: %w", err)
+	}
+	defer r.Close()
+
+	return io.Copy(w, r)
+}
+
+// Raw returns a pointer to the raw spec.
+func (s *spec) Raw() *specs.Spec {
+	return s.Spec
+}
+
+// normalizePath ensures that the specified path has a supported extension
+func (s *spec) normalizePath(path string) (string, error) {
+	if ext := filepath.Ext(path); ext != ".yaml" && ext != ".json" {
+		path += s.extension()
+	}
+
+	if filepath.Clean(filepath.Dir(path)) == "." {
+		pwd, err := os.Getwd()
+		if err != nil {
+			return path, fmt.Errorf("failed to get current working directory: %v", err)
+		}
+		path = filepath.Join(pwd, path)
+	}
+
+	return path, nil
+}
+
+func (s *spec) extension() string {
+	switch s.format {
+	case FormatJSON:
+		return ".json"
+	case FormatYAML:
+		return ".yaml"
+	}
+
+	return ".yaml"
+}

pkg/nvcdi/transform/api.go

@@ -0,0 +1,24 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+
+// Transformer defines the API for applying arbitrary transforms to a spec in-place
+type Transformer interface {
+	Transform(*specs.Spec) error
+}

pkg/nvcdi/transform/deduplicate.go

@@ -0,0 +1,151 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type dedupe struct{}
+
+var _ Transformer = (*dedupe)(nil)
+
+// NewDedupe creates a transformer that deduplicates container edits.
+func NewDedupe() (Transformer, error) {
+	return &dedupe{}, nil
+}
+
+// Transform removes duplicate entris from devices and common container edits.
+func (d dedupe) Transform(spec *specs.Spec) error {
+	if spec == nil {
+		return nil
+	}
+	if err := d.transformEdits(&spec.ContainerEdits); err != nil {
+		return err
+	}
+	var updatedDevices []specs.Device
+	for _, device := range spec.Devices {
+		if err := d.transformEdits(&device.ContainerEdits); err != nil {
+			return err
+		}
+		updatedDevices = append(updatedDevices, device)
+	}
+	spec.Devices = updatedDevices
+	return nil
+}
+
+func (d dedupe) transformEdits(edits *specs.ContainerEdits) error {
+	deviceNodes, err := d.deduplicateDeviceNodes(edits.DeviceNodes)
+	if err != nil {
+		return err
+	}
+	edits.DeviceNodes = deviceNodes
+
+	envs, err := d.deduplicateEnvs(edits.Env)
+	if err != nil {
+		return err
+	}
+	edits.Env = envs
+
+	hooks, err := d.deduplicateHooks(edits.Hooks)
+	if err != nil {
+		return err
+	}
+	edits.Hooks = hooks
+
+	mounts, err := d.deduplicateMounts(edits.Mounts)
+	if err != nil {
+		return err
+	}
+	edits.Mounts = mounts
+
+	return nil
+}
+
+func (d dedupe) deduplicateDeviceNodes(entities []*specs.DeviceNode) ([]*specs.DeviceNode, error) {
+	seen := make(map[string]bool)
+	var deviceNodes []*specs.DeviceNode
+	for _, e := range entities {
+		if e == nil {
+			continue
+		}
+		id, err := deviceNode(*e).id()
+		if err != nil {
+			return nil, err
+		}
+		if seen[id] {
+			continue
+		}
+		seen[id] = true
+		deviceNodes = append(deviceNodes, e)
+	}
+	return deviceNodes, nil
+}
+
+func (d dedupe) deduplicateEnvs(entities []string) ([]string, error) {
+	seen := make(map[string]bool)
+	var envs []string
+	for _, e := range entities {
+		id := e
+		if seen[id] {
+			continue
+		}
+		seen[id] = true
+		envs = append(envs, e)
+	}
+	return envs, nil
+}
+
+func (d dedupe) deduplicateHooks(entities []*specs.Hook) ([]*specs.Hook, error) {
+	seen := make(map[string]bool)
+	var hooks []*specs.Hook
+	for _, e := range entities {
+		if e == nil {
+			continue
+		}
+		id, err := hook(*e).id()
+		if err != nil {
+			return nil, err
+		}
+		if seen[id] {
+			continue
+		}
+		seen[id] = true
+		hooks = append(hooks, e)
+	}
+	return hooks, nil
+}
+
+func (d dedupe) deduplicateMounts(entities []*specs.Mount) ([]*specs.Mount, error) {
+	seen := make(map[string]bool)
+	var mounts []*specs.Mount
+	for _, e := range entities {
+		if e == nil {
+			continue
+		}
+		id, err := mount(*e).id()
+		if err != nil {
+			return nil, err
+		}
+		if seen[id] {
+			continue
+		}
+		seen[id] = true
+		mounts = append(mounts, e)
+	}
+	return mounts, nil
+}

pkg/nvcdi/transform/deduplicate_test.go

@@ -0,0 +1,250 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"testing"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeduplicate(t *testing.T) {
+	testCases := []struct {
+		description   string
+		spec          *specs.Spec
+		expectedError error
+		expectedSpec  *specs.Spec
+	}{
+		{
+			description: "nil spec",
+		},
+		{
+			description: "duplicate deviceNode is removed",
+			spec: &specs.Spec{
+				ContainerEdits: specs.ContainerEdits{
+					DeviceNodes: []*specs.DeviceNode{
+						{
+							Path: "/dev/gpu0",
+						},
+						{
+							Path: "/dev/gpu0",
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				ContainerEdits: specs.ContainerEdits{
+					DeviceNodes: []*specs.DeviceNode{
+						{
+							Path: "/dev/gpu0",
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "duplicate deviceNode is remved from device edits",
+			spec: &specs.Spec{
+				Devices: []specs.Device{
+					{
+						Name: "device0",
+						ContainerEdits: specs.ContainerEdits{
+							DeviceNodes: []*specs.DeviceNode{
+								{
+									Path: "/dev/gpu0",
+								},
+								{
+									Path: "/dev/gpu0",
+								},
+							},
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Devices: []specs.Device{
+					{
+						Name: "device0",
+						ContainerEdits: specs.ContainerEdits{
+							DeviceNodes: []*specs.DeviceNode{
+								{
+									Path: "/dev/gpu0",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "duplicate hook is removed",
+			spec: &specs.Spec{
+				ContainerEdits: specs.ContainerEdits{
+					Hooks: []*specs.Hook{
+						{
+							HookName: "createContainer",
+							Path:     "/usr/bin/nvidia-ctk",
+							Args:     []string{"nvidia-ctk", "hook", "chmod", "--mode", "755", "--path", "/dev/dri"},
+						},
+						{
+							HookName: "createContainer",
+							Path:     "/usr/bin/nvidia-ctk",
+							Args:     []string{"nvidia-ctk", "hook", "chmod", "--mode", "755", "--path", "/dev/dri"},
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				ContainerEdits: specs.ContainerEdits{
+					Hooks: []*specs.Hook{
+						{
+							HookName: "createContainer",
+							Path:     "/usr/bin/nvidia-ctk",
+							Args:     []string{"nvidia-ctk", "hook", "chmod", "--mode", "755", "--path", "/dev/dri"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "duplicate mount is removed",
+			spec: &specs.Spec{
+				Devices: []specs.Device{
+					{
+						Name: "device0",
+						ContainerEdits: specs.ContainerEdits{
+							Mounts: []*specs.Mount{
+								{
+									HostPath:      "/host/mount2",
+									ContainerPath: "/mount2",
+								},
+								{
+									HostPath:      "/host/mount2",
+									ContainerPath: "/mount2",
+								},
+								{
+									HostPath:      "/host/mount1",
+									ContainerPath: "/mount1",
+								},
+							},
+						},
+					},
+				},
+				ContainerEdits: specs.ContainerEdits{
+					Mounts: []*specs.Mount{
+						{
+							HostPath:      "/host/mount1",
+							ContainerPath: "/mount1",
+							Options:       []string{"bind", "ro"},
+							Type:          "tmpfs",
+						},
+						{
+							HostPath:      "/host/mount1",
+							ContainerPath: "/mount1",
+							Options:       []string{"bind", "ro"},
+							Type:          "tmpfs",
+						},
+						{
+							HostPath:      "/host/mount1",
+							ContainerPath: "/mount1",
+							Options:       []string{"bind", "ro"},
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Devices: []specs.Device{
+					{
+						Name: "device0",
+						ContainerEdits: specs.ContainerEdits{
+							Mounts: []*specs.Mount{
+								{
+									HostPath:      "/host/mount2",
+									ContainerPath: "/mount2",
+								},
+								{
+									HostPath:      "/host/mount1",
+									ContainerPath: "/mount1",
+								},
+							},
+						},
+					},
+				},
+				ContainerEdits: specs.ContainerEdits{
+					Mounts: []*specs.Mount{
+						{
+							HostPath:      "/host/mount1",
+							ContainerPath: "/mount1",
+							Options:       []string{"bind", "ro"},
+							Type:          "tmpfs",
+						},
+						{
+							HostPath:      "/host/mount1",
+							ContainerPath: "/mount1",
+							Options:       []string{"bind", "ro"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "duplicate env is removed",
+			spec: &specs.Spec{
+				Devices: []specs.Device{
+					{
+						Name: "device0",
+						ContainerEdits: specs.ContainerEdits{
+							Env: []string{"ENV1=VAL1", "ENV1=VAL1", "ENV2=ONE_VALUE", "ENV2=ANOTHER_VALUE"},
+						},
+					},
+				},
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"ENV1=VAL1", "ENV1=VAL1", "ENV2=ONE_VALUE", "ENV2=ANOTHER_VALUE"},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Devices: []specs.Device{
+					{
+						Name: "device0",
+						ContainerEdits: specs.ContainerEdits{
+							Env: []string{"ENV1=VAL1", "ENV2=ONE_VALUE", "ENV2=ANOTHER_VALUE"},
+						},
+					},
+				},
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"ENV1=VAL1", "ENV2=ONE_VALUE", "ENV2=ANOTHER_VALUE"},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			d := dedupe{}
+
+			err := d.Transform(tc.spec)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			require.EqualValues(t, tc.expectedSpec, tc.spec)
+		})
+	}
+}

pkg/nvcdi/transform/edits.go

@@ -0,0 +1,166 @@
+/*
+*
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"encoding/json"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type containerEdits specs.ContainerEdits
+
+// IsEmpty returns true if the edits are empty.
+func (e containerEdits) IsEmpty() bool {
+	// Devices with empty edits are invalid
+	if len(e.DeviceNodes) > 0 {
+		return false
+	}
+	if len(e.Env) > 0 {
+		return false
+	}
+	if len(e.Hooks) > 0 {
+		return false
+	}
+	if len(e.Mounts) > 0 {
+		return false
+	}
+
+	return true
+}
+
+func (e *containerEdits) getEntityIds() ([]string, error) {
+	if e == nil {
+		return nil, nil
+	}
+	uniqueIDs := make(map[string]bool)
+
+	deviceNodes, err := e.getDeviceNodeIDs()
+	if err != nil {
+		return nil, err
+	}
+	for k := range deviceNodes {
+		uniqueIDs[k] = true
+	}
+
+	envs, err := e.getEnvIDs()
+	if err != nil {
+		return nil, err
+	}
+	for k := range envs {
+		uniqueIDs[k] = true
+	}
+
+	hooks, err := e.getHookIDs()
+	if err != nil {
+		return nil, err
+	}
+	for k := range hooks {
+		uniqueIDs[k] = true
+	}
+
+	mounts, err := e.getMountIDs()
+	if err != nil {
+		return nil, err
+	}
+	for k := range mounts {
+		uniqueIDs[k] = true
+	}
+
+	var ids []string
+	for k := range uniqueIDs {
+		ids = append(ids, k)
+	}
+
+	return ids, nil
+}
+
+func (e *containerEdits) getDeviceNodeIDs() (map[string]bool, error) {
+	deviceIDs := make(map[string]bool)
+	for _, entity := range e.DeviceNodes {
+		id, err := deviceNode(*entity).id()
+		if err != nil {
+			return nil, err
+		}
+		deviceIDs[id] = true
+	}
+	return deviceIDs, nil
+}
+
+func (e *containerEdits) getEnvIDs() (map[string]bool, error) {
+	envIDs := make(map[string]bool)
+	for _, entity := range e.Env {
+		id, err := env(entity).id()
+		if err != nil {
+			return nil, err
+		}
+		envIDs[id] = true
+	}
+	return envIDs, nil
+}
+
+func (e *containerEdits) getHookIDs() (map[string]bool, error) {
+	hookIDs := make(map[string]bool)
+	for _, entity := range e.Hooks {
+		id, err := hook(*entity).id()
+		if err != nil {
+			return nil, err
+		}
+		hookIDs[id] = true
+	}
+	return hookIDs, nil
+}
+
+func (e *containerEdits) getMountIDs() (map[string]bool, error) {
+	mountIDs := make(map[string]bool)
+	for _, entity := range e.Mounts {
+		id, err := mount(*entity).id()
+		if err != nil {
+			return nil, err
+		}
+		mountIDs[id] = true
+	}
+	return mountIDs, nil
+}
+
+type deviceNode specs.DeviceNode
+
+func (dn deviceNode) id() (string, error) {
+	b, err := json.Marshal(dn)
+	return string(b), err
+}
+
+type env string
+
+func (e env) id() (string, error) {
+	return string(e), nil
+}
+
+type mount specs.Mount
+
+func (m mount) id() (string, error) {
+	b, err := json.Marshal(m)
+	return string(b), err
+}
+
+type hook specs.Hook
+
+func (m hook) id() (string, error) {
+	b, err := json.Marshal(m)
+	return string(b), err
+}

pkg/nvcdi/transform/no-op.go

@@ -0,0 +1,35 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type noop struct{}
+
+var _ Transformer = (*noop)(nil)
+
+// NewNoopTransformer returns a no-op transformer
+func NewNoopTransformer() Transformer {
+	return noop{}
+}
+
+// Transform is a no-op
+func (n noop) Transform(spec *specs.Spec) error {
+	return nil
+}

pkg/nvcdi/transform/remove.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"fmt"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type remove map[string]bool
+
+func newRemover(ids ...string) Transformer {
+	r := make(remove)
+	for _, id := range ids {
+		r[id] = true
+	}
+	return r
+}
+
+// Transform remove the specified entities from the spec.
+func (r remove) Transform(spec *specs.Spec) error {
+	if spec == nil {
+		return nil
+	}
+
+	for _, device := range spec.Devices {
+		if err := r.transformEdits(&device.ContainerEdits); err != nil {
+			return fmt.Errorf("failed to remove edits from device %q: %w", device.Name, err)
+		}
+	}
+
+	return r.transformEdits(&spec.ContainerEdits)
+}
+
+func (r remove) transformEdits(edits *specs.ContainerEdits) error {
+	if edits == nil {
+		return nil
+	}
+
+	var deviceNodes []*specs.DeviceNode
+	for _, entity := range edits.DeviceNodes {
+		id, err := deviceNode(*entity).id()
+		if err != nil {
+			return err
+		}
+		if r[id] {
+			continue
+		}
+		deviceNodes = append(deviceNodes, entity)
+	}
+	edits.DeviceNodes = deviceNodes
+
+	var envs []string
+	for _, entity := range edits.Env {
+		id := entity
+		if r[id] {
+			continue
+		}
+		envs = append(envs, entity)
+	}
+	edits.Env = envs
+
+	var hooks []*specs.Hook
+	for _, entity := range edits.Hooks {
+		id, err := hook(*entity).id()
+		if err != nil {
+			return err
+		}
+		if r[id] {
+			continue
+		}
+		hooks = append(hooks, entity)
+	}
+	edits.Hooks = hooks
+
+	var mounts []*specs.Mount
+	for _, entity := range edits.Mounts {
+		id, err := mount(*entity).id()
+		if err != nil {
+			return err
+		}
+		if r[id] {
+			continue
+		}
+		mounts = append(mounts, entity)
+	}
+	edits.Mounts = mounts
+
+	return nil
+}