Merge branch 'fix-libnvidia-container-tag' into 'main'

Fix setting of LIBNVIDIA_CONTAINER_TAG

See merge request nvidia/container-toolkit/container-toolkit!201

.common-ci.yml

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker:stable
+  image: docker
   services:
-    - name: docker:stable-dind
+    - name: docker:dind
       command: ["--experimental"]
 
 variables:
@@ -210,13 +210,6 @@ release:staging-centos7:
   needs:
     - image-centos7
 
-release:staging-centos8:
-  extends:
-    - .release:staging
-    - .dist-centos8
-  needs:
-    - image-centos8
-
 release:staging-ubi8:
   extends:
     - .release:staging

.gitlab-ci.yml

@@ -231,16 +231,6 @@ image-centos7:
     - package-centos7-ppc64le
     - package-centos7-x86_64
 
-image-centos8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos8
-  needs:
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
 image-ubi8:
   extends:
     - .image-build

.gitmodules

@@ -5,6 +5,8 @@
 [submodule "third_party/nvidia-container-runtime"]
 	path = third_party/nvidia-container-runtime
 	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
+	branch = main
 [submodule "third_party/nvidia-docker"]
 	path = third_party/nvidia-docker
 	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
+	branch = main

.nvidia-ci.yml

@@ -70,11 +70,6 @@ image-centos7:
     - .image-pull
     - .dist-centos7
 
-image-centos8:
-  extends:
-    - .image-pull
-    - .dist-centos8
-
 image-ubi8:
   extends:
     - .image-pull
@@ -154,23 +149,6 @@ scan-centos7-arm64:
     - image-centos7
     - scan-centos7-amd64
 
-scan-centos8-amd64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-amd64
-  needs:
-    - image-centos8
-
-scan-centos8-arm64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-arm64
-  needs:
-    - image-centos8
-    - scan-centos8-amd64
-
 scan-ubuntu18.04-amd64:
   extends:
     - .scan
@@ -179,15 +157,6 @@ scan-ubuntu18.04-amd64:
   needs:
     - image-ubuntu18.04
 
-scan-ubuntu18.04-arm64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-arm64
-  needs:
-    - image-ubuntu18.04
-    - scan-ubuntu18.04-amd64
-
 scan-ubuntu20.04-amd64:
   extends:
     - .scan
@@ -253,11 +222,6 @@ release:ngc-centos7:
     - .release:ngc
     - .dist-centos7
 
-release:ngc-centos8:
-  extends:
-    - .release:ngc
-    - .dist-centos8
-
 release:ngc-ubuntu18.04:
   extends:
     - .release:ngc

CHANGELOG.md

@@ -1,5 +1,22 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.11.1-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.1-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
+## v1.10.0
+
+* Promote v1.10.0-rc.3 to v1.10.0
+
 ## v1.10.0-rc.3
 
 * Use default config instead of raising an error if config file cannot be found

Makefile

@@ -39,7 +39,7 @@ CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
 CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate licenses $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
@@ -102,6 +102,9 @@ misspell:
 vet:
 	go vet $(MODULE)/...
 
+licenses:
+	go-licenses csv $(MODULE)/...
+
 COVERAGE_FILE := coverage.out
 test: build cmds
 	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...

build/container/Dockerfile.centos

@@ -67,8 +67,8 @@ ARG TARGETARCH
 ENV PACKAGE_ARCH ${TARGETARCH}
 RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
     yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
     ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
@@ -85,7 +85,7 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-COPY ./LICENSE /licenses/LICENSE
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES

build/container/Dockerfile.packaging

@@ -26,4 +26,4 @@ COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
-COPY ./LICENSE /licenses/LICENSE
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

build/container/Dockerfile.ubuntu

@@ -75,8 +75,8 @@ RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
     fi
 
 RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
     ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
@@ -93,7 +93,7 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-COPY ./LICENSE /licenses/LICENSE
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES

build/container/Makefile

@@ -43,8 +43,8 @@ OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu18.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+DEFAULT_PUSH_TARGET := ubuntu20.04
+DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7
 
 META_TARGETS := packaging
 
@@ -110,10 +110,10 @@ build-ubi8: DOCKERFILE_SUFFIX := centos
 build-ubi8: PACKAGE_DIST = centos8
 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
-build-centos%: BASE_DIST = $(*)
-build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: PACKAGE_DIST = $(BASE_DIST)
-build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-centos7: BASE_DIST = $(*)
+build-centos7: DOCKERFILE_SUFFIX := centos
+build-centos7: PACKAGE_DIST = $(BASE_DIST)
+build-centos7: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging

build/container/multi-arch.mk

@@ -30,5 +30,8 @@ push-short:
 # We only have x86_64 packages for centos7
 build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
 
+# We only generate amd64 image for ubuntu18.04
+build-ubuntu18.04: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
 # We only generate a single image for packaging targets
 build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64

cmd/nvidia-container-runtime-hook/container_config.go

@@ -165,7 +165,7 @@ func isPrivileged(s *Spec) bool {
 	return false
 }
 
-func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
+func getDevicesFromEnvvar(image image.CUDA) *string {
 	// Build a list of envvars to consider.
 	envVars := []string{envNVVisibleDevices}
 	if envSwarmGPU != nil {
@@ -173,35 +173,14 @@ func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
 		envVars = append([]string{*envSwarmGPU}, envVars...)
 	}
 
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
-	for _, envVar := range envVars {
-		if devs, ok := env[envVar]; ok {
-			devices = &devs
-			break
-		}
-	}
-
-	// Environment variable unset with legacy image: default to "all".
-	if devices == nil && legacyImage {
-		all := "all"
-		return &all
-	}
-
-	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
+	devices := image.DevicesFromEnvvars(envVars...)
+	if len(devices) == 0 {
 		return nil
 	}
 
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		empty := ""
-		return &empty
-	}
+	devicesString := strings.Join(devices, ",")
 
-	// Any other value.
-	return devices
+	return &devicesString
 }
 
 func getDevicesFromMounts(mounts []Mount) *string {
@@ -241,7 +220,7 @@ func getDevicesFromMounts(mounts []Mount) *string {
 	return &ret
 }
 
-func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
+func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
 		devices := getDevicesFromMounts(mounts)
@@ -251,7 +230,7 @@ func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, p
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(env, legacyImage)
+	devices := getDevicesFromEnvvar(image)
 	if devices == nil {
 		return nil
 	}
@@ -307,7 +286,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	legacyImage := image.IsLegacy()
 
 	var devices string
-	if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil {
+	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
 		devices = *d
 	} else {
 		// 'nil' devices means this is not a GPU container.

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -4,6 +4,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -671,7 +672,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				hookConfig := getDefaultHookConfig()
 				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
+				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)
 			}
 
 			// For all other tests, just grab the devices and check the results
@@ -693,7 +694,6 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 		description     string
 		envSwarmGPU     *string
 		env             map[string]string
-		legacyImage     bool
 		expectedDevices *string
 	}{
 		{
@@ -729,13 +729,15 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
 			env: map[string]string{
 				envNVVisibleDevices: gpuID,
+				envCUDAVersion:      "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description:     "empty env returns all for legacy image",
-			legacyImage:     true,
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envCUDAVersion: "legacy",
+			},
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
@@ -781,16 +783,16 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
 			description: "empty env returns all for legacy image",
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
@@ -834,8 +836,8 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
@@ -860,7 +862,7 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 	for i, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			envSwarmGPU = tc.envSwarmGPU
-			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
+			devices := getDevicesFromEnvvar(image.CUDA(tc.env))
 			if tc.expectedDevices == nil {
 				require.Nil(t, devices, "%d: %v", i, tc)
 				return

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -34,7 +34,7 @@ type CLIConfig struct {
 	Ldconfig    *string  `toml:"ldconfig"`
 }
 
-// HookConfig : options for the nvidia-container-toolkit.
+// HookConfig : options for the nvidia-container-runtime-hook.
 type HookConfig struct {
 	DisableRequire                 bool               `toml:"disable-require"`
 	SwarmResource                  *string            `toml:"swarm-resource"`

cmd/nvidia-container-runtime-hook/main.go

@@ -75,7 +75,7 @@ func doPrestart() {
 	cli := hook.NvidiaContainerCLI
 
 	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
 	}
 
 	container := getContainerConfig(hook)

cmd/nvidia-container-runtime/README.md

@@ -2,24 +2,86 @@
 
 The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
 
-## Standard Mode
+## Configuration
 
-In the standard mode configuration, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from project [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+The NVIDIA Container Runtime uses file-based configuration, with the config stored in `/etc/nvidia-container-runtime/config.toml`. The `/etc` path can be overridden using the `XDG_CONFIG_HOME` environment variable with the `${XDG_CONFIG_HOME}/nvidia-container-runtime/config.toml` file used instead if this environment variable is set.
 
-## Experimental Mode
+This config file may contain options for other components of the NVIDIA container stack and for the NVIDIA Container Runtime, the relevant config section is `nvidia-container-runtime`
 
-The NVIDIA Container Runtime can be configured in an experimental mode by setting the following options in the runtime's `config.toml` file:
+### Logging
+
+The `log-level` config option (default: `"info"`) specifies the log level to use and the `debug` option, if set, specifies a log file to which logs for the NVIDIA Container Runtime must be written.
+
+In addition to this, the NVIDIA Container Runtime considers the value of `--log` and `--log-format` flags that may be passed to it by a container runtime such as docker or containerd. If the `--debug` flag is present the log-level specified in the config file is overridden as `"debug"`.
+
+### Low-level Runtime Path
+
+The `runtimes` config option allows for the low-level runtime to be specified. The first entry in this list that is an existing executable file is used as the low-level runtime. If the entry is not a path, the `PATH` is searched for a matching executable. If the entry is a path this is checked instead.
+
+The default value for this setting is:
+```toml
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+```
+
+and if, for example, `crun` is to be used instead this can be changed to:
+```toml
+runtimes = [
+    "crun",
+]
+```
+
+### Runtime Mode
+
+The `mode` config option (default `"auto"`) controls the high-level behaviour of the runtime.
+
+#### Auto Mode
+
+When `mode` is set to `"auto"`, the runtime employs heuristics to determine which mode to use based on, for example, the platform where the runtime is being run.
+
+#### Legacy Mode
+
+When `mode` is set to `"legacy"`, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from the [libnvidia-container](https://github.com/NVIDIA/libnvidia-container) project.
+
+#### CSV Mode
+
+When `mode` is set to `"csv"`, CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` define the devices and mounts that are to be injected into a container when it is created. The search path for the files can be overridden by modifying the `nvidia-container-runtime.modes.csv.mount-spec-path` in the config as below:
 
 ```toml
 [nvidia-container-runtime]
-experimental = true
+    [nvidia-container-runtime.modes.csv]
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
 ```
 
-When this setting is enabled, the modifications made to the OCI specification are controlled by the `nvidia-container-runtime.discover-mode` option, with the following mode supported:
-
-* `"legacy"`: This mode mirrors the behaviour of the standard mode, inserting the NVIDIA Container Runtime Hook as a `prestart` hook into the container's OCI specification.
-* `"csv"`: This mode uses CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` to define the devices and mounts that are to be injected into a container when it is created.
+This mode is primarily targeted at Tegra-based systems without NVML available.
 
 ### Notes on using the docker CLI
 
-The `docker` CLI supports the `--gpus` flag to select GPUs for inclusion in a container. Since specifying this flag inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification. When experimental mode is activated, the NVIDIA Container Runtime detects the presence of the hook and raises an error. This requirement will be relaxed in the near future.
+Note that only the `"legacy"` NVIDIA Container Runtime mode is directly compatible with the `--gpus` flag implemented by the `docker` CLI (assuming the NVIDIA Container Runtime is not used). The reason for this is that `docker` inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification.
+
+
+If a different mode is explicitly set or detected, the NVIDIA Container Runtime Hook will raise the following error when `--gpus` is set:
+```
+$ docker run --rm --gpus all ubuntu:18.04
+docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'csv'
+invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.: unknown.
+```
+Here NVIDIA Container Runtime must be used explicitly. The recommended way to do this is to specify the `--runtime=nvidia` command line argument as part of the `docker run` commmand as follows:
+```
+$ docker run --rm --gpus all --runtime=nvidia ubuntu:18.04
+```
+
+Alternatively the NVIDIA Container Runtime can be set as the default runtime for docker. This can be done by modifying the `/etc/docker/daemon.json` file as follows:
+```json
+{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}
+```

cmd/nvidia-container-runtime/main_test.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/require"

cmd/nvidia-container-runtime/runtime_factory.go

@@ -19,9 +19,9 @@ package main
 import (
 	"fmt"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 	"github.com/sirupsen/logrus"
@@ -62,11 +62,37 @@ func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv [
 
 // newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
 func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
+	modeModifier, err := newModeModifier(logger, cfg, ociSpec, argv)
+	if err != nil {
+		return nil, err
+	}
+
+	gdsModifier, err := modifier.NewGDSModifier(logger, cfg, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	mofedModifier, err := modifier.NewMOFEDModifier(logger, cfg, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	modifiers := modifier.Merge(
+		modeModifier,
+		gdsModifier,
+		mofedModifier,
+	)
+	return modifiers, nil
+}
+
+func newModeModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
 	switch info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode) {
 	case "legacy":
 		return modifier.NewStableRuntimeModifier(logger), nil
 	case "csv":
 		return modifier.NewCSVModifier(logger, cfg, ociSpec)
+	case "cdi":
+		return modifier.NewCDIModifier(logger, cfg, ociSpec)
 	}
 
 	return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)

cmd/nvidia-ctk/README.md

@@ -1,3 +1,17 @@
 # NVIDIA Container Toolkit CLI
 
 The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.

cmd/nvidia-ctk/main.go

@@ -20,6 +20,7 @@ import (
 	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
@@ -70,6 +71,7 @@ func main() {
 	// Define the subcommands
 	c.Commands = []*cli.Command{
 		hook.NewCommand(logger),
+		runtime.NewCommand(logger),
 	}
 
 	// Run the CLI

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -0,0 +1,154 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package configure
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultRuntime = "docker"
+
+	defaultDockerConfigFilePath = "/etc/docker/daemon.json"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an configure command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line config
+type config struct {
+	dryRun         bool
+	runtime        string
+	configFilePath string
+	nvidiaOptions  nvidia.Options
+}
+
+func (m command) build() *cli.Command {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the 'configure' command
+	configure := cli.Command{
+		Name:  "configure",
+		Usage: "Add a runtime to the specified container engine",
+		Action: func(c *cli.Context) error {
+			return m.configureWrapper(c, &config)
+		},
+	}
+
+	configure.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "update the runtime configuration as required but don't write changes to disk",
+			Destination: &config.dryRun,
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Usage:       "the target runtime engine. One of [docker]",
+			Value:       defaultRuntime,
+			Destination: &config.runtime,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to the config file for the target runtime",
+			Destination: &config.configFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-name",
+			Usage:       "specify the name of the NVIDIA runtime that will be added",
+			Value:       nvidia.RuntimeName,
+			Destination: &config.nvidiaOptions.RuntimeName,
+		},
+		&cli.StringFlag{
+			Name:        "runtime-path",
+			Usage:       "specify the path to the NVIDIA runtime executable",
+			Value:       nvidia.RuntimeExecutable,
+			Destination: &config.nvidiaOptions.RuntimePath,
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Usage:       "set the specified runtime as the default runtime",
+			Destination: &config.nvidiaOptions.SetAsDefault,
+		},
+	}
+
+	return &configure
+}
+
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.runtime {
+	case "docker":
+		return m.configureDocker(c, config)
+	}
+
+	return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+}
+
+// configureDocker updates the docker config to enable the NVIDIA Container Runtime
+func (m command) configureDocker(c *cli.Context, config *config) error {
+	configFilePath := config.configFilePath
+	if configFilePath == "" {
+		configFilePath = defaultDockerConfigFilePath
+	}
+
+	cfg, err := docker.LoadConfig(configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	defaultRuntime := config.nvidiaOptions.DefaultRuntime()
+	runtimeConfig := config.nvidiaOptions.Runtime().DockerRuntimesConfig()
+	err = docker.UpdateConfig(cfg, defaultRuntime, runtimeConfig)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.dryRun {
+		output, err := json.MarshalIndent(cfg, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to convert to JSON: %v", err)
+		}
+		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
+		return nil
+	}
+	err = docker.FlushConfig(cfg, configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	m.logger.Infof("It is recommended that the docker daemon be restarted.")
+
+	return nil
+}

cmd/nvidia-ctk/runtime/nvidia/nvidia.go

@@ -0,0 +1,75 @@
+/*
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package nvidia
+
+const (
+	// RuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	RuntimeName = "nvidia"
+	// RuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	RuntimeExecutable = "nvidia-container-runtime"
+)
+
+// Options specifies the options for the NVIDIA Container Runtime w.r.t a container engine such as docker.
+type Options struct {
+	SetAsDefault bool
+	RuntimeName  string
+	RuntimePath  string
+}
+
+// Runtime defines an NVIDIA runtime with a name and a executable
+type Runtime struct {
+	Name string
+	Path string
+}
+
+// DefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o Options) DefaultRuntime() string {
+	if !o.SetAsDefault {
+		return ""
+	}
+
+	return o.RuntimeName
+}
+
+// Runtime creates a runtime struct based on the options.
+func (o Options) Runtime() Runtime {
+	path := o.RuntimePath
+
+	if o.RuntimePath == "" {
+		path = RuntimeExecutable
+	}
+
+	r := Runtime{
+		Name: o.RuntimeName,
+		Path: path,
+	}
+
+	return r
+}
+
+// DockerRuntimesConfig generatest the expected docker config for the specified runtime
+func (r Runtime) DockerRuntimesConfig() map[string]interface{} {
+	runtimes := make(map[string]interface{})
+	runtimes[r.Name] = map[string]interface{}{
+		"path": r.Path,
+		"args": []string{},
+	}
+
+	return runtimes
+}

cmd/nvidia-ctk/runtime/runtime.go

@@ -0,0 +1,49 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type runtimeCommand struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := runtimeCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m runtimeCommand) build() *cli.Command {
+	// Create the 'runtime' command
+	runtime := cli.Command{
+		Name:  "runtime",
+		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	runtime.Subcommands = []*cli.Command{
+		configure.NewCommand(m.logger),
+	}
+
+	return &runtime
+}

config/config.toml.amazonlinux

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.centos

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.debian

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.opensuse-leap

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu

@@ -16,7 +16,7 @@ ldconfig = "@/sbin/ldconfig.real"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
 
 # Specify the runtimes to consider. This list is processed in order and the PATH
 # searched for matching executables unless the entry is an absolute path.
@@ -24,3 +24,9 @@ runtimes = [
     "docker-runc",
     "runc",
 ]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

docker/Dockerfile.amazonlinux

@@ -61,13 +61,16 @@ COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "release_date $(date +'%a %b %d %Y')" \
              -D "git_commit ${GIT_COMMIT}" \
              -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
              -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.centos

@@ -59,13 +59,16 @@ COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "release_date $(date +'%a %b %d %Y')" \
              -D "git_commit ${GIT_COMMIT}" \
              -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
              -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -65,14 +65,17 @@ RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \
             "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
-    dch --append "Bump libnvidia-container dependency to ${REVISION}" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER1_VERSION}" && \
     dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/nvidia-container-toolkit_*.deb /dist

docker/Dockerfile.devel

@@ -18,3 +18,4 @@ RUN go install golang.org/x/lint/golint@latest
 RUN go install github.com/matryer/moq@latest
 RUN go install github.com/gordonklaus/ineffassign@latest
 RUN go install github.com/client9/misspell/cmd/misspell@latest
+RUN go install github.com/google/go-licenses@latest

docker/Dockerfile.opensuse-leap

@@ -57,13 +57,16 @@ COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "release_date $(date +'%a %b %d %Y')" \
              -D "git_commit ${GIT_COMMIT}" \
              -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
              -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -58,14 +58,17 @@ COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \
             "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
-    dch --append "Bump libnvidia-container dependency to ${REVISION}" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" && \
     dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -85,32 +85,41 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 --%: docker-build-%
 	@
 
+LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
+LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
 --ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+--ubuntu%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
 --ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
 --debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+--debian%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
 --debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
 --centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
+--amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
+--opensuse-leap%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
+--rhel%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
@@ -131,6 +140,7 @@ docker-build-%:
 	    --build-arg PKG_NAME="$(LIB_NAME)" \
 	    --build-arg PKG_VERS="$(LIB_VERSION)" \
 	    --build-arg PKG_REV="$(PKG_REV)" \
+		--build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
 	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
 		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \

go.mod

@@ -5,13 +5,13 @@ go 1.14
 require (
 	github.com/BurntSushi/toml v1.0.0
 	github.com/NVIDIA/go-nvml v0.11.6-0
-	github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
+	github.com/container-orchestrated-devices/container-device-interface v0.4.1-0.20220614144320-dc973e22f674
 	github.com/containers/podman/v4 v4.0.3
+	github.com/opencontainers/runc v1.1.3
 	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
 	github.com/pelletier/go-toml v1.9.4
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
 	github.com/urfave/cli/v2 v2.3.0
 	golang.org/x/mod v0.5.0
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9

go.sum

@@ -214,10 +214,9 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd/go.mod h1:YqXyXS/oVW3ix0IHVXitKBq3RZoCF8ccm5RPmRBraUI=
-github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b h1:QmL39oNzTgkl9iU7cTT2NyMcJErrZpxYhmYddYIYSz0=
-github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b/go.mod h1:E1zcucIkq9P3eyNmY+68dBQsTcsXJh9cgRo2IVNScKQ=
+github.com/container-orchestrated-devices/container-device-interface v0.4.1-0.20220614144320-dc973e22f674 h1:fLkFha0Ck6uy0HW7c0jht0HPBtE8EJGSFBANnga1QtY=
+github.com/container-orchestrated-devices/container-device-interface v0.4.1-0.20220614144320-dc973e22f674/go.mod h1:ZToWfSyUH5l9Rk7/bjkUUkNLz4b1mE+CVUVafuikDPY=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
@@ -432,6 +431,7 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fsouza/go-dockerclient v1.7.7/go.mod h1:njNCXvoZj3sLPjf3yO0DPHf1mdLdCPDYPc14GskKA4Y=
 github.com/fsouza/go-dockerclient v1.7.8/go.mod h1:7cvopLQDrW3dJ5mcx2LzWMBfmpv/fq7MZUEPcQlAtLw=
@@ -461,7 +461,6 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
@@ -477,7 +476,6 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@@ -624,7 +622,6 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -678,7 +675,6 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -749,7 +745,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -787,7 +782,6 @@ github.com/ldez/gomoddirectives v0.2.2/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdB
 github.com/ldez/tagliatelle v0.2.0/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3mFukHs88=
 github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
@@ -805,7 +799,6 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7
 github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
 github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
-github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI=
 github.com/matoous/godox v0.0.0-20210227103229-6504466cf951/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
 github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
@@ -907,7 +900,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA
 github.com/nishanths/exhaustive v0.2.3/go.mod h1:bhIX678Nx8inLM9PbpvK1yv6oGtoP8BfaIeMzgBNKvc=
 github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ=
 github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
-github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -958,8 +950,10 @@ github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rm
 github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
 github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
 github.com/opencontainers/runc v1.0.3/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
-github.com/opencontainers/runc v1.1.0 h1:O9+X96OcDjkmmZyfaG996kV7yq8HsoU2h1XRRQcefG8=
 github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
+github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
+github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w=
+github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
@@ -989,7 +983,6 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
-github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -1086,6 +1079,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/securego/gosec/v2 v2.9.1/go.mod h1:oDcDLcatOJxkCGaCaq8lua1jTnYf6Sou4wdiJ1n4iHc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -1178,9 +1172,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1
 github.com/tomarrell/wrapcheck/v2 v2.4.0/go.mod h1:68bQ/eJg55BROaRTbMjC7vuhL2OgfoG8bLp9ZyoBfyY=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
 github.com/tommy-muehle/go-mnd/v2 v2.4.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
-github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d h1:hq9X/cf03C5rCx9yWhY7eMHiNxmhTMJAc5DQBq9BfnI=
-github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d/go.mod h1:oFPCwcQpP90RVZxlBdgPN+iu2tPkboPUa4xaVEI6pO4=
-github.com/tsaikd/govalidator v0.0.0-20161031084447-986f2244fc69/go.mod h1:yJymgtZhuWi1Ih5t37Ej381BGZFZvlb9YMTwBxB/QjU=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA=
 github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
@@ -1586,7 +1577,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1620,7 +1610,6 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191001123449-8b695b21ef34/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

internal/config/docker/docker.go

@@ -0,0 +1,115 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// LoadConfig loads the docker config from disk
+func LoadConfig(configFilePath string) (map[string]interface{}, error) {
+	log.Infof("Loading docker config from %v", configFilePath)
+
+	info, err := os.Stat(configFilePath)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(map[string]interface{})
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return cfg, nil
+}
+
+// UpdateConfig updates the docker config to include the nvidia runtimes
+func UpdateConfig(config map[string]interface{}, defaultRuntime string, newRuntimes map[string]interface{}) error {
+	if defaultRuntime != "" {
+		config["default-runtime"] = defaultRuntime
+	}
+
+	// Read the existing runtimes
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	// Add / update the runtime definitions
+	for name, rt := range newRuntimes {
+		runtimes[name] = rt
+	}
+
+	// Update the runtimes definition
+	if len(runtimes) > 0 {
+		config["runtimes"] = runtimes
+	}
+	return nil
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(cfg map[string]interface{}, configFilePath string) error {
+	log.Infof("Flushing docker config to %v", configFilePath)
+
+	output, err := json.MarshalIndent(cfg, "", "    ")
+	if err != nil {
+		return fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(configFilePath)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(configFilePath)
+		if err != nil {
+			return fmt.Errorf("unable to open %v for writing: %v", configFilePath, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(string(output))
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}

internal/config/docker/docker_test.go

@@ -0,0 +1,228 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateConfigDefaultRuntime(t *testing.T) {
+	testCases := []struct {
+		config                     map[string]interface{}
+		defaultRuntime             string
+		runtimeName                string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{
+			defaultRuntime:             "",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			defaultRuntime:             "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "ALREADY_SET",
+			},
+			defaultRuntime:             "",
+			expectedDefaultRuntimeName: "ALREADY_SET",
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "ALREADY_SET",
+			},
+			defaultRuntime:             "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			if tc.config == nil {
+				tc.config = make(map[string]interface{})
+			}
+			err := UpdateConfig(tc.config, tc.defaultRuntime, nil)
+			require.NoError(t, err)
+
+			defaultRuntimeName := tc.config["default-runtime"]
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+		})
+	}
+}
+
+func TestUpdateConfigRuntimes(t *testing.T) {
+	testCases := []struct {
+		config         map[string]interface{}
+		runtimes       map[string]interface{}
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config: map[string]interface{}{},
+			runtimes: map[string]interface{}{
+				"runtime1": map[string]interface{}{
+					"path": "/test/runtime/dir/runtime1",
+					"args": []string{},
+				},
+				"runtime2": map[string]interface{}{
+					"path": "/test/runtime/dir/runtime2",
+					"args": []string{},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+					"runtime2": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime2",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "runtime1",
+						"args": []string{},
+					},
+				},
+			},
+			runtimes: map[string]interface{}{
+				"runtime1": map[string]interface{}{
+					"path": "/test/runtime/dir/runtime1",
+					"args": []string{},
+				},
+				"runtime2": map[string]interface{}{
+					"path": "/test/runtime/dir/runtime2",
+					"args": []string{},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+					"runtime2": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime2",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+				},
+			},
+			runtimes: map[string]interface{}{
+				"runtime1": map[string]interface{}{
+					"path": "/test/runtime/dir/runtime1",
+					"args": []string{},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			runtimes: map[string]interface{}{
+				"runtime1": map[string]interface{}{
+					"path": "/test/runtime/dir/runtime1",
+					"args": []string{},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			err := UpdateConfig(tc.config, "", tc.runtimes)
+			require.NoError(t, err)
+
+			configContent, err := json.MarshalIndent(tc.config, "", "    ")
+			require.NoError(t, err)
+
+			expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+			require.NoError(t, err)
+
+			require.EqualValues(t, string(expectedContent), string(configContent))
+		})
+
+	}
+}

internal/config/image/cuda_image.go

@@ -112,6 +112,36 @@ func (i CUDA) HasDisableRequire() bool {
 	return false
 }
 
+// DevicesFromEnvvars returns the devices requested by the image through environment variables
+func (i CUDA) DevicesFromEnvvars(envVars ...string) []string {
+	// Grab a reference to devices from the first envvar
+	// in the list that actually exists in the environment.
+	var devices *string
+	for _, envVar := range envVars {
+		if devs, ok := i[envVar]; ok {
+			devices = &devs
+			break
+		}
+	}
+
+	// Environment variable unset with legacy image: default to "all".
+	if devices == nil && i.IsLegacy() {
+		return []string{"all"}
+	}
+
+	// Environment variable unset or empty or "void": return nil
+	if devices == nil || len(*devices) == 0 || *devices == "void" {
+		return nil
+	}
+
+	// Environment variable set to "none": reset to "".
+	if *devices == "none" {
+		return []string{""}
+	}
+
+	return strings.Split(*devices, ",")
+}
+
 func (i CUDA) legacyVersion() (string, error) {
 	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
 	if err != nil {

internal/config/runtime.go

@@ -44,6 +44,12 @@ type RuntimeConfig struct {
 // modesConfig defines (optional) per-mode configs
 type modesConfig struct {
 	CSV csvModeConfig `toml:"csv"`
+	CDI cdiModeConfig `toml:"cdi"`
+}
+
+type cdiModeConfig struct {
+	// SpecDirs allows for the default spec dirs for CDI to be overridden
+	SpecDirs []string `toml:"spec-dirs"`
 }
 
 type csvModeConfig struct {

internal/discover/char_devices.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
+func NewCharDeviceDiscoverer(logger *logrus.Logger, devices []string, root string) Discover {
+	locator := lookup.NewCharDeviceLocator(logger, root)
+
+	return NewDeviceDiscoverer(logger, locator, root, devices)
+}
+
+// NewDeviceDiscoverer creates a discoverer which locates the specified set of device nodes using the specified locator.
+func NewDeviceDiscoverer(logger *logrus.Logger, locator lookup.Locator, root string, devices []string) Discover {
+	m := NewMounts(logger, locator, root, devices).(*mounts)
+
+	return (*charDevices)(m)
+}
+
+// Mounts returns the discovered mounts for the charDevices.
+// Since this explicitly specifies a device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices.
+// Here the device nodes are first discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	devicesAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range devicesAsMounts {
+		devices = append(devices, Device(mount))
+	}
+
+	return devices, nil
+}

internal/discover/char_devices_test.go

@@ -0,0 +1,83 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCharDevices(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description          string
+		input                *charDevices
+		expectedMounts       []Mount
+		expectedMountsError  error
+		expectedDevicesError error
+		expectedDevices      []Device
+	}{
+		{
+			description: "dev mounts are empty",
+			input: (*charDevices)(
+				&mounts{
+					lookup: &lookup.LocatorMock{
+						LocateFunc: func(string) ([]string, error) {
+							return []string{"located"}, nil
+						},
+					},
+					required: []string{"required"},
+				},
+			),
+			expectedDevices: []Device{{Path: "located", HostPath: "located"}},
+		},
+		{
+			description:          "dev devices returns error for nil lookup",
+			input:                &charDevices{},
+			expectedDevicesError: fmt.Errorf("no lookup defined"),
+		},
+	}
+
+	for _, tc := range testCases {
+		logHook.Reset()
+
+		t.Run(tc.description, func(t *testing.T) {
+			tc.input.logger = logger
+
+			mounts, err := tc.input.Mounts()
+			if tc.expectedMountsError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedMounts, mounts)
+
+			devices, err := tc.input.Devices()
+			if tc.expectedDevicesError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedDevices, devices)
+		})
+	}
+}

internal/discover/csv.go

@@ -24,11 +24,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// charDevices is a discover for a list of character devices
-type charDevices mounts
-
-var _ Discover = (*charDevices)(nil)
-
 // NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
 // The constructed discoverer is comprised of a list, with each element in the list being associated with a
 // single CSV files.
@@ -47,23 +42,21 @@ func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discov
 		csv.MountSpecSym: symlinkLocator,
 	}
 
-	var discoverers []Discover
+	var mountSpecs []*csv.MountSpec
 	for _, filename := range files {
-		d, err := NewFromCSVFile(logger, locators, filename)
+		targets, err := loadCSVFile(logger, filename)
 		if err != nil {
 			logger.Warnf("Skipping CSV file %v: %v", filename, err)
 			continue
 		}
-		discoverers = append(discoverers, d)
+		mountSpecs = append(mountSpecs, targets...)
 	}
 
-	return &list{discoverers: discoverers}, nil
+	return newFromMountSpecs(logger, locators, root, mountSpecs)
 }
 
-// NewFromCSVFile creates a discoverer for the specified CSV file. A logger is also supplied.
-// The constructed discoverer is comprised of a list, with each element in the list being associated with a particular
-// MountSpecType.
-func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, filename string) (Discover, error) {
+// loadCSVFile loads the specified CSV file and returns the list of mount specs
+func loadCSVFile(logger *logrus.Logger, filename string) ([]*csv.MountSpec, error) {
 	// Create a discoverer for each file-kind combination
 	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
 	if err != nil {
@@ -73,12 +66,12 @@ func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup
 		return nil, fmt.Errorf("CSV file is empty")
 	}
 
-	return newFromMountSpecs(logger, locators, targets)
+	return targets, nil
 }
 
 // newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
 // A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
-func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, targets []*csv.MountSpec) (Discover, error) {
+func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, root string, targets []*csv.MountSpec) (Discover, error) {
 	if len(targets) == 0 {
 		return &None{}, nil
 	}
@@ -99,41 +92,16 @@ func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]loo
 			return nil, fmt.Errorf("no locator defined for '%v'", t)
 		}
 
-		m := &mounts{
-			logger:   logger,
-			lookup:   locator,
-			required: candidatesByType[t],
-		}
-
+		var m Discover
 		switch t {
 		case csv.MountSpecDev:
-			// For device mount specs, we insert a charDevices into the list of discoverers.
-			discoverers = append(discoverers, (*charDevices)(m))
+			m = NewDeviceDiscoverer(logger, locator, root, candidatesByType[t])
 		default:
-			discoverers = append(discoverers, m)
+			m = NewMounts(logger, locator, root, candidatesByType[t])
 		}
+		discoverers = append(discoverers, m)
+
 	}
 
 	return &list{discoverers: discoverers}, nil
 }
-
-// Mounts returns the discovered mounts for the charDevices. Since this explicitly specifies a
-// device list, the mounts are nil.
-func (d *charDevices) Mounts() ([]Mount, error) {
-	return nil, nil
-}
-
-// Devices returns the discovered devices for the charDevices. Here the device nodes are first
-// discovered as mounts and these are converted to devices.
-func (d *charDevices) Devices() ([]Device, error) {
-	devicesAsMounts, err := (*mounts)(d).Mounts()
-	if err != nil {
-		return nil, err
-	}
-	var devices []Device
-	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
-	}
-
-	return devices, nil
-}

internal/discover/csv_test.go

@@ -26,63 +26,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestCharDevices(t *testing.T) {
-	logger, logHook := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description          string
-		input                *charDevices
-		expectedMounts       []Mount
-		expectedMountsError  error
-		expectedDevicesError error
-		expectedDevices      []Device
-	}{
-		{
-			description: "dev mounts are empty",
-			input: (*charDevices)(
-				&mounts{
-					lookup: &lookup.LocatorMock{
-						LocateFunc: func(string) ([]string, error) {
-							return []string{"located"}, nil
-						},
-					},
-					required: []string{"required"},
-				},
-			),
-			expectedDevices: []Device{{Path: "located"}},
-		},
-		{
-			description:          "dev devices returns error for nil lookup",
-			input:                &charDevices{},
-			expectedDevicesError: fmt.Errorf("no lookup defined"),
-		},
-	}
-
-	for _, tc := range testCases {
-		logHook.Reset()
-
-		t.Run(tc.description, func(t *testing.T) {
-			tc.input.logger = logger
-
-			mounts, err := tc.input.Mounts()
-			if tc.expectedMountsError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			require.ElementsMatch(t, tc.expectedMounts, mounts)
-
-			devices, err := tc.input.Devices()
-			if tc.expectedDevicesError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			require.ElementsMatch(t, tc.expectedDevices, devices)
-		})
-	}
-}
-
 func TestNewFromMountSpec(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
@@ -93,6 +36,7 @@ func TestNewFromMountSpec(t *testing.T) {
 
 	testCases := []struct {
 		description        string
+		root               string
 		targets            []*csv.MountSpec
 		expectedError      error
 		expectedDiscoverer Discover
@@ -133,12 +77,50 @@ func TestNewFromMountSpec(t *testing.T) {
 						&mounts{
 							logger:   logger,
 							lookup:   locators["dev"],
+							root:     "/",
 							required: []string{"dev0", "dev1"},
 						},
 					),
 					&mounts{
 						logger:   logger,
 						lookup:   locators["lib"],
+						root:     "/",
+						required: []string{"lib0"},
+					},
+				},
+			},
+		},
+		{
+			description: "sets root",
+			targets: []*csv.MountSpec{
+				{
+					Type: "dev",
+					Path: "dev0",
+				},
+				{
+					Type: "lib",
+					Path: "lib0",
+				},
+				{
+					Type: "dev",
+					Path: "dev1",
+				},
+			},
+			root: "/some/root",
+			expectedDiscoverer: &list{
+				discoverers: []Discover{
+					(*charDevices)(
+						&mounts{
+							logger:   logger,
+							lookup:   locators["dev"],
+							root:     "/some/root",
+							required: []string{"dev0", "dev1"},
+						},
+					),
+					&mounts{
+						logger:   logger,
+						lookup:   locators["lib"],
+						root:     "/some/root",
 						required: []string{"lib0"},
 					},
 				},
@@ -148,7 +130,7 @@ func TestNewFromMountSpec(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			discoverer, err := newFromMountSpecs(logger, locators, tc.targets)
+			discoverer, err := newFromMountSpecs(logger, locators, tc.root, tc.targets)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 				return

internal/discover/discover.go

@@ -24,12 +24,14 @@ type Config struct {
 
 // Device represents a discovered character device.
 type Device struct {
-	Path string
+	HostPath string
+	Path     string
 }
 
 // Mount represents a discovered mount.
 type Mount struct {
-	Path string
+	HostPath string
+	Path     string
 }
 
 // Hook represents a discovered hook.

internal/discover/gds.go

@@ -0,0 +1,77 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type gdsDeviceDiscoverer struct {
+	None
+	logger  *logrus.Logger
+	devices Discover
+	mounts  Discover
+}
+
+// NewGDSDiscoverer creates a discoverer for GPUDirect Storage devices and mounts.
+func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{"/dev/nvidia-fs*"},
+		root,
+	)
+
+	udev := NewMounts(
+		logger,
+		lookup.NewDirectoryLocator(logger, root),
+		root,
+		[]string{"/run/udev"},
+	)
+
+	cufile := NewMounts(
+		logger,
+		lookup.NewFileLocator(logger, root),
+		root,
+		[]string{"/etc/cufile.json"},
+	)
+
+	d := gdsDeviceDiscoverer{
+		logger:  logger,
+		devices: devices,
+		mounts:  Merge(udev, cufile),
+	}
+
+	return &d, nil
+}
+
+// Devices discovers the nvidia-fs device nodes for use with GPUDirect Storage
+func (d *gdsDeviceDiscoverer) Devices() ([]Device, error) {
+	return d.devices.Devices()
+}
+
+// Mounts discovers the required mounts for GPUDirect Storage.
+// If no devices are discovered the discovered mounts are empty
+func (d *gdsDeviceDiscoverer) Mounts() ([]Mount, error) {
+	devices, err := d.Devices()
+	if err != nil || len(devices) == 0 {
+		d.logger.Debugf("No nvidia-fs devices detected; skipping detection of mounts")
+		return nil, nil
+	}
+
+	return d.mounts.Mounts()
+}

internal/discover/ldconfig.go

@@ -100,7 +100,7 @@ func getLibDirs(mounts []Mount) []string {
 		if exists {
 			continue
 		}
-		checked[dir] = isLibName(filepath.Base(m.Path))
+		checked[dir] = isLibName(m.Path)
 
 		if checked[dir] {
 			paths = append(paths, dir)
@@ -114,13 +114,18 @@ func getLibDirs(mounts []Mount) []string {
 
 // isLibName checks if the specified filename is a library (i.e. ends in `.so*`)
 func isLibName(filename string) bool {
-	parts := strings.Split(filename, ".")
 
-	for _, p := range parts {
-		if p == "so" {
-			return true
-		}
+	base := filepath.Base(filename)
+
+	isLib, err := filepath.Match("lib?*.so*", base)
+	if !isLib || err != nil {
+		return false
 	}
 
-	return false
+	parts := strings.Split(base, ".so")
+	if len(parts) == 1 {
+		return true
+	}
+
+	return parts[len(parts)-1] == "" || strings.HasPrefix(parts[len(parts)-1], ".")
 }

internal/discover/ldconfig_test.go

@@ -0,0 +1,65 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsLibName(t *testing.T) {
+	testCases := []struct {
+		name  string
+		isLib bool
+	}{
+		{
+			name:  "",
+			isLib: false,
+		},
+		{
+			name:  "lib/not/.so",
+			isLib: false,
+		},
+		{
+			name:  "lib.so",
+			isLib: false,
+		},
+		{
+			name:  "notlibcuda.so",
+			isLib: false,
+		},
+		{
+			name:  "libcuda.so",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.so.1",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.soNOT",
+			isLib: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.isLib, isLibName(tc.name))
+		})
+	}
+}

internal/discover/list.go

@@ -27,8 +27,8 @@ type list struct {
 
 var _ Discover = (*list)(nil)
 
-// NewList creates a discoverer that is the composite of a list of discoveres.
-func NewList(d ...Discover) Discover {
+// Merge creates a discoverer that is the composite of a list of discoveres.
+func Merge(d ...Discover) Discover {
 	l := list{
 		discoverers: d,
 	}

internal/discover/mofed.go

@@ -0,0 +1,35 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+// NewMOFEDDiscoverer creates a discoverer for MOFED devices.
+func NewMOFEDDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{
+			"/dev/infiniband/uverbs*",
+			"/dev/infiniband/rdma_cm",
+		},
+		root,
+	)
+
+	return devices, nil
+}

internal/discover/mounts.go

@@ -18,6 +18,8 @@ package discover
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
@@ -31,6 +33,7 @@ type mounts struct {
 	None
 	logger   *logrus.Logger
 	lookup   lookup.Locator
+	root     string
 	required []string
 	sync.Mutex
 	cache []Mount
@@ -38,6 +41,16 @@ type mounts struct {
 
 var _ Discover = (*mounts)(nil)
 
+// NewMounts creates a discoverer for the required mounts using the specified locator.
+func NewMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) Discover {
+	return &mounts{
+		logger:   logger,
+		lookup:   lookup,
+		root:     filepath.Join("/", root),
+		required: required,
+	}
+}
+
 func (d *mounts) Mounts() ([]Mount, error) {
 	if d.lookup == nil {
 		return nil, fmt.Errorf("no lookup defined")
@@ -51,7 +64,7 @@ func (d *mounts) Mounts() ([]Mount, error) {
 	d.Lock()
 	defer d.Unlock()
 
-	paths := make(map[string]bool)
+	uniqueMounts := make(map[string]Mount)
 
 	for _, candidate := range d.required {
 		d.logger.Debugf("Locating %v", candidate)
@@ -66,20 +79,39 @@ func (d *mounts) Mounts() ([]Mount, error) {
 		}
 		d.logger.Debugf("Located %v as %v", candidate, located)
 		for _, p := range located {
-			paths[p] = true
+			if _, ok := uniqueMounts[p]; ok {
+				d.logger.Debugf("Skipping duplicate mount %v", p)
+				continue
+			}
+
+			r := d.relativeTo(p)
+			if r == "" {
+				r = p
+			}
+
+			d.logger.Infof("Selecting %v as %v", p, r)
+			uniqueMounts[p] = Mount{
+				HostPath: p,
+				Path:     r,
+			}
 		}
 	}
 
 	var mounts []Mount
-	for path := range paths {
-		d.logger.Infof("Selecting %v", path)
-		mount := Mount{
-			Path: path,
-		}
-		mounts = append(mounts, mount)
+	for _, m := range uniqueMounts {
+		mounts = append(mounts, m)
 	}
 
 	d.cache = mounts
 
-	return mounts, nil
+	return d.cache, nil
+}
+
+// relativeTo returns the path relative to the root for the file locator
+func (d *mounts) relativeTo(path string) string {
+	if d.root == "/" {
+		return path
+	}
+
+	return strings.TrimPrefix(path, d.root)
 }

internal/discover/mounts_test.go

@@ -70,7 +70,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required"},
 			},
-			expectedMounts: []Mount{{Path: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
 		},
 		{
 			description:   "mounts removes located duplicates",
@@ -83,7 +83,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
 		},
 		{
 			description: "mounts skips located errors",
@@ -98,7 +98,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "error", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0"}, {Path: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
 		},
 		{
 			description: "mounts skips unlocated",
@@ -113,10 +113,10 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "empty", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0"}, {Path: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
 		},
 		{
-			description: "mounts skips unlocated",
+			description: "mounts adds multiple",
 			input: &mounts{
 				lookup: &lookup.LocatorMock{
 					LocateFunc: func(s string) ([]string, error) {
@@ -129,10 +129,25 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "required0"},
-				{Path: "multiple0"},
-				{Path: "multiple1"},
-				{Path: "required1"},
+				{Path: "required0", HostPath: "required0"},
+				{Path: "multiple0", HostPath: "multiple0"},
+				{Path: "multiple1", HostPath: "multiple1"},
+				{Path: "required1", HostPath: "required1"},
+			},
+		},
+		{
+			description: "mounts uses relative path",
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(s string) ([]string, error) {
+						return []string{"/some/root/located"}, nil
+					},
+				},
+				root:     "/some/root",
+				required: []string{"required0", "multiple", "required1"},
+			},
+			expectedMounts: []Mount{
+				{Path: "/located", HostPath: "/some/root/located"},
 			},
 		},
 	}

internal/edits/device.go

@@ -17,29 +17,51 @@
 package edits
 
 import (
+	"fmt"
+
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+
+	"github.com/opencontainers/runc/libcontainer/devices"
 )
 
 type device discover.Device
 
 // toEdits converts a discovered device to CDI Container Edits.
-func (d device) toEdits() *cdi.ContainerEdits {
+func (d device) toEdits() (*cdi.ContainerEdits, error) {
+	deviceNode, err := d.toSpec()
+	if err != nil {
+		return nil, err
+	}
+
 	e := cdi.ContainerEdits{
 		ContainerEdits: &specs.ContainerEdits{
-			DeviceNodes: []*specs.DeviceNode{d.toSpec()},
+			DeviceNodes: []*specs.DeviceNode{deviceNode},
 		},
 	}
-	return &e
+	return &e, nil
 }
 
 // toSpec converts a discovered Device to a CDI Spec Device. Note
 // that missing info is filled in when edits are applied by querying the Device node.
-func (d device) toSpec() *specs.DeviceNode {
+func (d device) toSpec() (*specs.DeviceNode, error) {
+	// NOTE: This mirrors what cri-o does.
+	// https://github.com/cri-o/cri-o/blob/ca3bb80a3dda0440659fcf8da8ed6f23211de94e/internal/config/device/device.go#L93
+	// This can be removed once https://github.com/container-orchestrated-devices/container-device-interface/issues/72 is addressed
+	dev, err := devices.DeviceFromPath(d.HostPath, "rwm")
+	if err != nil {
+		return nil, fmt.Errorf("failed to query device node %v: %v", d.HostPath, err)
+	}
 	s := specs.DeviceNode{
-		Path: d.Path,
+		Path:     d.Path,
+		Type:     string(dev.Type),
+		Major:    dev.Major,
+		Minor:    dev.Minor,
+		FileMode: &dev.FileMode,
+		UID:      &dev.Uid,
+		GID:      &dev.Gid,
 	}
 
-	return &s
+	return &s, nil
 }

internal/edits/edits.go

@@ -51,7 +51,11 @@ func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier,
 
 	c := cdi.ContainerEdits{}
 	for _, d := range devices {
-		c.Append(device(d).toEdits())
+		edits, err := device(d).toEdits()
+		if err != nil {
+			return nil, fmt.Errorf("failed to created container edits for device: %v", err)
+		}
+		c.Append(edits)
 	}
 
 	for _, m := range mounts {

internal/edits/mount.go

@@ -38,8 +38,7 @@ func (d mount) toEdits() *cdi.ContainerEdits {
 // that missing info is filled in when edits are applied by querying the Mount node.
 func (d mount) toSpec() *specs.Mount {
 	s := specs.Mount{
-		HostPath: d.Path,
-		// TODO: We need to allow the container path to be customised
+		HostPath:      d.HostPath,
 		ContainerPath: d.Path,
 		Options: []string{
 			"ro",

internal/lookup/executable.go

@@ -49,18 +49,19 @@ func NewExecutableLocator(logger *log.Logger, root string) Locator {
 
 var _ Locator = (*executable)(nil)
 
-// Locate finds executable files in the path. If a relative or absolute path is specified, the prefix paths are not considered.
-func (p executable) Locate(filename string) ([]string, error) {
+// Locate finds executable files with the specified pattern in the path.
+// If a relative or absolute path is specified, the prefix paths are not considered.
+func (p executable) Locate(pattern string) ([]string, error) {
 	// For absolute paths we ensure that it is executable
-	if strings.Contains(filename, "/") {
-		err := assertExecutable(filename)
+	if strings.Contains(pattern, "/") {
+		err := assertExecutable(pattern)
 		if err != nil {
-			return nil, fmt.Errorf("absolute path %v is not an executable file: %v", filename, err)
+			return nil, fmt.Errorf("absolute path %v is not an executable file: %v", pattern, err)
 		}
-		return []string{filename}, nil
+		return []string{pattern}, nil
 	}
 
-	return p.file.Locate(filename)
+	return p.file.Locate(pattern)
 }
 
 // assertExecutable checks whether the specified path is an execuable file.

internal/lookup/file.go

@@ -50,22 +50,29 @@ func newFileLocator(logger *log.Logger, root string) file {
 
 var _ Locator = (*file)(nil)
 
-// Locate attempts to find the specified file. All prefixes are searched and any matching
-// candidates are returned. If no matches are found, an error is returned.
-func (p file) Locate(filename string) ([]string, error) {
+// Locate attempts to find files with names matching the specified pattern.
+// All prefixes are searched and any matching candidates are returned. If no matches are found, an error is returned.
+func (p file) Locate(pattern string) ([]string, error) {
 	var filenames []string
 	for _, prefix := range p.prefixes {
-		candidate := filepath.Join(prefix, filename)
-		p.logger.Debugf("Checking candidate '%v'", candidate)
-		err := p.filter(candidate)
+		pathPattern := filepath.Join(prefix, pattern)
+		candidates, err := filepath.Glob(pathPattern)
 		if err != nil {
-			p.logger.Debugf("Candidate '%v' does not meet requirements: %v", candidate, err)
-			continue
+			p.logger.Debugf("Checking pattern '%v' failed: %v", pathPattern, err)
+		}
+
+		for _, candidate := range candidates {
+			p.logger.Debugf("Checking candidate '%v'", candidate)
+			err := p.filter(candidate)
+			if err != nil {
+				p.logger.Debugf("Candidate '%v' does not meet requirements: %v", candidate, err)
+				continue
+			}
+			filenames = append(filenames, candidate)
 		}
-		filenames = append(filenames, candidate)
 	}
-	if len(filename) == 0 {
-		return nil, fmt.Errorf("file %v not found", filename)
+	if len(filenames) == 0 {
+		return nil, fmt.Errorf("pattern %v not found", pattern)
 	}
 	return filenames, nil
 }

internal/lookup/locator_mock.go

@@ -20,6 +20,9 @@ var _ Locator = &LocatorMock{}
 // 			LocateFunc: func(s string) ([]string, error) {
 // 				panic("mock out the Locate method")
 // 			},
+// 			RelativeFunc: func(s string) (string, error) {
+// 				panic("mock out the Relative method")
+// 			},
 // 		}
 //
 // 		// use mockedLocator in code that requires Locator
@@ -30,6 +33,9 @@ type LocatorMock struct {
 	// LocateFunc mocks the Locate method.
 	LocateFunc func(s string) ([]string, error)
 
+	// RelativeFunc mocks the Relative method.
+	RelativeFunc func(s string) (string, error)
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// Locate holds details about calls to the Locate method.
@@ -37,8 +43,14 @@ type LocatorMock struct {
 			// S is the s argument value.
 			S string
 		}
+		// Relative holds details about calls to the Relative method.
+		Relative []struct {
+			// S is the s argument value.
+			S string
+		}
 	}
-	lockLocate sync.RWMutex
+	lockLocate   sync.RWMutex
+	lockRelative sync.RWMutex
 }
 
 // Locate calls LocateFunc.
@@ -75,3 +87,38 @@ func (mock *LocatorMock) LocateCalls() []struct {
 	mock.lockLocate.RUnlock()
 	return calls
 }
+
+// Relative calls RelativeFunc.
+func (mock *LocatorMock) Relative(s string) (string, error) {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockRelative.Lock()
+	mock.calls.Relative = append(mock.calls.Relative, callInfo)
+	mock.lockRelative.Unlock()
+	if mock.RelativeFunc == nil {
+		var (
+			sOut   string
+			errOut error
+		)
+		return sOut, errOut
+	}
+	return mock.RelativeFunc(s)
+}
+
+// RelativeCalls gets all the calls that were made to Relative.
+// Check the length with:
+//     len(mockedLocator.RelativeCalls())
+func (mock *LocatorMock) RelativeCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockRelative.RLock()
+	calls = mock.calls.Relative
+	mock.lockRelative.RUnlock()
+	return calls
+}

internal/lookup/symlinks.go

@@ -52,10 +52,10 @@ func NewSymlinkLocator(logger *logrus.Logger, root string) Locator {
 	return &l
 }
 
-// Locate finds the specified file at the specified root. If the file is a symlink, the link is followed and all candidates
-// to the final target are returned.
-func (p symlinkChain) Locate(filename string) ([]string, error) {
-	candidates, err := p.file.Locate(filename)
+// Locate finds the specified pattern at the specified root.
+// If the file is a symlink, the link is followed and all candidates to the final target are returned.
+func (p symlinkChain) Locate(pattern string) ([]string, error) {
+	candidates, err := p.file.Locate(pattern)
 	if err != nil {
 		return nil, err
 	}
@@ -104,14 +104,15 @@ func (p symlinkChain) Locate(filename string) ([]string, error) {
 	return filenames, nil
 }
 
-// Locate finds the specified file at the specified root. If the file is a symlink, the link is resolved and the target returned.
-func (p symlink) Locate(filename string) ([]string, error) {
-	candidates, err := p.file.Locate(filename)
+// Locate finds the specified pattern at the specified root.
+// If the file is a symlink, the link is resolved and the target returned.
+func (p symlink) Locate(pattern string) ([]string, error) {
+	candidates, err := p.file.Locate(pattern)
 	if err != nil {
 		return nil, err
 	}
 	if len(candidates) != 1 {
-		return nil, fmt.Errorf("failed to uniquely resolve symlink %v: %v", filename, candidates)
+		return nil, fmt.Errorf("failed to uniquely resolve symlink %v: %v", pattern, candidates)
 	}
 
 	target, err := filepath.EvalSymlinks(candidates[0])

internal/modifier/cdi.go

@@ -0,0 +1,128 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+type cdiModifier struct {
+	logger   *logrus.Logger
+	specDirs []string
+	devices  []string
+}
+
+// NewCDIModifier creates an OCI spec modifier that determines the modifications to make based on the
+// CDI specifications available on the system. The NVIDIA_VISIBLE_DEVICES enviroment variable is
+// used to select the devices to include.
+func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	devices, err := getDevicesFromSpec(ociSpec)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get required devices from OCI specification: %v", err)
+	}
+	if len(devices) == 0 {
+		logger.Debugf("No devices requested; no modification required.")
+		return nil, nil
+	}
+
+	specDirs := cdi.DefaultSpecDirs
+	if len(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs) > 0 {
+		specDirs = cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs
+	}
+
+	m := cdiModifier{
+		logger:   logger,
+		specDirs: specDirs,
+		devices:  devices,
+	}
+
+	return m, nil
+}
+
+func getDevicesFromSpec(ociSpec oci.Spec) ([]string, error) {
+	rawSpec, err := ociSpec.Load()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	image, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	envDevices := image.DevicesFromEnvvars(visibleDevicesEnvvar)
+
+	_, annotationDevices, err := cdi.ParseAnnotations(rawSpec.Annotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse container annotations: %v", err)
+	}
+
+	uniqueDevices := make(map[string]struct{})
+	for _, name := range append(envDevices, annotationDevices...) {
+		if !cdi.IsQualifiedName(name) {
+			name = cdi.QualifiedName("nvidia.com", "gpu", name)
+		}
+		uniqueDevices[name] = struct{}{}
+	}
+
+	var devices []string
+	for name := range uniqueDevices {
+		devices = append(devices, name)
+	}
+
+	return devices, nil
+}
+
+// Modify loads the CDI registry and injects the specified CDI devices into the OCI runtime specification.
+func (m cdiModifier) Modify(spec *specs.Spec) error {
+	registry := cdi.GetRegistry(
+		cdi.WithSpecDirs(m.specDirs...),
+		cdi.WithAutoRefresh(false),
+	)
+	if err := registry.Refresh(); err != nil {
+		m.logger.Debugf("The following error was triggered when refreshing the CDI registry: %v", err)
+	}
+
+	devices := m.devices
+	for _, d := range devices {
+		if d == "nvidia.com/gpu=all" {
+			devices = []string{}
+			for _, candidate := range registry.DeviceDB().ListDevices() {
+				if strings.HasPrefix(candidate, "nvidia.com/gpu=") {
+					devices = append(devices, candidate)
+				}
+			}
+			break
+		}
+	}
+
+	m.logger.Debugf("Injecting devices using CDI: %v", devices)
+	_, err := registry.InjectDevices(spec, devices...)
+	if err != nil {
+		return fmt.Errorf("failed to inject CDI devices: %v", err)
+	}
+
+	return nil
+}

internal/modifier/csv.go

@@ -24,10 +24,8 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/cuda"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements"
-	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -52,11 +50,13 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
 	}
 
-	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
-	// if no modification is required.
-	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
-	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
-		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
+	image, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}
 	logger.Infof("Constructing modifier from config: %+v", *cfg)
@@ -66,14 +66,7 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
 	}
 
-	// TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the
-	// visible devices are checked.
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := checkRequirements(logger, &image); err != nil {
+	if err := checkRequirements(logger, image); err != nil {
 		return nil, fmt.Errorf("requirements not met: %v", err)
 	}
 
@@ -82,8 +75,7 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
 	}
 
-	nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
-	if nvidiaRequireJetpack != "csv-mounts=all" {
+	if nvidiaRequireJetpack, _ := image[nvidiaRequireJetpackEnvvar]; nvidiaRequireJetpack != "csv-mounts=all" {
 		csvFiles = csv.BaseFilesOnly(csvFiles)
 	}
 
@@ -92,48 +84,37 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
 	}
 
-	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
-	}
-
 	createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
 	}
 
-	d := discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
-
-	return newModifierFromDiscoverer(logger, d)
-}
-
-// newModifierFromDiscoverer created a modifier that aplies the discovered
-// modifications to an OCI spec if require by the runtime wrapper.
-func newModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
-	m := csvMode{
-		logger:     logger,
-		discoverer: d,
-	}
-	return &m, nil
-}
-
-// Modify applies the required modifications to the incomming OCI spec. These modifications
-// are applied in-place.
-func (m csvMode) Modify(spec *specs.Spec) error {
-	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
+	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
 	if err != nil {
-		return fmt.Errorf("failed to remove existing hooks: %v", err)
+		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
 	}
 
-	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
+	d := discover.Merge(
+		csvDiscoverer,
+		createSymlinksHook,
+		// The ldcacheUpdateHook is added last to ensure that the created symlinks are included
+		ldcacheUpdateHook,
+	)
+
+	discoverModifier, err := NewModifierFromDiscoverer(logger, d)
 	if err != nil {
-		return fmt.Errorf("failed to get required container edits: %v", err)
+		return nil, fmt.Errorf("failed to construct modifier: %v", err)
 	}
 
-	return specEdits.Modify(spec)
+	modifiers := Merge(
+		nvidiaContainerRuntimeHookRemover{logger},
+		discoverModifier,
+	)
+
+	return modifiers, nil
 }
 
-func checkRequirements(logger *logrus.Logger, image *image.CUDA) error {
+func checkRequirements(logger *logrus.Logger, image image.CUDA) error {
 	if image.HasDisableRequire() {
 		// TODO: We could print the real value here instead
 		logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true)

internal/modifier/csv_test.go

@@ -21,7 +21,6 @@ import (
 	"testing"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	testlog "github.com/sirupsen/logrus/hooks/test"
@@ -95,105 +94,15 @@ func TestNewCSVModifier(t *testing.T) {
 	}
 }
 
-func TestExperimentalModifier(t *testing.T) {
+func TestCSVModifierRemovesHook(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
 	testCases := []struct {
 		description   string
-		discover      *discover.DiscoverMock
 		spec          *specs.Spec
 		expectedError error
 		expectedSpec  *specs.Spec
 	}{
-		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
-		},
-		{
-			description: "failed hooks discoverer returns error",
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					return nil, fmt.Errorf("discover.Hooks error")
-				},
-			},
-			expectedError: fmt.Errorf("discover.Hooks error"),
-		},
-		{
-			description: "discovered hooks are injected into spec",
-			spec:        &specs.Spec{},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/a",
-							Args:      []string{"/hook/a", "arga"},
-						},
-						{
-							Lifecycle: "createContainer",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-					CreateContainer: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "existing hooks are maintained",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
 		{
 			description: "modification removes existing nvidia-container-runtime-hook",
 			spec: &specs.Spec{
@@ -206,26 +115,9 @@ func TestExperimentalModifier(t *testing.T) {
 					},
 				},
 			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
 			expectedSpec: &specs.Spec{
 				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
+					Prestart: []specs.Hook{},
 				},
 			},
 		},
@@ -241,26 +133,9 @@ func TestExperimentalModifier(t *testing.T) {
 					},
 				},
 			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
 			expectedSpec: &specs.Spec{
 				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
+					Prestart: []specs.Hook{},
 				},
 			},
 		},
@@ -268,17 +143,16 @@ func TestExperimentalModifier(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			m, err := newModifierFromDiscoverer(logger, tc.discover)
-			require.NoError(t, err)
+			m := nvidiaContainerRuntimeHookRemover{logger: logger}
 
-			err = m.Modify(tc.spec)
+			err := m.Modify(tc.spec)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
 			}
 
-			require.EqualValues(t, tc.expectedSpec, tc.spec)
+			require.Empty(t, tc.spec.Hooks.Prestart)
 		})
 	}
 }

internal/modifier/discover.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+type discoverModifier struct {
+	logger     *logrus.Logger
+	discoverer discover.Discover
+}
+
+// NewModifierFromDiscoverer creates a modifier that applies the discovered
+// modifications to an OCI spec if required by the runtime wrapper.
+func NewModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	m := discoverModifier{
+		logger:     logger,
+		discoverer: d,
+	}
+	return &m, nil
+}
+
+// Modify applies the modifications required by discoverer to the incomming OCI spec.
+// These modifications are applied in-place.
+func (m discoverModifier) Modify(spec *specs.Spec) error {
+	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
+	if err != nil {
+		return fmt.Errorf("failed to get required container edits: %v", err)
+	}
+
+	return specEdits.Modify(spec)
+}

internal/modifier/discover_test.go

@@ -0,0 +1,145 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDiscoverModifier(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		discover      *discover.DiscoverMock
+		spec          *specs.Spec
+		expectedError error
+		expectedSpec  *specs.Spec
+	}{
+		{
+			description: "empty discoverer does not modify spec",
+			discover:    &discover.DiscoverMock{},
+		},
+		{
+			description: "failed hooks discoverer returns error",
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					return nil, fmt.Errorf("discover.Hooks error")
+				},
+			},
+			expectedError: fmt.Errorf("discover.Hooks error"),
+		},
+		{
+			description: "discovered hooks are injected into spec",
+			spec:        &specs.Spec{},
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					hooks := []discover.Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/hook/a",
+							Args:      []string{"/hook/a", "arga"},
+						},
+						{
+							Lifecycle: "createContainer",
+							Path:      "/hook/b",
+							Args:      []string{"/hook/b", "argb"},
+						},
+					}
+					return hooks, nil
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+					},
+					CreateContainer: []specs.Hook{
+						{
+							Path: "/hook/b",
+							Args: []string{"/hook/b", "argb"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "existing hooks are maintained",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+					},
+				},
+			},
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					hooks := []discover.Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/hook/b",
+							Args:      []string{"/hook/b", "argb"},
+						},
+					}
+					return hooks, nil
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+						{
+							Path: "/hook/b",
+							Args: []string{"/hook/b", "argb"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			m, err := NewModifierFromDiscoverer(logger, tc.discover)
+			require.NoError(t, err)
+
+			err = m.Modify(tc.spec)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.EqualValues(t, tc.expectedSpec, tc.spec)
+		})
+	}
+}

internal/modifier/gds.go

@@ -0,0 +1,61 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	nvidiaGDSEnvvar = "NVIDIA_GDS"
+)
+
+// NewGDSModifier creates the modifiers for GDS devices.
+// If the spec does not contain the NVIDIA_GDS=enabled environment variable no changes are made.
+func NewGDSModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	rawSpec, err := ociSpec.Load()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	image, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+		logger.Infof("No modification required; no devices requested")
+		return nil, nil
+	}
+
+	if gds, _ := image[nvidiaGDSEnvvar]; gds != "enabled" {
+		return nil, nil
+	}
+
+	d, err := discover.NewGDSDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct discoverer for GDS devices: %v", err)
+	}
+
+	return NewModifierFromDiscoverer(logger, d)
+}

internal/modifier/hook_remover_test.go

@@ -0,0 +1,111 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHookRemover(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		spec          *specs.Spec
+		expectedError error
+		expectedSpec  *specs.Spec
+	}{
+		{
+			description: "existing hooks are maintained",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "modification removes existing nvidia-container-runtime-hook",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/path/to/nvidia-container-runtime-hook",
+							Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: nil,
+				},
+			},
+		},
+		{
+			description: "modification removes existing nvidia-container-toolkit",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/path/to/nvidia-container-toolkit",
+							Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
+						},
+					},
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: nil,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			m := nvidiaContainerRuntimeHookRemover{logger: logger}
+
+			err := m.Modify(tc.spec)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.EqualValues(t, tc.expectedSpec, tc.spec)
+		})
+	}
+}

internal/modifier/list.go

@@ -0,0 +1,54 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type list struct {
+	modifiers []oci.SpecModifier
+}
+
+// Merge merges a set of OCI specification modifiers as a list.
+// This can be used to compose modifiers.
+func Merge(modifiers ...oci.SpecModifier) oci.SpecModifier {
+	var filteredModifiers []oci.SpecModifier
+	for _, m := range modifiers {
+		if m == nil {
+			continue
+		}
+		filteredModifiers = append(filteredModifiers, m)
+	}
+
+	return list{
+		modifiers: filteredModifiers,
+	}
+}
+
+// Modify applies a list of modifiers in sequence and returns on any errors encountered.
+func (m list) Modify(spec *specs.Spec) error {
+	for _, mm := range m.modifiers {
+		err := mm.Modify(spec)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/modifier/mofed.go

@@ -0,0 +1,61 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	nvidiaMOFEDEnvvar = "NVIDIA_MOFED"
+)
+
+// NewMOFEDModifier creates the modifiers for MOFED devices.
+// If the spec does not contain the NVIDIA_MOFED=enabled environment variable no changes are made.
+func NewMOFEDModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	rawSpec, err := ociSpec.Load()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	image, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+		logger.Infof("No modification required; no devices requested")
+		return nil, nil
+	}
+
+	if mofed, _ := image[nvidiaMOFEDEnvvar]; mofed != "enabled" {
+		return nil, nil
+	}
+
+	d, err := discover.NewMOFEDDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct discoverer for MOFED devices: %v", err)
+	}
+
+	return NewModifierFromDiscoverer(logger, d)
+}

internal/modifier/stable.go

@@ -20,7 +20,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
@@ -61,8 +60,8 @@ func (m stableRuntimeModifier) Modify(spec *specs.Spec) error {
 		spec.Hooks = &specs.Hooks{}
 	} else if len(spec.Hooks.Prestart) != 0 {
 		for _, hook := range spec.Hooks.Prestart {
-			if strings.Contains(hook.Path, config.NVIDIAContainerRuntimeHookExecutable) {
-				m.logger.Infof("existing nvidia prestart hook found in OCI spec")
+			if isNVIDIAContainerRuntimeHook(&hook) {
+				m.logger.Infof("Existing nvidia prestart hook (%v) found in OCI spec", hook.Path)
 				return nil
 			}
 		}

oci-nvidia-hook

@@ -1,2 +1,2 @@
 #!/bin/sh
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" exec /usr/bin/nvidia-container-toolkit "$@"
+PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" exec /usr/bin/nvidia-container-runtime-hook "$@"

oci-nvidia-hook.json

@@ -1,8 +1,8 @@
 {
     "version": "1.0.0",
     "hook": {
-        "path": "/usr/bin/nvidia-container-toolkit",
-        "args": ["nvidia-container-toolkit", "prestart"],
+        "path": "/usr/bin/nvidia-container-runtime-hook",
+        "args": ["nvidia-container-runtime-hook", "prestart"],
         "env": [
             "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
         ]

packaging/debian/control

@@ -3,15 +3,15 @@ Section: @SECTION@utils
 Priority: optional
 Maintainer: NVIDIA CORPORATION <cudatools@nvidia.com>
 Standards-Version: 3.9.8
-Homepage: https://github.com/NVIDIA/nvidia-container-runtime/wiki
-Vcs-Git: https://github.com/NVIDIA/nvidia-container-runtime
-Vcs-Browser: https://github.com/NVIDIA/nvidia-container-runtime
+Homepage: https://github.com/NVIDIA/nvidia-container-toolkit
+Vcs-Git: https://github.com/NVIDIA/nvidia-container-toolkit
+Vcs-Browser: https://github.com/NVIDIA/nvidia-container-toolkit
 Build-Depends: debhelper (>= 9)
 
 Package: nvidia-container-toolkit
 Architecture: any
-Depends: ${misc:Depends}, libnvidia-container-tools (>= @LIBNVIDIA_CONTAINER_VERSION@), libnvidia-container-tools (<< 2.0.0), libseccomp2
+Depends: ${misc:Depends}, libnvidia-container-tools (>= @LIBNVIDIA_CONTAINER_TOOLS_VERSION@), libnvidia-container-tools (<< 2.0.0), libseccomp2
 Breaks: nvidia-container-runtime (<= 3.5.0-1), nvidia-container-runtime-hook
 Replaces: nvidia-container-runtime (<= 3.5.0-1), nvidia-container-runtime-hook
-Description: NVIDIA container runtime hook
- Provides a OCI hook to enable GPU support in containers.
+Description: NVIDIA Container toolkit
+ Provides tools and utilities to enable GPU support in containers.

packaging/debian/nvidia-container-toolkit.install

@@ -1,4 +1,4 @@
 config.toml /etc/nvidia-container-runtime
-nvidia-container-toolkit /usr/bin
+nvidia-container-runtime-hook /usr/bin
 nvidia-container-runtime /usr/bin
 nvidia-ctk /usr/bin

packaging/debian/nvidia-container-toolkit.postinst

@@ -7,9 +7,9 @@ NVIDIA_CONTAINER_TOOLKIT=/usr/bin/nvidia-container-toolkit
 
 case "$1" in
     configure)
-        if [ -f "${NVIDIA_CONTAINER_TOOLKIT}" ]; then
-		if [ ! -e "${NVIDIA_CONTAINER_RUNTIME_HOOK}" ]; then
-			ln -s ${NVIDIA_CONTAINER_TOOLKIT} ${NVIDIA_CONTAINER_RUNTIME_HOOK}
+        if [ -f "${NVIDIA_CONTAINER_RUNTIME_HOOK}" ]; then
+		if [ ! -e "${NVIDIA_CONTAINER_TOOLKIT}" ]; then
+			ln -s ${NVIDIA_CONTAINER_RUNTIME_HOOK} ${NVIDIA_CONTAINER_TOOLKIT}
 		fi
         fi
     ;;

packaging/debian/nvidia-container-toolkit.postrm

@@ -7,7 +7,7 @@ NVIDIA_CONTAINER_TOOLKIT=/usr/bin/nvidia-container-toolkit
 
 case "$1" in
     purge)
-        [ -L "${NVIDIA_CONTAINER_RUNTIME_HOOK}" ] && rm ${NVIDIA_CONTAINER_RUNTIME_HOOK}
+        [ -L "${NVIDIA_CONTAINER_TOOLKIT}" ] && rm ${NVIDIA_CONTAINER_TOOLKIT}
     ;;
 
     upgrade|failed-upgrade|remove|abort-install|abort-upgrade|disappear)

packaging/debian/prepare

@@ -3,7 +3,7 @@
 set -e
 
 sed -i "s;@SECTION@;${SECTION:+$SECTION/};g" debian/control
-sed -i "s;@LIBNVIDIA_CONTAINER_VERSION@;${LIBNVIDIA_CONTAINER_VERSION:+$LIBNVIDIA_CONTAINER_VERSION};g" debian/control
+sed -i "s;@LIBNVIDIA_CONTAINER_TOOLS_VERSION@;${LIBNVIDIA_CONTAINER_TOOLS_VERSION:+$LIBNVIDIA_CONTAINER_TOOLS_VERSION};g" debian/control
 
 if [ -n "$DISTRIB" ]; then
     sed -i "s;UNRELEASED;$DISTRIB;" debian/changelog

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -6,11 +6,11 @@ Group: Development Tools
 Vendor: NVIDIA CORPORATION
 Packager: NVIDIA CORPORATION <cudatools@nvidia.com>
 
-Summary: NVIDIA container runtime hook
-URL: https://github.com/NVIDIA/nvidia-container-runtime
+Summary: NVIDIA Container Toolkit
+URL: https://github.com/NVIDIA/nvidia-container-toolkit
 License: Apache-2.0
 
-Source0: nvidia-container-toolkit
+Source0: nvidia-container-runtime-hook
 Source1: nvidia-container-runtime
 Source2: nvidia-ctk
 Source3: config.toml
@@ -21,7 +21,7 @@ Source6: LICENSE
 Obsoletes: nvidia-container-runtime <= 3.5.0-1, nvidia-container-runtime-hook
 Provides: nvidia-container-runtime
 Provides: nvidia-container-runtime-hook
-Requires: libnvidia-container-tools >= %{libnvidia_container_version}, libnvidia-container-tools < 2.0.0
+Requires: libnvidia-container-tools >= %{libnvidia_container_tools_version}, libnvidia-container-tools < 2.0.0
 
 %if 0%{?suse_version}
 Requires: libseccomp2
@@ -31,14 +31,14 @@ Requires: libseccomp
 %endif
 
 %description
-Provides a OCI hook to enable GPU support in containers.
+Provides tools and utilities to enable GPU support in containers.
 
 %prep
 cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} .
 
 %install
 mkdir -p %{buildroot}%{_bindir}
-install -m 755 -t %{buildroot}%{_bindir} nvidia-container-toolkit
+install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime-hook
 install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime
 install -m 755 -t %{buildroot}%{_bindir} nvidia-ctk
 
@@ -52,14 +52,14 @@ mkdir -p %{buildroot}/usr/share/containers/oci/hooks.d
 install -m 644 -t %{buildroot}/usr/share/containers/oci/hooks.d oci-nvidia-hook.json
 
 %posttrans
-ln -sf %{_bindir}/nvidia-container-toolkit %{_bindir}/nvidia-container-runtime-hook
+ln -sf %{_bindir}/nvidia-container-runtime-hook %{_bindir}/nvidia-container-toolkit
 
 %postun
-rm -f %{_bindir}/nvidia-container-runtime-hook
+rm -f %{_bindir}/nvidia-container-runtime-toolkit
 
 %files
 %license LICENSE
-%{_bindir}/nvidia-container-toolkit
+%{_bindir}/nvidia-container-runtime-hook
 %{_bindir}/nvidia-container-runtime
 %{_bindir}/nvidia-ctk
 %config /etc/nvidia-container-runtime/config.toml
@@ -70,4 +70,4 @@ rm -f %{_bindir}/nvidia-container-runtime-hook
 # As of 1.10.0-1 we generate the release information automatically
 * %{release_date} NVIDIA CORPORATION <cudatools@nvidia.com> %{version}-%{release}
 - See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/%{git_commit}/CHANGELOG.md
-- Bump libnvidia-container dependency to libnvidia-container-tools >= %{libnvidia_container_version}
+- Bump libnvidia-container dependency to libnvidia-container-tools >= %{libnvidia_container_tools_version}

scripts/Dockerfile.sign.rpm

@@ -1,3 +1,3 @@
-FROM centos:8
+FROM quay.io/centos/centos:stream8
 
 RUN yum install -y createrepo rpm-sign pinentry

scripts/build-all-components.sh

@@ -25,7 +25,7 @@ function assert_usage() {
     exit 1
 }
 
-set -e -x
+set -e
 
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
 PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
@@ -52,13 +52,16 @@ ${SCRIPTS_DIR}/get-component-versions.sh
 # Build libnvidia-container
 make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET}
 
-# Build nvidia-container-toolkit
-make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} ${TARGET}
-
-if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} ]]; then
+if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} || -z ${LIBNVIDIA_CONTAINER_VERSION} ]]; then
 eval $(${SCRIPTS_DIR}/get-component-versions.sh)
 fi
 
+# Build nvidia-container-toolkit
+make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} \
+    LIBNVIDIA_CONTAINER_VERSION="${LIBNVIDIA_CONTAINER_VERSION}" \
+    LIBNVIDIA_CONTAINER_TAG="${LIBNVIDIA_CONTAINER_TAG}" \
+        ${TARGET}
+
 # We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets
 # The LIB_TAG is also overridden to match the TOOLKIT_TAG.
 # Build nvidia-container-runtime

scripts/build-packages.sh

@@ -19,7 +19,7 @@
 # as well as the components included in the third_party folder.
 # All required packages are generated in the specified dist folder.
 
-set -e -x
+set -e
 
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
 PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
@@ -61,17 +61,26 @@ fi
 
 eval $(${SCRIPTS_DIR}/get-component-versions.sh)
 
-if [[ "${NVIDIA_CONTAINER_TOOLKIT_VERSION}${NVIDIA_CONTAINER_TOOLKIT_TAG:+~${NVIDIA_CONTAINER_TOOLKIT_TAG}}" != "${LIBNVIDIA_CONTAINER_VERSION}" ]]; then
+
+if [[ -n ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
+echo "Allowing mismatched versions for release candidate "
+: ${ALLOW_VERSION_MISMATCH:=true}
+fi
+
+if [[ "${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" != "${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}" ]]; then
     set +x
     echo "The libnvidia-container and nvidia-container-toolkit versions do not match."
-    echo "lib: '${LIBNVIDIA_CONTAINER_VERSION}'"
-    echo "toolkit: '${NVIDIA_CONTAINER_TOOLKIT_VERSION}${NVIDIA_CONTAINER_TOOLKIT_TAG:+~${NVIDIA_CONTAINER_TOOLKIT_TAG}}'"
+    echo "lib: '${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}'"
+    echo "toolkit: '${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}'"
     set -x
     [[ ${ALLOW_VERSION_MISMATCH} == "true" ]] || exit 1
+    echo "Continuing with mismatched version"
 fi
 
 export NVIDIA_CONTAINER_TOOLKIT_VERSION
 export NVIDIA_CONTAINER_TOOLKIT_TAG
+export LIBNVIDIA_CONTAINER_VERSION
+export LIBNVIDIA_CONTAINER_TAG
 export NVIDIA_CONTAINER_RUNTIME_VERSION
 export NVIDIA_DOCKER_VERSION
 

scripts/get-component-versions.sh

@@ -36,6 +36,9 @@ NVIDIA_DOCKER_ROOT=${PROJECT_ROOT}/third_party/nvidia-docker
 # Get version for libnvidia-container
 libnvidia_container_version_tag=$(grep "#define NVC_VERSION" ${LIBNVIDIA_CONTAINER_ROOT}/src/nvc.h \
     | sed -e 's/#define NVC_VERSION[[:space:]]"\(.*\)"/\1/')
+libnvidia_container_version=${libnvidia_container_version_tag%%~*}
+libnvidia_container_tag=${libnvidia_container_version_tag##${libnvidia_container_version}}
+libnvidia_container_tag=${libnvidia_container_tag##\~}
 
 versions_makefile=${NVIDIA_CONTAINER_TOOLKIT_ROOT}/versions.mk
 # Get version for nvidia-container-toolit
@@ -53,7 +56,8 @@ nvidia_docker_version=$(grep -m 1 "^NVIDIA_DOCKER_VERSION := " ${versions_makefi
 nvidia_docker_tag=${nvidia_container_toolkit_tag}
 nvidia_docker_version_tag="${nvidia_docker_version}${nvidia_docker_tag:+~${nvidia_docker_tag}}"
 
-echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version_tag}"
+echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version}"
+echo "LIBNVIDIA_CONTAINER_TAG=${libnvidia_container_tag}"
 echo "LIBNVIDIA_CONTAINER_PACKAGE_VERSION=${libnvidia_container_version_tag//\~/-}"
 echo "NVIDIA_CONTAINER_TOOLKIT_VERSION=${nvidia_container_toolkit_version}"
 echo "NVIDIA_CONTAINER_TOOLKIT_TAG=${nvidia_container_toolkit_tag}"

scripts/release-packages.sh

@@ -116,8 +116,19 @@ function sync() {
         return
     fi
     mkdir -p ${dst}
-    cp ${src}/libnvidia-container*.${pkg_type} ${dst}
-    cp ${src}/nvidia-container-toolkit*.${pkg_type} ${dst}
+
+    for f in $(ls ${src}/libnvidia-container*.${pkg_type} ${src}/nvidia-container-toolkit*.${pkg_type}); do
+        df=${dst}/$(basename ${f})
+        df_stable=${df//"/experimental/"/"/stable/"}
+        if [[ -f "${df}" ]]; then
+            echo "${df} already exists; skipping"
+        elif [[ ${REPO} == "experimental" && -f ${df_stable} ]]; then
+            echo "${df_stable} already exists; skipping"
+        else
+            cp ${f} ${df}
+        fi
+
+    done
     if [[ ${REPO} == "stable" ]]; then
         cp ${src}/nvidia-container-runtime*.${pkg_type} ${dst}
         cp ${src}/nvidia-docker*.${pkg_type} ${dst}

test/container/toolkit_test.sh

@@ -23,7 +23,7 @@ testing::toolkit::install() {
 		READLINK="greadlink"
 	fi
 
-	testing::docker_run::toolkit::shell 'toolkit install /usr/local/nvidia/toolkit'
+	testing::docker_run::toolkit::shell 'toolkit install --toolkit-root=/usr/local/nvidia/toolkit'
 	docker run --rm -v "${shared_dir}:/work" alpine sh -c "chown -R ${uid}:${gid} /work/"
 
 	# Ensure toolkit dir is correctly setup
@@ -35,14 +35,15 @@ testing::toolkit::install() {
 	test -e "$(${READLINK} -f "${shared_dir}/usr/local/nvidia/toolkit/libnvidia-container-go.so.1")"
 
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-cli"
-	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-hook"
+	test -L "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit"
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
 
 	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
 	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
 
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-cli.real"
-	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit.real"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-hook.real"
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
 
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
@@ -65,7 +66,7 @@ testing::toolkit::install() {
 testing::toolkit::delete() {
 	testing::docker_run::toolkit::shell 'mkdir -p /usr/local/nvidia/delete-toolkit'
 	testing::docker_run::toolkit::shell 'touch /usr/local/nvidia/delete-toolkit/test.file'
-	testing::docker_run::toolkit::shell 'toolkit delete /usr/local/nvidia/delete-toolkit'
+	testing::docker_run::toolkit::shell 'toolkit delete --toolkit-root=/usr/local/nvidia/delete-toolkit'
 
 	test ! -z "$(ls -A "${shared_dir}/usr/local/nvidia")"
 	test ! -e "${shared_dir}/usr/local/nvidia/delete-toolkit"

tools/container/crio/crio.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	hooks "github.com/containers/podman/v4/pkg/hooks/1.0.0"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	log "github.com/sirupsen/logrus"
@@ -164,7 +165,7 @@ func getHookPath(hooksDir string, hookFilename string) string {
 }
 
 func generateOciHook(toolkitDir string) hooks.Hook {
-	hookPath := filepath.Join(toolkitDir, "nvidia-container-toolkit")
+	hookPath := filepath.Join(toolkitDir, config.NVIDIAContainerRuntimeHookExecutable)
 	envPath := "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:" + toolkitDir
 	always := true
 
@@ -173,7 +174,7 @@ func generateOciHook(toolkitDir string) hooks.Hook {
 		Stages:  []string{"prestart"},
 		Hook: rspec.Hook{
 			Path: hookPath,
-			Args: []string{"nvidia-container-toolkit", "prestart"},
+			Args: []string{filepath.Base(config.NVIDIAContainerRuntimeHookExecutable), "prestart"},
 			Env:  []string{envPath},
 		},
 		When: hooks.When{

tools/container/docker/docker.go

@@ -17,16 +17,14 @@
 package main
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
 	"syscall"
 	"time"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 )
@@ -247,52 +245,15 @@ func ParseArgs(c *cli.Context) (string, error) {
 
 // LoadConfig loads the docker config from disk
 func LoadConfig(config string) (map[string]interface{}, error) {
-	log.Infof("Loading config: %v", config)
-
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	cfg := make(map[string]interface{})
-
-	if os.IsNotExist(err) {
-		log.Infof("Config file does not exist, creating new one")
-		return cfg, nil
-	}
-
-	readBytes, err := ioutil.ReadFile(config)
-	if err != nil {
-		return nil, fmt.Errorf("unable to read config: %v", err)
-	}
-
-	reader := bytes.NewReader(readBytes)
-	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-	return cfg, nil
+	return docker.LoadConfig(config)
 }
 
 // UpdateConfig updates the docker config to include the nvidia runtimes
 func UpdateConfig(config map[string]interface{}, o *options) error {
 	defaultRuntime := o.getDefaultRuntime()
-	if defaultRuntime != "" {
-		config["default-runtime"] = defaultRuntime
-	}
+	runtimes := o.runtimes()
 
-	runtimes := make(map[string]interface{})
-	if _, exists := config["runtimes"]; exists {
-		runtimes = config["runtimes"].(map[string]interface{})
-	}
-
-	for name, rt := range o.runtimes() {
-		runtimes[name] = rt
-	}
-
-	config["runtimes"] = runtimes
-	return nil
+	return docker.UpdateConfig(config, defaultRuntime, runtimes)
 }
 
 //RevertConfig reverts the docker config to remove the nvidia runtime
@@ -320,36 +281,7 @@ func RevertConfig(config map[string]interface{}) error {
 
 // FlushConfig flushes the updated/reverted config out to disk
 func FlushConfig(cfg map[string]interface{}, config string) error {
-	log.Infof("Flushing config")
-
-	output, err := json.MarshalIndent(cfg, "", "    ")
-	if err != nil {
-		return fmt.Errorf("unable to convert to JSON: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(config)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(config)
-		if err != nil {
-			return fmt.Errorf("unable to open %v for writing: %v", config, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(string(output))
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
+	return docker.FlushConfig(cfg, config)
 }
 
 // RestartDocker restarts docker depending on the value of restartModeFlag

tools/container/nvidia-toolkit/run.go

@@ -32,7 +32,6 @@ var signalReceived = make(chan bool, 1)
 
 var destinationArg string
 var noDaemonFlag bool
-var toolkitArgsFlag string
 var runtimeFlag string
 var runtimeArgsFlag string
 
@@ -44,7 +43,7 @@ func main() {
 	c := cli.NewApp()
 	c.Name = "nvidia-toolkit"
 	c.Usage = "Install the nvidia-container-toolkit for use by a given runtime"
-	c.UsageText = "DESTINATION [-n | --no-daemon] [-t | --toolkit-args] [-r | --runtime] [-u | --runtime-args]"
+	c.UsageText = "DESTINATION [-n | --no-daemon] [-r | --runtime] [-u | --runtime-args]"
 	c.Description = "DESTINATION points to the host path underneath which the nvidia-container-toolkit should be installed.\nIt will be installed at ${DESTINATION}/toolkit"
 	c.Version = Version
 	c.Action = Run
@@ -58,14 +57,6 @@ func main() {
 			Destination: &noDaemonFlag,
 			EnvVars:     []string{"NO_DAEMON"},
 		},
-		&cli.StringFlag{
-			Name:        "toolkit-args",
-			Aliases:     []string{"t"},
-			Usage:       "arguments to pass to the underlying 'toolkit' command",
-			Value:       defaultToolkitArgs,
-			Destination: &toolkitArgsFlag,
-			EnvVars:     []string{"TOOLKIT_ARGS"},
-		},
 		&cli.StringFlag{
 			Name:        "runtime",
 			Aliases:     []string{"r"},
@@ -221,17 +212,21 @@ func initialize() error {
 }
 
 func installToolkit() error {
-	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
-
 	log.Infof("Installing toolkit")
 
-	cmdline := fmt.Sprintf("%v install %v %v\n", toolkitCommand, toolkitArgsFlag, toolkitDir)
-	cmd := exec.Command("sh", "-c", cmdline)
+	cmdline := []string{
+		toolkitCommand,
+		"install",
+		"--toolkit-root",
+		filepath.Join(destinationArg, toolkitSubDir),
+	}
+
+	cmd := exec.Command("sh", "-c", strings.Join(cmdline, " "))
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err := cmd.Run()
 	if err != nil {
-		return fmt.Errorf("error running %v command: %v", toolkitCommand, err)
+		return fmt.Errorf("error running %v command: %v", cmdline, err)
 	}
 
 	return nil

tools/container/toolkit/toolkit.go

@@ -33,19 +33,27 @@ const (
 	DefaultNvidiaDriverRoot = "/run/nvidia/driver"
 
 	nvidiaContainerCliSource         = "/usr/bin/nvidia-container-cli"
-	nvidiaContainerRuntimeHookSource = "/usr/bin/nvidia-container-toolkit"
+	nvidiaContainerRuntimeHookSource = "/usr/bin/nvidia-container-runtime-hook"
 
 	nvidiaContainerToolkitConfigSource = "/etc/nvidia-container-runtime/config.toml"
 	configFilename                     = "config.toml"
 )
 
-var toolkitDirArg string
-var nvidiaDriverRootFlag string
-var nvidiaContainerRuntimeDebugFlag string
-var nvidiaContainerRuntimeLogLevelFlag string
-var nvidiaContainerCLIDebugFlag string
+type options struct {
+	DriverRoot               string
+	ContainerRuntimeDebug    string
+	ContainerRuntimeLogLevel string
+	ContainerCLIDebug        string
+	toolkitRoot              string
+
+	acceptNVIDIAVisibleDevicesWhenUnprivileged bool
+	acceptNVIDIAVisibleDevicesAsVolumeMounts   bool
+}
 
 func main() {
+
+	opts := options{}
+
 	// Create the top-level CLI
 	c := cli.NewApp()
 	c.Name = "toolkit"
@@ -57,16 +65,24 @@ func main() {
 	install.Name = "install"
 	install.Usage = "Install the components of the NVIDIA container toolkit"
 	install.ArgsUsage = "<toolkit_directory>"
-	install.Before = parseArgs
-	install.Action = Install
+	install.Before = func(c *cli.Context) error {
+		return validateOptions(c, &opts)
+	}
+	install.Action = func(c *cli.Context) error {
+		return Install(c, &opts)
+	}
 
 	// Create the 'delete' command
 	delete := cli.Command{}
 	delete.Name = "delete"
 	delete.Usage = "Delete the NVIDIA container toolkit"
 	delete.ArgsUsage = "<toolkit_directory>"
-	delete.Before = parseArgs
-	delete.Action = Delete
+	delete.Before = func(c *cli.Context) error {
+		return validateOptions(c, &opts)
+	}
+	delete.Action = func(c *cli.Context) error {
+		return Delete(c, &opts)
+	}
 
 	// Register the subcommand with the top-level CLI
 	c.Commands = []*cli.Command{
@@ -78,30 +94,50 @@ func main() {
 		&cli.StringFlag{
 			Name:        "nvidia-driver-root",
 			Value:       DefaultNvidiaDriverRoot,
-			Destination: &nvidiaDriverRootFlag,
+			Destination: &opts.DriverRoot,
 			EnvVars:     []string{"NVIDIA_DRIVER_ROOT"},
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-container-runtime-debug",
 			Usage:       "Specify the location of the debug log file for the NVIDIA Container Runtime",
-			Destination: &nvidiaContainerRuntimeDebugFlag,
+			Destination: &opts.ContainerRuntimeDebug,
 			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_DEBUG"},
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-container-runtime-debug-log-level",
-			Destination: &nvidiaContainerRuntimeLogLevelFlag,
+			Destination: &opts.ContainerRuntimeLogLevel,
 			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_LOG_LEVEL"},
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-container-cli-debug",
 			Usage:       "Specify the location of the debug log file for the NVIDIA Container CLI",
-			Destination: &nvidiaContainerCLIDebugFlag,
+			Destination: &opts.ContainerCLIDebug,
 			EnvVars:     []string{"NVIDIA_CONTAINER_CLI_DEBUG"},
 		},
+		&cli.BoolFlag{
+			Name:        "accept-nvidia-visible-devices-envvar-when-unprivileged",
+			Usage:       "Set the accept-nvidia-visible-devices-envvar-when-unprivileged config option",
+			Destination: &opts.acceptNVIDIAVisibleDevicesWhenUnprivileged,
+			EnvVars:     []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_ENVVAR_WHEN_UNPRIVILEGED"},
+		},
+		&cli.BoolFlag{
+			Name:        "accept-nvidia-visible-devices-as-volume-mounts",
+			Usage:       "Set the accept-nvidia-visible-devices-as-volume-mounts config option",
+			Destination: &opts.acceptNVIDIAVisibleDevicesWhenUnprivileged,
+			EnvVars:     []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_AS_VOLUME_MOUNTS"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-root",
+			Usage:       "The directory where the NVIDIA Container toolkit is to be installed",
+			Required:    true,
+			Destination: &opts.toolkitRoot,
+			EnvVars:     []string{"TOOLKIT_ROOT"},
+		},
 	}
 
 	// Update the subcommand flags with the common subcommand flags
 	install.Flags = append([]cli.Flag{}, flags...)
+	delete.Flags = append([]cli.Flag{}, flags...)
 
 	// Run the top-level CLI
 	if err := c.Run(os.Args); err != nil {
@@ -109,24 +145,19 @@ func main() {
 	}
 }
 
-// parseArgs parses the command line arguments to the CLI
-func parseArgs(c *cli.Context) error {
-	args := c.Args()
-
-	log.Infof("Parsing arguments: %v", args.Slice())
-	if c.NArg() != 1 {
-		return fmt.Errorf("incorrect number of arguments")
+// validateOptions checks whether the specified options are valid
+func validateOptions(c *cli.Context, opts *options) error {
+	if opts.toolkitRoot == "" {
+		return fmt.Errorf("invalid --toolkit-root option: %v", opts.toolkitRoot)
 	}
-	toolkitDirArg = args.Get(0)
-	log.Infof("Successfully parsed arguments")
 
 	return nil
 }
 
 // Delete removes the NVIDIA container toolkit
-func Delete(cli *cli.Context) error {
-	log.Infof("Deleting NVIDIA container toolkit from '%v'", toolkitDirArg)
-	err := os.RemoveAll(toolkitDirArg)
+func Delete(cli *cli.Context, opts *options) error {
+	log.Infof("Deleting NVIDIA container toolkit from '%v'", opts.toolkitRoot)
+	err := os.RemoveAll(opts.toolkitRoot)
 	if err != nil {
 		return fmt.Errorf("error deleting toolkit directory: %v", err)
 	}
@@ -135,44 +166,44 @@ func Delete(cli *cli.Context) error {
 
 // Install installs the components of the NVIDIA container toolkit.
 // Any existing installation is removed.
-func Install(cli *cli.Context) error {
-	log.Infof("Installing NVIDIA container toolkit to '%v'", toolkitDirArg)
+func Install(cli *cli.Context, opts *options) error {
+	log.Infof("Installing NVIDIA container toolkit to '%v'", opts.toolkitRoot)
 
 	log.Infof("Removing existing NVIDIA container toolkit installation")
-	err := os.RemoveAll(toolkitDirArg)
+	err := os.RemoveAll(opts.toolkitRoot)
 	if err != nil {
 		return fmt.Errorf("error removing toolkit directory: %v", err)
 	}
 
-	toolkitConfigDir := filepath.Join(toolkitDirArg, ".config", "nvidia-container-runtime")
+	toolkitConfigDir := filepath.Join(opts.toolkitRoot, ".config", "nvidia-container-runtime")
 	toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename)
 
-	err = createDirectories(toolkitDirArg, toolkitConfigDir)
+	err = createDirectories(opts.toolkitRoot, toolkitConfigDir)
 	if err != nil {
 		return fmt.Errorf("could not create required directories: %v", err)
 	}
 
-	err = installContainerLibraries(toolkitDirArg)
+	err = installContainerLibraries(opts.toolkitRoot)
 	if err != nil {
 		return fmt.Errorf("error installing NVIDIA container library: %v", err)
 	}
 
-	err = installContainerRuntimes(toolkitDirArg, nvidiaDriverRootFlag)
+	err = installContainerRuntimes(opts.toolkitRoot, opts.DriverRoot)
 	if err != nil {
 		return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
 	}
 
-	nvidiaContainerCliExecutable, err := installContainerCLI(toolkitDirArg)
+	nvidiaContainerCliExecutable, err := installContainerCLI(opts.toolkitRoot)
 	if err != nil {
 		return fmt.Errorf("error installing NVIDIA container CLI: %v", err)
 	}
 
-	_, err = installRuntimeHook(toolkitDirArg, toolkitConfigPath)
+	_, err = installRuntimeHook(opts.toolkitRoot, toolkitConfigPath)
 	if err != nil {
 		return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
 	}
 
-	err = installToolkitConfig(toolkitConfigPath, nvidiaDriverRootFlag, nvidiaContainerCliExecutable)
+	err = installToolkitConfig(toolkitConfigPath, nvidiaContainerCliExecutable, opts)
 	if err != nil {
 		return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err)
 	}
@@ -185,8 +216,8 @@ func Install(cli *cli.Context) error {
 // A predefined set of library candidates are considered, with the first one
 // resulting in success being installed to the toolkit folder. The install process
 // resolves the symlink for the library and copies the versioned library itself.
-func installContainerLibraries(toolkitDir string) error {
-	log.Infof("Installing NVIDIA container library to '%v'", toolkitDir)
+func installContainerLibraries(toolkitRoot string) error {
+	log.Infof("Installing NVIDIA container library to '%v'", toolkitRoot)
 
 	libs := []string{
 		"libnvidia-container.so.1",
@@ -194,7 +225,7 @@ func installContainerLibraries(toolkitDir string) error {
 	}
 
 	for _, l := range libs {
-		err := installLibrary(l, toolkitDir)
+		err := installLibrary(l, toolkitRoot)
 		if err != nil {
 			return fmt.Errorf("failed to install %s: %v", l, err)
 		}
@@ -204,15 +235,15 @@ func installContainerLibraries(toolkitDir string) error {
 }
 
 // installLibrary installs the specified library to the toolkit directory.
-func installLibrary(libName string, toolkitDir string) error {
+func installLibrary(libName string, toolkitRoot string) error {
 	libraryPath, err := findLibrary("", libName)
 	if err != nil {
 		return fmt.Errorf("error locating NVIDIA container library: %v", err)
 	}
 
-	installedLibPath, err := installFileToFolder(toolkitDir, libraryPath)
+	installedLibPath, err := installFileToFolder(toolkitRoot, libraryPath)
 	if err != nil {
-		return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitDir, err)
+		return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitRoot, err)
 	}
 	log.Infof("Installed '%v' to '%v'", libraryPath, installedLibPath)
 
@@ -220,7 +251,7 @@ func installLibrary(libName string, toolkitDir string) error {
 		return nil
 	}
 
-	err = installSymlink(toolkitDir, libName, installedLibPath)
+	err = installSymlink(toolkitRoot, libName, installedLibPath)
 	if err != nil {
 		return fmt.Errorf("error installing symlink for NVIDIA container library: %v", err)
 	}
@@ -230,7 +261,7 @@ func installLibrary(libName string, toolkitDir string) error {
 
 // installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring
 // that the settings are updated to match the desired install and nvidia driver directories.
-func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvidiaContainerCliExecutablePath string) error {
+func installToolkitConfig(toolkitConfigPath string, nvidiaContainerCliExecutablePath string, opts *options) error {
 	log.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
 
 	config, err := toml.LoadFile(nvidiaContainerToolkitConfigSource)
@@ -244,6 +275,10 @@ func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvid
 	}
 	defer targetConfig.Close()
 
+	// Set the options in the root toml table
+	config.Set("accept-nvidia-visible-devices-envvar-when-unprivileged", opts.acceptNVIDIAVisibleDevicesWhenUnprivileged)
+	config.Set("accept-nvidia-visible-devices-as-volume-mounts", opts.acceptNVIDIAVisibleDevicesAsVolumeMounts)
+
 	nvidiaContainerCliKey := func(p string) []string {
 		return []string{"nvidia-container-cli", p}
 	}
@@ -253,17 +288,17 @@ func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvid
 	ldconfigPath := fmt.Sprintf("%s", config.GetPath(nvidiaContainerCliKey("ldconfig")))
 
 	// Use the driver run root as the root:
-	driverLdconfigPath := "@" + filepath.Join(nvidiaDriverDir, strings.TrimPrefix(ldconfigPath, "@/"))
+	driverLdconfigPath := "@" + filepath.Join(opts.DriverRoot, strings.TrimPrefix(ldconfigPath, "@/"))
 
-	config.SetPath(nvidiaContainerCliKey("root"), nvidiaDriverDir)
+	config.SetPath(nvidiaContainerCliKey("root"), opts.DriverRoot)
 	config.SetPath(nvidiaContainerCliKey("path"), nvidiaContainerCliExecutablePath)
 	config.SetPath(nvidiaContainerCliKey("ldconfig"), driverLdconfigPath)
 
 	// Set the debug options if selected
 	debugOptions := map[string]string{
-		"nvidia-container-runtime.debug":     nvidiaContainerRuntimeDebugFlag,
-		"nvidia-container-runtime.log-level": nvidiaContainerRuntimeLogLevelFlag,
-		"nvidia-container-cli.debug":         nvidiaContainerCLIDebugFlag,
+		"nvidia-container-runtime.debug":     opts.ContainerRuntimeDebug,
+		"nvidia-container-runtime.log-level": opts.ContainerRuntimeLogLevel,
+		"nvidia-container-cli.debug":         opts.ContainerCLIDebug,
 	}
 	for key, value := range debugOptions {
 		if value == "" {
@@ -284,11 +319,11 @@ func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvid
 
 // installContainerCLI sets up the NVIDIA container CLI executable, copying the executable
 // and implementing the required wrapper
-func installContainerCLI(toolkitDir string) (string, error) {
+func installContainerCLI(toolkitRoot string) (string, error) {
 	log.Infof("Installing NVIDIA container CLI from '%v'", nvidiaContainerCliSource)
 
 	env := map[string]string{
-		"LD_LIBRARY_PATH": toolkitDir,
+		"LD_LIBRARY_PATH": toolkitRoot,
 	}
 
 	e := executable{
@@ -300,7 +335,7 @@ func installContainerCLI(toolkitDir string) (string, error) {
 		env: env,
 	}
 
-	installedPath, err := e.install(toolkitDir)
+	installedPath, err := e.install(toolkitRoot)
 	if err != nil {
 		return "", fmt.Errorf("error installing NVIDIA container CLI: %v", err)
 	}
@@ -309,7 +344,7 @@ func installContainerCLI(toolkitDir string) (string, error) {
 
 // installRuntimeHook sets up the NVIDIA runtime hook, copying the executable
 // and implementing the required wrapper
-func installRuntimeHook(toolkitDir string, configFilePath string) (string, error) {
+func installRuntimeHook(toolkitRoot string, configFilePath string) (string, error) {
 	log.Infof("Installing NVIDIA container runtime hook from '%v'", nvidiaContainerRuntimeHookSource)
 
 	argLines := []string{
@@ -319,18 +354,18 @@ func installRuntimeHook(toolkitDir string, configFilePath string) (string, error
 	e := executable{
 		source: nvidiaContainerRuntimeHookSource,
 		target: executableTarget{
-			dotfileName: "nvidia-container-toolkit.real",
-			wrapperName: "nvidia-container-toolkit",
+			dotfileName: "nvidia-container-runtime-hook.real",
+			wrapperName: "nvidia-container-runtime-hook",
 		},
 		argLines: argLines,
 	}
 
-	installedPath, err := e.install(toolkitDir)
+	installedPath, err := e.install(toolkitRoot)
 	if err != nil {
 		return "", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
 	}
 
-	err = installSymlink(toolkitDir, "nvidia-container-runtime-hook", installedPath)
+	err = installSymlink(toolkitRoot, "nvidia-container-toolkit", installedPath)
 	if err != nil {
 		return "", fmt.Errorf("error installing symlink to NVIDIA container runtime hook: %v", err)
 	}
@@ -340,8 +375,8 @@ func installRuntimeHook(toolkitDir string, configFilePath string) (string, error
 
 // installSymlink creates a symlink in the toolkitDirectory that points to the specified target.
 // Note: The target is assumed to be local to the toolkit directory
-func installSymlink(toolkitDir string, link string, target string) error {
-	symlinkPath := filepath.Join(toolkitDir, link)
+func installSymlink(toolkitRoot string, link string, target string) error {
+	symlinkPath := filepath.Join(toolkitRoot, link)
 	targetPath := filepath.Base(target)
 	log.Infof("Creating symlink '%v' -> '%v'", symlinkPath, targetPath)
 

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go

@@ -22,6 +22,8 @@ import (
 	"strings"
 	"sync"
 
+	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/fsnotify/fsnotify"
 	"github.com/hashicorp/go-multierror"
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -33,30 +35,46 @@ type Option func(*Cache) error
 // Cache stores CDI Specs loaded from Spec directories.
 type Cache struct {
 	sync.Mutex
-	specDirs []string
-	specs    map[string][]*Spec
-	devices  map[string]*Device
-	errors   map[string][]error
+	specDirs  []string
+	specs     map[string][]*Spec
+	devices   map[string]*Device
+	errors    map[string][]error
+	dirErrors map[string]error
+
+	autoRefresh bool
+	watch       *watch
+}
+
+// WithAutoRefresh returns an option to control automatic Cache refresh.
+// By default auto-refresh is enabled, the list of Spec directories are
+// monitored and the Cache is automatically refreshed whenever a change
+// is detected. This option can be used to disable this behavior when a
+// manually refreshed mode is preferable.
+func WithAutoRefresh(autoRefresh bool) Option {
+	return func(c *Cache) error {
+		c.autoRefresh = autoRefresh
+		return nil
+	}
 }
 
 // NewCache creates a new CDI Cache. The cache is populated from a set
 // of CDI Spec directories. These can be specified using a WithSpecDirs
 // option. The default set of directories is exposed in DefaultSpecDirs.
 func NewCache(options ...Option) (*Cache, error) {
-	c := &Cache{}
-
-	if err := c.Configure(options...); err != nil {
-		return nil, err
-	}
-	if len(c.specDirs) == 0 {
-		c.Configure(WithSpecDirs(DefaultSpecDirs...))
+	c := &Cache{
+		autoRefresh: true,
+		watch:       &watch{},
 	}
 
-	return c, c.Refresh()
+	WithSpecDirs(DefaultSpecDirs...)(c)
+	c.Lock()
+	defer c.Unlock()
+
+	return c, c.configure(options...)
 }
 
-// Configure applies options to the cache. Updates the cache if options have
-// changed.
+// Configure applies options to the Cache. Updates and refreshes the
+// Cache if options have changed.
 func (c *Cache) Configure(options ...Option) error {
 	if len(options) == 0 {
 		return nil
@@ -65,17 +83,54 @@ func (c *Cache) Configure(options ...Option) error {
 	c.Lock()
 	defer c.Unlock()
 
+	return c.configure(options...)
+}
+
+// Configure the Cache. Start/stop CDI Spec directory watch, refresh
+// the Cache if necessary.
+func (c *Cache) configure(options ...Option) error {
+	var err error
+
 	for _, o := range options {
-		if err := o(c); err != nil {
+		if err = o(c); err != nil {
 			return errors.Wrapf(err, "failed to apply cache options")
 		}
 	}
 
+	c.dirErrors = make(map[string]error)
+
+	c.watch.stop()
+	if c.autoRefresh {
+		c.watch.setup(c.specDirs, c.dirErrors)
+		c.watch.start(&c.Mutex, c.refresh, c.dirErrors)
+	}
+	c.refresh()
+
 	return nil
 }
 
 // Refresh rescans the CDI Spec directories and refreshes the Cache.
+// In manual refresh mode the cache is always refreshed. In auto-
+// refresh mode the cache is only refreshed if it is out of date.
 func (c *Cache) Refresh() error {
+	c.Lock()
+	defer c.Unlock()
+
+	// force a refresh in manual mode
+	if refreshed, err := c.refreshIfRequired(!c.autoRefresh); refreshed {
+		return err
+	}
+
+	// collect and return cached errors, much like refresh() does it
+	var result error
+	for _, err := range c.errors {
+		result = multierror.Append(result, err...)
+	}
+	return result
+}
+
+// Refresh the Cache by rescanning CDI Spec directories and files.
+func (c *Cache) refresh() error {
 	var (
 		specs      = map[string][]*Spec{}
 		devices    = map[string]*Device{}
@@ -135,9 +190,6 @@ func (c *Cache) Refresh() error {
 		delete(devices, conflict)
 	}
 
-	c.Lock()
-	defer c.Unlock()
-
 	c.specs = specs
 	c.devices = devices
 	c.errors = specErrors
@@ -149,6 +201,17 @@ func (c *Cache) Refresh() error {
 	return nil
 }
 
+// RefreshIfRequired triggers a refresh if necessary.
+func (c *Cache) refreshIfRequired(force bool) (bool, error) {
+	// We need to refresh if
+	// - it's forced by an explicitly call to Refresh() in manual mode
+	// - a missing Spec dir appears (added to watch) in auto-refresh mode
+	if force || (c.autoRefresh && c.watch.update(c.dirErrors)) {
+		return true, c.refresh()
+	}
+	return false, nil
+}
+
 // InjectDevices injects the given qualified devices to an OCI Spec. It
 // returns any unresolvable devices and an error if injection fails for
 // any of the devices.
@@ -162,6 +225,8 @@ func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, e
 	c.Lock()
 	defer c.Unlock()
 
+	c.refreshIfRequired(false)
+
 	edits := &ContainerEdits{}
 	specs := map[*Spec]struct{}{}
 
@@ -190,11 +255,46 @@ func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, e
 	return nil, nil
 }
 
+// WriteSpec writes a Spec file with the given content. Priority is used
+// as an index into the list of Spec directories to pick a directory for
+// the file, adjusting for any under- or overflows. If name has a "json"
+// or "yaml" extension it choses the encoding. Otherwise JSON encoding
+// is used with a "json" extension.
+func (c *Cache) WriteSpec(raw *cdi.Spec, name string) error {
+	var (
+		specDir string
+		path    string
+		prio    int
+		spec    *Spec
+		err     error
+	)
+
+	if len(c.specDirs) == 0 {
+		return errors.New("no Spec directories to write to")
+	}
+
+	prio = len(c.specDirs) - 1
+	specDir = c.specDirs[prio]
+	path = filepath.Join(specDir, name)
+	if ext := filepath.Ext(path); ext != ".json" && ext != ".yaml" {
+		path += ".json"
+	}
+
+	spec, err = NewSpec(raw, path, prio)
+	if err != nil {
+		return err
+	}
+
+	return spec.Write(true)
+}
+
 // GetDevice returns the cached device for the given qualified name.
 func (c *Cache) GetDevice(device string) *Device {
 	c.Lock()
 	defer c.Unlock()
 
+	c.refreshIfRequired(false)
+
 	return c.devices[device]
 }
 
@@ -205,6 +305,8 @@ func (c *Cache) ListDevices() []string {
 	c.Lock()
 	defer c.Unlock()
 
+	c.refreshIfRequired(false)
+
 	for name := range c.devices {
 		devices = append(devices, name)
 	}
@@ -220,6 +322,8 @@ func (c *Cache) ListVendors() []string {
 	c.Lock()
 	defer c.Unlock()
 
+	c.refreshIfRequired(false)
+
 	for vendor := range c.specs {
 		vendors = append(vendors, vendor)
 	}
@@ -238,6 +342,8 @@ func (c *Cache) ListClasses() []string {
 	c.Lock()
 	defer c.Unlock()
 
+	c.refreshIfRequired(false)
+
 	for _, specs := range c.specs {
 		for _, spec := range specs {
 			cmap[spec.GetClass()] = struct{}{}
@@ -256,6 +362,8 @@ func (c *Cache) GetVendorSpecs(vendor string) []*Spec {
 	c.Lock()
 	defer c.Unlock()
 
+	c.refreshIfRequired(false)
+
 	return c.specs[vendor]
 }
 
@@ -268,12 +376,158 @@ func (c *Cache) GetSpecErrors(spec *Spec) []error {
 // GetErrors returns all errors encountered during the last
 // cache refresh.
 func (c *Cache) GetErrors() map[string][]error {
-	return c.errors
+	c.Lock()
+	defer c.Unlock()
+
+	errors := map[string][]error{}
+	for path, errs := range c.errors {
+		errors[path] = errs
+	}
+	for path, err := range c.dirErrors {
+		errors[path] = []error{err}
+	}
+
+	return errors
 }
 
 // GetSpecDirectories returns the CDI Spec directories currently in use.
 func (c *Cache) GetSpecDirectories() []string {
+	c.Lock()
+	defer c.Unlock()
+
 	dirs := make([]string, len(c.specDirs))
 	copy(dirs, c.specDirs)
 	return dirs
 }
+
+// GetSpecDirErrors returns any errors related to configured Spec directories.
+func (c *Cache) GetSpecDirErrors() map[string]error {
+	if c.dirErrors == nil {
+		return nil
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	errors := make(map[string]error)
+	for dir, err := range c.dirErrors {
+		errors[dir] = err
+	}
+	return errors
+}
+
+// Our fsnotify helper wrapper.
+type watch struct {
+	watcher *fsnotify.Watcher
+	tracked map[string]bool
+}
+
+// Setup monitoring for the given Spec directories.
+func (w *watch) setup(dirs []string, dirErrors map[string]error) {
+	var (
+		dir string
+		err error
+	)
+	w.tracked = make(map[string]bool)
+	for _, dir = range dirs {
+		w.tracked[dir] = false
+	}
+
+	w.watcher, err = fsnotify.NewWatcher()
+	if err != nil {
+		for _, dir := range dirs {
+			dirErrors[dir] = errors.Wrap(err, "failed to create watcher")
+		}
+		return
+	}
+
+	w.update(dirErrors)
+}
+
+// Start watching Spec directories for relevant changes.
+func (w *watch) start(m *sync.Mutex, refresh func() error, dirErrors map[string]error) {
+	go w.watch(m, refresh, dirErrors)
+}
+
+// Stop watching directories.
+func (w *watch) stop() {
+	if w.watcher == nil {
+		return
+	}
+
+	w.watcher.Close()
+	w.tracked = nil
+}
+
+// Watch Spec directory changes, triggering a refresh if necessary.
+func (w *watch) watch(m *sync.Mutex, refresh func() error, dirErrors map[string]error) {
+	watch := w.watcher
+	if watch == nil {
+		return
+	}
+	for {
+		select {
+		case event, ok := <-watch.Events:
+			if !ok {
+				return
+			}
+
+			if (event.Op & (fsnotify.Rename | fsnotify.Remove | fsnotify.Write)) == 0 {
+				continue
+			}
+			if event.Op == fsnotify.Write {
+				if ext := filepath.Ext(event.Name); ext != ".json" && ext != ".yaml" {
+					continue
+				}
+			}
+
+			m.Lock()
+			if event.Op == fsnotify.Remove && w.tracked[event.Name] {
+				w.update(dirErrors, event.Name)
+			} else {
+				w.update(dirErrors)
+			}
+			refresh()
+			m.Unlock()
+
+		case _, ok := <-watch.Errors:
+			if !ok {
+				return
+			}
+		}
+	}
+}
+
+// Update watch with pending/missing or removed directories.
+func (w *watch) update(dirErrors map[string]error, removed ...string) bool {
+	var (
+		dir    string
+		ok     bool
+		err    error
+		update bool
+	)
+
+	for dir, ok = range w.tracked {
+		if ok {
+			continue
+		}
+
+		err = w.watcher.Add(dir)
+		if err == nil {
+			w.tracked[dir] = true
+			delete(dirErrors, dir)
+			update = true
+		} else {
+			w.tracked[dir] = false
+			dirErrors[dir] = errors.Wrap(err, "failed to monitor for changes")
+		}
+	}
+
+	for _, dir = range removed {
+		w.tracked[dir] = false
+		dirErrors[dir] = errors.New("directory removed")
+		update = true
+	}
+
+	return update
+}

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/doc.go

@@ -67,6 +67,21 @@
 //
 // Cache Refresh
 //
+// By default the CDI Spec cache monitors the configured Spec directories
+// and automatically refreshes itself when necessary. This behavior can be
+// disabled using the WithAutoRefresh(false) option.
+//
+// Failure to set up monitoring for a Spec directory causes the directory to
+// get ignored and an error to be recorded among the Spec directory errors.
+// These errors can be queried using the GetSpecDirErrors() function. If the
+// error condition is transient, for instance a missing directory which later
+// gets created, the corresponding error will be removed once the condition
+// is over.
+//
+// With auto-refresh enabled injecting any CDI devices can be done without
+// an explicit call to Refresh(), using a code snippet similar to the
+// following:
+//
 // In a runtime implementation one typically wants to make sure the
 // CDI Spec cache is up to date before performing device injection.
 // A code snippet similar to the following accmplishes that:
@@ -127,4 +142,24 @@
 // The default directories are '/etc/cdi' and '/var/run/cdi'. By putting
 // dynamically generated Spec files under '/var/run/cdi', those take
 // precedence over static ones in '/etc/cdi'.
+//
+// CDI Spec Validation
+//
+// This package performs both syntactic and semantic validation of CDI
+// Spec file data when a Spec file is loaded via the registry or using
+// the ReadSpec API function. As part of the semantic verification, the
+// Spec file is verified against the CDI Spec JSON validation schema.
+//
+// If a valid externally provided JSON validation schema is found in
+// the filesystem at /etc/cdi/schema/schema.json it is loaded and used
+// as the default validation schema. If such a file is not found or
+// fails to load, an embedded no-op schema is used.
+//
+// The used validation schema can also be changed programmatically using
+// the SetSchema API convenience function. This function also accepts
+// the special "builtin" (BuiltinSchemaName) and "none" (NoneSchemaName)
+// schema names which switch the used schema to the in-repo validation
+// schema embedded into the binary or the now default no-op schema
+// correspondingly. Other names are interpreted as the path to the actual
+// validation schema to load and use.
 package cdi