Merge branch 'fix-hook' into 'main'

Handle empty root in config

See merge request nvidia/container-toolkit/container-toolkit!454

.nvidia-ci.yml

@@ -140,15 +140,6 @@ scan-centos7-amd64:
   needs:
     - image-centos7
 
-scan-centos7-arm64:
-  extends:
-    - .dist-centos7
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-centos7
-    - scan-centos7-amd64
-
 scan-ubuntu20.04-amd64:
   extends:
     - .dist-ubuntu20.04

CHANGELOG.md

@@ -1,5 +1,20 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.14.0-rc.2
+
+* Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
+* Remove dependency on coreutils when installing package on RPM-based systems.
+* Create ouput folders if required when running `nvidia-ctk runtime configure`
+* Generate default config as post-install step.
+* Added support for detecting GSP firmware at custom paths when generating CDI specifications.
+* Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.
+
+* [libnvidia-container] Include Shared Compiler Library (libnvidia-gpucomp.so) in the list of compute libaries.
+
+* [toolkit-container] Ensure that common envvars have higher priority when configuring the container engines.
+* [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
+
 ## v1.14.0-rc.1
 
 * Add support for updating containerd configs to the `nvidia-ctk runtime configure` command.
@@ -13,10 +28,11 @@
 * Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
 * Add option to load kernel modules when creating device nodes
 * Add option to create device nodes when creating `/dev/char` symlinks
-* Bump CUDA base image version to 12.1.1.
 
 * [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
 
+* [toolkit-container] Allow same envars for all runtime configs
+
 ## v1.13.1
 
 * Update `update-ldcache` hook to only update ldcache if it exists.

build/container/Dockerfile.centos

@@ -17,19 +17,29 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN yum install -y \
+    wget make git gcc \
+     && \
+    rm -rf /var/cache/yum/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .

build/container/Dockerfile.ubuntu

@@ -17,19 +17,28 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN apt-get update && \
+    apt-get install -y wget make git gcc \
+    && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .

cmd/nvidia-container-runtime-hook/container_config.go

@@ -38,8 +38,10 @@ type nvidiaConfig struct {
 	MigConfigDevices   string
 	MigMonitorDevices  string
 	DriverCapabilities string
-	Requirements       []string
-	DisableRequire     bool
+	// Requirements defines the requirements DSL for the container to run.
+	// This is empty if no specific requirements are needed, or if requirements are
+	// explicitly disabled.
+	Requirements []string
 }
 
 type containerConfig struct {
@@ -327,15 +329,12 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 		log.Panicln("failed to get requirements", err)
 	}
 
-	disableRequire := image.HasDisableRequire()
-
 	return &nvidiaConfig{
 		Devices:            devices,
 		MigConfigDevices:   migConfigDevices,
 		MigMonitorDevices:  migMonitorDevices,
 		DriverCapabilities: driverCapabilities,
 		Requirements:       requirements,
-		DisableRequire:     disableRequire,
 	}
 }
 
@@ -353,7 +352,10 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	image, err := image.New(
+		image.WithEnv(s.Process.Env),
+		image.WithDisableRequire(hook.DisableRequire),
+	)
 	if err != nil {
 		log.Panicln(err)
 	}

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -40,7 +40,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -54,7 +53,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -86,7 +84,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -100,7 +97,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -115,7 +111,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -130,7 +125,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -145,7 +139,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -162,7 +155,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -179,8 +171,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -211,7 +202,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -243,7 +233,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -257,7 +246,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -272,7 +260,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -287,7 +274,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -302,7 +288,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -319,7 +304,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -336,8 +320,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -351,7 +334,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -367,7 +349,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				MigConfigDevices:   "mig0,mig1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -393,7 +374,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				MigMonitorDevices:  "mig0,mig1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -525,7 +505,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
 
 			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
-			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
 		})
 	}
 }

cmd/nvidia-container-runtime-hook/main.go

@@ -92,7 +92,7 @@ func doPrestart() {
 	rootfs := getRootfsPath(container)
 
 	args := []string{getCLIPath(cli)}
-	if cli.Root != nil {
+	if cli.Root != nil && *cli.Root != "" {
 		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
 	}
 	if cli.LoadKmods {
@@ -137,10 +137,8 @@ func doPrestart() {
 		args = append(args, capabilityToCLI(cap))
 	}
 
-	if !hook.DisableRequire && !nvidia.DisableRequire {
-		for _, req := range nvidia.Requirements {
-			args = append(args, fmt.Sprintf("--require=%s", req))
-		}
+	for _, req := range nvidia.Requirements {
+		args = append(args, fmt.Sprintf("--require=%s", req))
 	}
 
 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -130,7 +130,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringSliceFlag{
 			Name:        "csv.file",
-			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CDI mode.",
+			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CSV mode.",
 			Value:       cli.NewStringSlice(csv.DefaultFileList()...),
 			Destination: &opts.csv.files,
 		},

cmd/nvidia-ctk/config/create-default/create-default.go

@@ -17,11 +17,14 @@
 package defaultsubcommand
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"regexp"
 
-	nvctkConfig "github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/urfave/cli/v2"
 )
@@ -32,7 +35,9 @@ type command struct {
 
 // options stores the subcommand options
 type options struct {
-	output string
+	config  string
+	output  string
+	inPlace bool
 }
 
 // NewCommand constructs a default command with the specified logger
@@ -61,9 +66,20 @@ func (m command) build() *cli.Command {
 	}
 
 	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "Specify the config file to process; The contents of this file overrides the default config",
+			Destination: &opts.config,
+		},
+		&cli.BoolFlag{
+			Name:        "in-place",
+			Aliases:     []string{"i"},
+			Usage:       "Modify the config file in-place",
+			Destination: &opts.inPlace,
+		},
 		&cli.StringFlag{
 			Name:        "output",
-			Usage:       "Specify the file to output the generated configuration for to. If this is '' the configuration is ouput to STDOUT.",
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
 			Destination: &opts.output,
 		},
 	}
@@ -72,31 +88,96 @@ func (m command) build() *cli.Command {
 }
 
 func (m command) validateFlags(c *cli.Context, opts *options) error {
+	if opts.inPlace {
+		if opts.output != "" {
+			return fmt.Errorf("cannot specify both --in-place and --output")
+		}
+		opts.output = opts.config
+	}
 	return nil
 }
 
 func (m command) run(c *cli.Context, opts *options) error {
-	defaultConfig, err := nvctkConfig.GetDefaultConfigToml()
-	if err != nil {
-		return fmt.Errorf("unable to get default config: %v", err)
+	if err := opts.ensureOutputFolder(); err != nil {
+		return fmt.Errorf("unable to create output directory: %v", err)
 	}
 
+	contents, err := opts.getFormattedConfig()
+	if err != nil {
+		return fmt.Errorf("unable to fix comments: %v", err)
+	}
+
+	if _, err := opts.Write(contents); err != nil {
+		return fmt.Errorf("unable to write to output: %v", err)
+	}
+
+	return nil
+}
+
+// getFormattedConfig returns the default config formatted as required from the specified config file.
+// The config is then formatted as required.
+// No indentation is used and comments are modified so that there is no space
+// after the '#' character.
+func (opts options) getFormattedConfig() ([]byte, error) {
+	cfg, err := config.Load(opts.config)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load or create config: %v", err)
+	}
+
+	buffer := bytes.NewBuffer(nil)
+
+	if _, err := cfg.Save(buffer); err != nil {
+		return nil, fmt.Errorf("unable to save config: %v", err)
+	}
+	return fixComments(buffer.Bytes())
+}
+
+func fixComments(contents []byte) ([]byte, error) {
+	r, err := regexp.Compile(`(\n*)\s*?#\s*(\S.*)`)
+	if err != nil {
+		return nil, fmt.Errorf("unable to compile regexp: %v", err)
+	}
+	replaced := r.ReplaceAll(contents, []byte("$1#$2"))
+
+	return replaced, nil
+}
+
+func (opts options) outputExists() (bool, error) {
+	if opts.output == "" {
+		return false, nil
+	}
+	_, err := os.Stat(opts.output)
+	if err == nil {
+		return true, nil
+	} else if !os.IsNotExist(err) {
+		return false, fmt.Errorf("unable to stat output file: %v", err)
+	}
+	return false, nil
+}
+
+func (opts options) ensureOutputFolder() error {
+	if opts.output == "" {
+		return nil
+	}
+	if dir := filepath.Dir(opts.output); dir != "" {
+		return os.MkdirAll(dir, 0755)
+	}
+	return nil
+}
+
+// Write writes the contents to the output file specified in the options.
+func (opts options) Write(contents []byte) (int, error) {
 	var output io.Writer
 	if opts.output == "" {
 		output = os.Stdout
 	} else {
 		outputFile, err := os.Create(opts.output)
 		if err != nil {
-			return fmt.Errorf("unable to create output file: %v", err)
+			return 0, fmt.Errorf("unable to create output file: %v", err)
 		}
 		defer outputFile.Close()
 		output = outputFile
 	}
 
-	_, err = defaultConfig.WriteTo(output)
-	if err != nil {
-		return fmt.Errorf("unable to write to output: %v", err)
-	}
-
-	return nil
+	return output.Write(contents)
 }

cmd/nvidia-ctk/config/create-default/create-default_test.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package defaultsubcommand
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFixComment(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "# comment",
+			expected: "#comment",
+		},
+		{
+			input:    " #comment",
+			expected: "#comment",
+		},
+		{
+			input:    " # comment",
+			expected: "#comment",
+		},
+		{
+			input: strings.Join([]string{
+				"some",
+				"# comment",
+				" # comment",
+				" #comment",
+				"other"}, "\n"),
+			expected: strings.Join([]string{
+				"some",
+				"#comment",
+				"#comment",
+				"#comment",
+				"other"}, "\n"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			actual, _ := fixComments([]byte(tc.input))
+			require.Equal(t, tc.expected, string(actual))
+		})
+	}
+}
+
+func TestGetFormattedConfig(t *testing.T) {
+	expectedLines := []string{
+		"#no-cgroups = false",
+		"#debug = \"/var/log/nvidia-container-toolkit.log\"",
+		"#debug = \"/var/log/nvidia-container-runtime.log\"",
+	}
+
+	opts := &options{}
+	contents, err := opts.getFormattedConfig()
+	require.NoError(t, err)
+	lines := strings.Split(string(contents), "\n")
+
+	for _, line := range expectedLines {
+		require.Contains(t, lines, line)
+	}
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -101,7 +101,10 @@ func (m command) run(c *cli.Context, cfg *config) error {
 
 	csvFiles := cfg.filenames.Value()
 
-	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
+	chainLocator := lookup.NewSymlinkChainLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(cfg.hostRoot),
+	)
 
 	var candidates []string
 	for _, file := range csvFiles {

cmd/nvidia-ctk/main.go

@@ -36,6 +36,8 @@ import (
 type options struct {
 	// Debug indicates whether the CLI is started in "debug" mode
 	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
 }
 
 func main() {
@@ -61,6 +63,12 @@ func main() {
 			Destination: &opts.Debug,
 			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CTK_QUIET"},
+		},
 	}
 
 	// Set log-level for all subcommands
@@ -69,6 +77,9 @@ func main() {
 		if opts.Debug {
 			logLevel = logrus.DebugLevel
 		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
+		}
 		logger.SetLevel(logLevel)
 		return nil
 	}

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -190,12 +190,14 @@ func (m command) configureWrapper(c *cli.Context, config *config) error {
 		return fmt.Errorf("unable to flush config: %v", err)
 	}
 
-	if n == 0 {
-		m.logger.Infof("Removed empty config from %v", outputPath)
-	} else {
-		m.logger.Infof("Wrote updated config to %v", outputPath)
+	if outputPath != "" {
+		if n == 0 {
+			m.logger.Infof("Removed empty config from %v", outputPath)
+		} else {
+			m.logger.Infof("Wrote updated config to %v", outputPath)
+		}
+		m.logger.Infof("It is recommended that %v daemon be restarted.", config.runtime)
 	}
-	m.logger.Infof("It is recommended that %v daemon be restarted.", config.runtime)
 
 	return nil
 }

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/BurntSushi/toml v1.2.1
 	github.com/NVIDIA/go-nvml v0.12.0-1
-	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
+	github.com/container-orchestrated-devices/container-device-interface v0.6.0
 	github.com/fsnotify/fsnotify v1.5.4
 	github.com/opencontainers/runtime-spec v1.1.0-rc.2
 	github.com/pelletier/go-toml v1.9.4

go.sum

@@ -5,8 +5,8 @@ github.com/NVIDIA/go-nvml v0.12.0-1 h1:6mdjtlFo+17dWL7VFPfuRMtf0061TF4DKls9pkSw6
 github.com/NVIDIA/go-nvml v0.12.0-1/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a h1:sP3PcgyIkRlHqfF3Jfpe/7G8kf/qpzG4C8r94y9hLbE=
-github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a/go.mod h1:xMRa4fJgXzSDFUCURSimOUgoSc+odohvO3uXT9xjqH0=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0 h1:aWwcz/Ep0Fd7ZuBjQGjU/jdPloM7ydhMW13h85jZNvk=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0/go.mod h1:OQlgtJtDrOxSQ1BWODC8OZK1tzi9W69wek+Jy17ndzo=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

internal/config/cli.go

@@ -18,5 +18,8 @@ package config
 
 // ContainerCLIConfig stores the options for the nvidia-container-cli
 type ContainerCLIConfig struct {
-	Root string `toml:"root"`
+	Root        string   `toml:"root"`
+	LoadKmods   bool     `toml:"load-kmods"`
+	Ldconfig    string   `toml:"ldconfig"`
+	Environment []string `toml:"environment"`
 }

internal/config/config.go

@@ -57,6 +57,7 @@ var (
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
+	DisableRequire           bool `toml:"disable-require"`
 	AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
 
 	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
@@ -74,13 +75,22 @@ func GetConfig() (*Config, error) {
 
 	configFilePath := path.Join(configDir, configFilePath)
 
+	return Load(configFilePath)
+}
+
+// Load loads the config from the specified file path.
+func Load(configFilePath string) (*Config, error) {
+	if configFilePath == "" {
+		return getDefault()
+	}
+
 	tomlFile, err := os.Open(configFilePath)
 	if err != nil {
-		return getDefaultConfig()
+		return getDefault()
 	}
 	defer tomlFile.Close()
 
-	cfg, err := loadConfigFrom(tomlFile)
+	cfg, err := LoadFrom(tomlFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read config values: %v", err)
 	}
@@ -88,21 +98,28 @@ func GetConfig() (*Config, error) {
 	return cfg, nil
 }
 
-// loadRuntimeConfigFrom reads the config from the specified Reader
-func loadConfigFrom(reader io.Reader) (*Config, error) {
-	toml, err := toml.LoadReader(reader)
+// LoadFrom reads the config from the specified Reader
+func LoadFrom(reader io.Reader) (*Config, error) {
+	var tree *toml.Tree
+	if reader != nil {
+		toml, err := toml.LoadReader(reader)
+		if err != nil {
+			return nil, err
+		}
+		tree = toml
+	}
+
+	return getFromTree(tree)
+}
+
+// getFromTree reads the nvidia container runtime config from the specified toml Tree.
+func getFromTree(toml *toml.Tree) (*Config, error) {
+	cfg, err := getDefault()
 	if err != nil {
 		return nil, err
 	}
-
-	return getConfigFrom(toml)
-}
-
-// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getConfigFrom(toml *toml.Tree) (*Config, error) {
-	cfg, err := getDefaultConfig()
-	if err != nil {
-		return nil, err
+	if toml == nil {
+		return cfg, nil
 	}
 
 	if err := toml.Unmarshal(cfg); err != nil {
@@ -112,92 +129,40 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 	return cfg, nil
 }
 
-// getDefaultConfig defines the default values for the config
-func getDefaultConfig() (*Config, error) {
-	tomlConfig, err := GetDefaultConfigToml()
-	if err != nil {
-		return nil, err
-	}
-
-	// tomlConfig above includes information about the default values and comments.
-	// we need to marshal it back to a string and then unmarshal it to strip the comments.
-	contents, err := tomlConfig.ToTomlString()
-	if err != nil {
-		return nil, err
-	}
-
-	reloaded, err := toml.Load(contents)
-	if err != nil {
-		return nil, err
-	}
-
-	d := Config{}
-	if err := reloaded.Unmarshal(&d); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal config: %v", err)
-	}
-
-	// The default value for the accept-nvidia-visible-devices-envvar-when-unprivileged is non-standard.
-	// As such we explicitly handle it being set here.
-	if reloaded.Get("accept-nvidia-visible-devices-envvar-when-unprivileged") == nil {
-		d.AcceptEnvvarUnprivileged = true
-	}
-	// The default value for the nvidia-container-runtime.debug is non-standard.
-	// As such we explicitly handle it being set here.
-	if reloaded.Get("nvidia-container-runtime.debug") == nil {
-		d.NVIDIAContainerRuntimeConfig.DebugFilePath = "/dev/null"
+// getDefault defines the default values for the config
+func getDefault() (*Config, error) {
+	d := Config{
+		AcceptEnvvarUnprivileged: true,
+		NVIDIAContainerCLIConfig: ContainerCLIConfig{
+			LoadKmods: true,
+			Ldconfig:  getLdConfigPath(),
+		},
+		NVIDIACTKConfig: CTKConfig{
+			Path: nvidiaCTKExecutable,
+		},
+		NVIDIAContainerRuntimeConfig: RuntimeConfig{
+			DebugFilePath: "/dev/null",
+			LogLevel:      "info",
+			Runtimes:      []string{"docker-runc", "runc"},
+			Mode:          "auto",
+			Modes: modesConfig{
+				CSV: csvModeConfig{
+					MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+				},
+				CDI: cdiModeConfig{
+					DefaultKind:        "nvidia.com/gpu",
+					AnnotationPrefixes: []string{cdi.AnnotationPrefix},
+					SpecDirs:           cdi.DefaultSpecDirs,
+				},
+			},
+		},
+		NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+			Path: NVIDIAContainerRuntimeHookExecutable,
+		},
 	}
 	return &d, nil
 }
 
-// GetDefaultConfigToml returns the default config as a toml Tree.
-func GetDefaultConfigToml() (*toml.Tree, error) {
-	tree, err := toml.TreeFromMap(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	tree.Set("disable-require", false)
-	tree.SetWithComment("swarm-resource", "", true, "DOCKER_RESOURCE_GPU")
-	tree.SetWithComment("accept-nvidia-visible-devices-envvar-when-unprivileged", "", true, true)
-	tree.SetWithComment("accept-nvidia-visible-devices-as-volume-mounts", "", true, false)
-
-	// nvidia-container-cli
-	tree.SetWithComment("nvidia-container-cli.root", "", true, "/run/nvidia/driver")
-	tree.SetWithComment("nvidia-container-cli.path", "", true, "/usr/bin/nvidia-container-cli")
-	tree.Set("nvidia-container-cli.environment", []string{})
-	tree.SetWithComment("nvidia-container-cli.debug", "", true, "/var/log/nvidia-container-toolkit.log")
-	tree.SetWithComment("nvidia-container-cli.ldcache", "", true, "/etc/ld.so.cache")
-	tree.Set("nvidia-container-cli.load-kmods", true)
-	tree.SetWithComment("nvidia-container-cli.no-cgroups", "", true, false)
-
-	tree.SetWithComment("nvidia-container-cli.user", "", getCommentedUserGroup(), getUserGroup())
-	tree.Set("nvidia-container-cli.ldconfig", getLdConfigPath())
-
-	// nvidia-container-runtime
-	tree.SetWithComment("nvidia-container-runtime.debug", "", true, "/var/log/nvidia-container-runtime.log")
-	tree.Set("nvidia-container-runtime.log-level", "info")
-
-	commentLines := []string{
-		"Specify the runtimes to consider. This list is processed in order and the PATH",
-		"searched for matching executables unless the entry is an absolute path.",
-	}
-	tree.SetWithComment("nvidia-container-runtime.runtimes", strings.Join(commentLines, "\n "), false, []string{"docker-runc", "runc"})
-
-	tree.Set("nvidia-container-runtime.mode", "auto")
-
-	tree.Set("nvidia-container-runtime.modes.csv.mount-spec-path", "/etc/nvidia-container-runtime/host-files-for-container.d")
-	tree.Set("nvidia-container-runtime.modes.cdi.default-kind", "nvidia.com/gpu")
-	tree.Set("nvidia-container-runtime.modes.cdi.annotation-prefixes", []string{cdi.AnnotationPrefix})
-
-	// nvidia-ctk
-	tree.Set("nvidia-ctk.path", nvidiaCTKExecutable)
-
-	// nvidia-container-runtime-hook
-	tree.Set("nvidia-container-runtime-hook.path", nvidiaContainerRuntimeHookExecutable)
-
-	return tree, nil
-}
-
 func getLdConfigPath() string {
 	if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
 		return "@/sbin/ldconfig.real"
@@ -205,11 +170,6 @@ func getLdConfigPath() string {
 	return "@/sbin/ldconfig"
 }
 
-// getUserGroup returns the user and group to use for the nvidia-container-cli and whether the config option should be commented.
-func getUserGroup() string {
-	return "root:video"
-}
-
 // getCommentedUserGroup returns whether the nvidia-container-cli user and group config option should be commented.
 func getCommentedUserGroup() bool {
 	uncommentIf := map[string]bool{
@@ -295,3 +255,67 @@ func resolveWithDefault(logger logger.Interface, label string, path string, defa
 
 	return resolvedPath
 }
+
+func (c Config) asCommentedToml() (*toml.Tree, error) {
+	contents, err := toml.Marshal(c)
+	if err != nil {
+		return nil, err
+	}
+	asToml, err := toml.LoadBytes(contents)
+	if err != nil {
+		return nil, err
+	}
+
+	commentedDefaults := map[string]interface{}{
+		"swarm-resource": "DOCKER_RESOURCE_GPU",
+		"accept-nvidia-visible-devices-envvar-when-unprivileged": true,
+		"accept-nvidia-visible-devices-as-volume-mounts":         false,
+		"nvidia-container-cli.root":                              "/run/nvidia/driver",
+		"nvidia-container-cli.path":                              "/usr/bin/nvidia-container-cli",
+		"nvidia-container-cli.debug":                             "/var/log/nvidia-container-toolkit.log",
+		"nvidia-container-cli.ldcache":                           "/etc/ld.so.cache",
+		"nvidia-container-cli.no-cgroups":                        false,
+		"nvidia-container-cli.user":                              "root:video",
+		"nvidia-container-runtime.debug":                         "/var/log/nvidia-container-runtime.log",
+	}
+	for k, v := range commentedDefaults {
+		set := asToml.Get(k)
+		if !shouldComment(k, v, set) {
+			continue
+		}
+		asToml.SetWithComment(k, "", true, v)
+	}
+
+	return asToml, nil
+}
+
+func shouldComment(key string, defaultValue interface{}, setTo interface{}) bool {
+	if key == "nvidia-container-cli.root" && setTo == "" {
+		return true
+	}
+	if key == "nvidia-container-cli.user" && !getCommentedUserGroup() {
+		return false
+	}
+	if key == "nvidia-container-runtime.debug" && setTo == "/dev/null" {
+		return true
+	}
+	if setTo == nil || defaultValue == setTo {
+		return true
+	}
+
+	return false
+}
+
+// Save writes the config to the specified writer.
+func (c Config) Save(w io.Writer) (int64, error) {
+	asToml, err := c.asCommentedToml()
+	if err != nil {
+		return 0, err
+	}
+
+	enc := toml.NewEncoder(w).Indentation("")
+	if err := enc.Encode(asToml); err != nil {
+		return 0, fmt.Errorf("invalid config: %v", err)
+	}
+	return 0, nil
+}

internal/config/config_test.go

@@ -17,6 +17,7 @@
 package config
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -49,17 +50,21 @@ func TestGetConfigWithCustomConfig(t *testing.T) {
 
 func TestGetConfig(t *testing.T) {
 	testCases := []struct {
-		description    string
-		contents       []string
-		expectedError  error
-		expectedConfig *Config
+		description     string
+		contents        []string
+		expectedError   error
+		inspectLdconfig bool
+		expectedConfig  *Config
 	}{
 		{
-			description: "empty config is default",
+			description:     "empty config is default",
+			inspectLdconfig: true,
 			expectedConfig: &Config{
 				AcceptEnvvarUnprivileged: true,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "",
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/dev/null",
@@ -73,6 +78,7 @@ func TestGetConfig(t *testing.T) {
 						CDI: cdiModeConfig{
 							DefaultKind:        "nvidia.com/gpu",
 							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
 						},
 					},
 				},
@@ -89,14 +95,16 @@ func TestGetConfig(t *testing.T) {
 			contents: []string{
 				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"nvidia-container-cli.root = \"/bar/baz\"",
+				"nvidia-container-cli.load-kmods = false",
+				"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
-				"nvidia-container-runtime.experimental = true",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
 				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
 				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"nvidia-container-runtime.modes.cdi.spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"nvidia-container-runtime-hook.path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
@@ -104,7 +112,9 @@ func TestGetConfig(t *testing.T) {
 			expectedConfig: &Config{
 				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -121,6 +131,10 @@ func TestGetConfig(t *testing.T) {
 								"cdi.k8s.io/",
 								"example.vendor.com/",
 							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
 						},
 					},
 				},
@@ -138,9 +152,10 @@ func TestGetConfig(t *testing.T) {
 				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
+				"load-kmods = false",
+				"ldconfig = \"/foo/bar/ldconfig\"",
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
-				"experimental = true",
 				"discover-mode = \"not-legacy\"",
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
@@ -148,6 +163,7 @@ func TestGetConfig(t *testing.T) {
 				"[nvidia-container-runtime.modes.cdi]",
 				"default-kind = \"example.vendor.com/device\"",
 				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"[nvidia-container-runtime-hook]",
@@ -158,7 +174,9 @@ func TestGetConfig(t *testing.T) {
 			expectedConfig: &Config{
 				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -175,6 +193,10 @@ func TestGetConfig(t *testing.T) {
 								"cdi.k8s.io/",
 								"example.vendor.com/",
 							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
 						},
 					},
 				},
@@ -192,14 +214,68 @@ func TestGetConfig(t *testing.T) {
 		t.Run(tc.description, func(t *testing.T) {
 			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
 
-			cfg, err := loadConfigFrom(reader)
+			cfg, err := LoadFrom(reader)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
 			}
 
+			// We first handle the ldconfig path since this is currently system-dependent.
+			if tc.inspectLdconfig {
+				ldconfig := cfg.NVIDIAContainerCLIConfig.Ldconfig
+				require.True(t, strings.HasPrefix(ldconfig, "@/sbin/ldconfig"))
+				remaining := strings.TrimPrefix(ldconfig, "@/sbin/ldconfig")
+				require.True(t, remaining == ".real" || remaining == "")
+
+				cfg.NVIDIAContainerCLIConfig.Ldconfig = "WAS_CHECKED"
+			}
+
 			require.EqualValues(t, tc.expectedConfig, cfg)
 		})
 	}
 }
+
+func TestConfigDefault(t *testing.T) {
+	config, err := getDefault()
+	require.NoError(t, err)
+
+	buffer := new(bytes.Buffer)
+	_, err = config.Save(buffer)
+	require.NoError(t, err)
+
+	var lines []string
+	for _, l := range strings.Split(buffer.String(), "\n") {
+		l = strings.TrimSpace(l)
+		if strings.HasPrefix(l, "# ") {
+			l = "#" + strings.TrimPrefix(l, "# ")
+		}
+		lines = append(lines, l)
+	}
+
+	// We take the lines from the config that was included in previous packages.
+	expectedLines := []string{
+		"disable-require = false",
+		"#swarm-resource = \"DOCKER_RESOURCE_GPU\"",
+		"#accept-nvidia-visible-devices-envvar-when-unprivileged = true",
+		"#accept-nvidia-visible-devices-as-volume-mounts = false",
+
+		"#root = \"/run/nvidia/driver\"",
+		"#path = \"/usr/bin/nvidia-container-cli\"",
+		"environment = []",
+		"#debug = \"/var/log/nvidia-container-toolkit.log\"",
+		"#ldcache = \"/etc/ld.so.cache\"",
+		"load-kmods = true",
+		"#no-cgroups = false",
+		"#user = \"root:video\"",
+
+		"[nvidia-container-runtime]",
+		"#debug = \"/var/log/nvidia-container-runtime.log\"",
+		"log-level = \"info\"",
+		"mode = \"auto\"",
+
+		"mount-spec-path = \"/etc/nvidia-container-runtime/host-files-for-container.d\"",
+	}
+
+	require.Subset(t, lines, expectedLines)
+}

internal/config/hook.go

@@ -27,7 +27,7 @@ type RuntimeHookConfig struct {
 
 // GetDefaultRuntimeHookConfig defines the default values for the config
 func GetDefaultRuntimeHookConfig() (*RuntimeHookConfig, error) {
-	cfg, err := getDefaultConfig()
+	cfg, err := getDefault()
 	if err != nil {
 		return nil, err
 	}

internal/config/image/builder.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"strings"
+)
+
+type builder struct {
+	env            []string
+	disableRequire bool
+}
+
+// New creates a new CUDA image from the input options.
+func New(opt ...Option) (CUDA, error) {
+	b := &builder{}
+	for _, o := range opt {
+		o(b)
+	}
+
+	return b.build()
+}
+
+// build creates a CUDA image from the builder.
+func (b builder) build() (CUDA, error) {
+	c := make(CUDA)
+
+	for _, e := range b.env {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid environment variable: %v", e)
+		}
+		c[parts[0]] = parts[1]
+	}
+
+	if b.disableRequire {
+		c[envNVDisableRequire] = "true"
+	}
+
+	return c, nil
+}
+
+// Option is a functional option for creating a CUDA image.
+type Option func(*builder)
+
+// WithDisableRequire sets the disable require option.
+func WithDisableRequire(disableRequire bool) Option {
+	return func(b *builder) {
+		b.disableRequire = disableRequire
+	}
+}
+
+// WithEnv sets the environment variables to use when creating the CUDA image.
+func WithEnv(env []string) Option {
+	return func(b *builder) {
+		b.env = env
+	}
+}

internal/config/image/cuda_image.go

@@ -42,27 +42,18 @@ type CUDA map[string]string
 // NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
 // The process environment is read (if present) to construc the CUDA Image.
 func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
-	if spec == nil || spec.Process == nil {
-		return NewCUDAImageFromEnv(nil)
+	var env []string
+	if spec != nil && spec.Process != nil {
+		env = spec.Process.Env
 	}
 
-	return NewCUDAImageFromEnv(spec.Process.Env)
+	return New(WithEnv(env))
 }
 
 // NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
 // is a list of strings of the form ENVAR=VALUE.
 func NewCUDAImageFromEnv(env []string) (CUDA, error) {
-	c := make(CUDA)
-
-	for _, e := range env {
-		parts := strings.SplitN(e, "=", 2)
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("invalid environment variable: %v", e)
-		}
-		c[parts[0]] = parts[1]
-	}
-
-	return c, nil
+	return New(WithEnv(env))
 }
 
 // IsLegacy returns whether the associated CUDA image is a "legacy" image. An
@@ -77,11 +68,9 @@ func (i CUDA) IsLegacy() bool {
 // GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
 // variables.
 func (i CUDA) GetRequirements() ([]string, error) {
-	// TODO: We need not process this if disable require is set, but this will be done
-	// in a single follow-up to ensure that the behavioural change is accurately captured.
-	// if i.HasDisableRequire() {
-	// 	return nil, nil
-	// }
+	if i.HasDisableRequire() {
+		return nil, nil
+	}
 
 	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
 	var requirements []string
@@ -159,9 +148,10 @@ func (i CUDA) GetDriverCapabilities() DriverCapabilities {
 }
 
 func (i CUDA) legacyVersion() (string, error) {
-	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
+	cudaVersion := i[envCUDAVersion]
+	majorMinor, err := parseMajorMinorVersion(cudaVersion)
 	if err != nil {
-		return "", fmt.Errorf("invalid CUDA version: %v", err)
+		return "", fmt.Errorf("invalid CUDA version %v: %v", cudaVersion, err)
 	}
 
 	return majorMinor, nil

internal/config/image/cuda_image_test.go

@@ -106,6 +106,16 @@ func TestGetRequirements(t *testing.T) {
 			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
 			requirements: []string{"cuda>=11.6", "brand=tesla"},
 		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores requirements",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores legacy image requirements",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
 	}
 
 	for _, tc := range testCases {

internal/config/runtime.go

@@ -48,7 +48,7 @@ type csvModeConfig struct {
 
 // GetDefaultRuntimeConfig defines the default values for the config
 func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
-	cfg, err := getDefaultConfig()
+	cfg, err := getDefault()
 	if err != nil {
 		return nil, err
 	}

internal/discover/csv.go

@@ -33,10 +33,13 @@ func NewFromCSVFiles(logger logger.Interface, files []string, driverRoot string)
 		return None{}, nil
 	}
 
-	symlinkLocator := lookup.NewSymlinkLocator(logger, driverRoot)
+	symlinkLocator := lookup.NewSymlinkLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(driverRoot),
+	)
 	locators := map[csv.MountSpecType]lookup.Locator{
 		csv.MountSpecDev: lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
-		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, driverRoot),
+		csv.MountSpecDir: lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
 		// Libraries and symlinks are handled in the same way
 		csv.MountSpecLib: symlinkLocator,
 		csv.MountSpecSym: symlinkLocator,

internal/discover/gds.go

@@ -38,7 +38,7 @@ func NewGDSDiscoverer(logger logger.Interface, root string) (Discover, error) {
 
 	udev := NewMounts(
 		logger,
-		lookup.NewDirectoryLocator(logger, root),
+		lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(root)),
 		root,
 		[]string{"/run/udev"},
 	)

internal/discover/symlinks.go

@@ -113,7 +113,10 @@ func (d symlinkHook) getSpecificLinks() ([]string, error) {
 }
 
 func (d symlinkHook) getCSVFileSymlinks() []string {
-	chainLocator := lookup.NewSymlinkChainLocator(d.logger, d.driverRoot)
+	chainLocator := lookup.NewSymlinkChainLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.driverRoot),
+	)
 
 	var candidates []string
 	for _, file := range d.csvFiles {

internal/info/auto.go

@@ -19,7 +19,7 @@ package info
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/info"
 )
 

internal/lookup/dir.go

@@ -19,17 +19,15 @@ package lookup
 import (
 	"fmt"
 	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
-// NewDirectoryLocator creates a Locator that can be used to find directories at the specified root. A logger
-// is also specified.
-func NewDirectoryLocator(logger logger.Interface, root string) Locator {
+// NewDirectoryLocator creates a Locator that can be used to find directories at the specified root.
+func NewDirectoryLocator(opts ...Option) Locator {
 	return NewFileLocator(
-		WithLogger(logger),
-		WithRoot(root),
-		WithFilter(assertDirectory),
+		append(
+			opts,
+			WithFilter(assertDirectory),
+		)...,
 	)
 }
 

internal/lookup/library.go

@@ -41,7 +41,7 @@ func NewLibraryLocator(logger logger.Interface, root string) (Locator, error) {
 
 	l := library{
 		logger:  logger,
-		symlink: NewSymlinkLocator(logger, root),
+		symlink: NewSymlinkLocator(WithLogger(logger), WithRoot(root)),
 		cache:   cache,
 	}
 

internal/lookup/symlinks.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 )
 
@@ -33,9 +32,8 @@ type symlink struct {
 }
 
 // NewSymlinkChainLocator creats a locator that can be used for locating files through symlinks.
-// A logger can also be specified.
-func NewSymlinkChainLocator(logger logger.Interface, root string) Locator {
-	f := newFileLocator(WithLogger(logger), WithRoot(root))
+func NewSymlinkChainLocator(opts ...Option) Locator {
+	f := newFileLocator(opts...)
 	l := symlinkChain{
 		file: *f,
 	}
@@ -44,9 +42,8 @@ func NewSymlinkChainLocator(logger logger.Interface, root string) Locator {
 }
 
 // NewSymlinkLocator creats a locator that can be used for locating files through symlinks.
-// A logger can also be specified.
-func NewSymlinkLocator(logger logger.Interface, root string) Locator {
-	f := newFileLocator(WithLogger(logger), WithRoot(root))
+func NewSymlinkLocator(opts ...Option) Locator {
+	f := newFileLocator(opts...)
 	l := symlink{
 		file: *f,
 	}

internal/modifier/cdi.go

@@ -24,7 +24,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -48,14 +48,9 @@ func NewCDIModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spe
 	}
 	logger.Debugf("Creating CDI modifier for devices: %v", devices)
 
-	specDirs := cdi.DefaultSpecDirs
-	if len(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs) > 0 {
-		specDirs = cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs
-	}
-
 	m := cdiModifier{
 		logger:   logger,
-		specDirs: specDirs,
+		specDirs: cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs,
 		devices:  devices,
 	}
 

internal/modifier/csv.go

@@ -45,17 +45,7 @@ const (
 
 // NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
 // The modifications are defined by CSV MountSpecs.
-func NewCSVModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	rawSpec, err := ociSpec.Load()
-	if err != nil {
-		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
+func NewCSVModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {
 	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil

internal/modifier/csv_test.go

@@ -17,11 +17,10 @@
 package modifier
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
@@ -31,54 +30,32 @@ func TestNewCSVModifier(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
 	testCases := []struct {
-		description    string
-		cfg            *config.Config
-		spec           oci.Spec
-		visibleDevices string
-		expectedError  error
-		expectedNil    bool
+		description   string
+		cfg           *config.Config
+		image         image.CUDA
+		expectedError error
+		expectedNil   bool
 	}{
 		{
-			description: "spec load error returns error",
-			spec: &oci.SpecMock{
-				LoadFunc: func() (*specs.Spec, error) {
-					return nil, fmt.Errorf("load failed")
-				},
-			},
-			expectedError: fmt.Errorf("load failed"),
+			description: "visible devices not set returns nil",
+			image:       image.CUDA{},
+			expectedNil: true,
 		},
 		{
-			description:    "visible devices not set returns nil",
-			visibleDevices: "NOT_SET",
-			expectedNil:    true,
+			description: "visible devices empty returns nil",
+			image:       image.CUDA{"NVIDIA_VISIBLE_DEVICES": ""},
+			expectedNil: true,
 		},
 		{
-			description:    "visible devices empty returns nil",
-			visibleDevices: "",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices 'void' returns nil",
-			visibleDevices: "void",
-			expectedNil:    true,
+			description: "visible devices 'void' returns nil",
+			image:       image.CUDA{"NVIDIA_VISIBLE_DEVICES": "void"},
+			expectedNil: true,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			spec := tc.spec
-			if spec == nil {
-				spec = &oci.SpecMock{
-					LookupEnvFunc: func(s string) (string, bool) {
-						if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
-							return tc.visibleDevices, true
-						}
-						return "", false
-					},
-				}
-			}
-
-			m, err := NewCSVModifier(logger, tc.cfg, spec)
+			m, err := NewCSVModifier(logger, tc.cfg, tc.image)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {

internal/runtime/runtime_factory.go

@@ -44,7 +44,7 @@ func newNVIDIAContainerRuntime(logger logger.Interface, cfg *config.Config, argv
 		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
 	}
 
-	specModifier, err := newSpecModifier(logger, cfg, ociSpec, argv)
+	specModifier, err := newSpecModifier(logger, cfg, ociSpec)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
 	}
@@ -61,7 +61,7 @@ func newNVIDIAContainerRuntime(logger logger.Interface, cfg *config.Config, argv
 }
 
 // newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
-func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
+func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
 	rawSpec, err := ociSpec.Load()
 	if err != nil {
 		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
@@ -73,7 +73,7 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp
 	}
 
 	mode := info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode, image)
-	modeModifier, err := newModeModifier(logger, mode, cfg, ociSpec, argv)
+	modeModifier, err := newModeModifier(logger, mode, cfg, ociSpec, image)
 	if err != nil {
 		return nil, err
 	}
@@ -106,12 +106,12 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp
 	return modifiers, nil
 }
 
-func newModeModifier(logger logger.Interface, mode string, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
+func newModeModifier(logger logger.Interface, mode string, cfg *config.Config, ociSpec oci.Spec, image image.CUDA) (oci.SpecModifier, error) {
 	switch mode {
 	case "legacy":
 		return modifier.NewStableRuntimeModifier(logger, cfg.NVIDIAContainerRuntimeHookConfig.Path), nil
 	case "csv":
-		return modifier.NewCSVModifier(logger, cfg, ociSpec)
+		return modifier.NewCSVModifier(logger, cfg, image)
 	case "cdi":
 		return modifier.NewCDIModifier(logger, cfg, ociSpec)
 	}

packaging/debian/nvidia-container-toolkit-base.postinst

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -e
+
+case "$1" in
+    configure)
+        /usr/bin/nvidia-ctk --quiet config default --in-place --config=/etc/nvidia-container-runtime/config.toml
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+#DEBHELPER#
+
+exit 0

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -53,17 +53,22 @@ mkdir -p %{buildroot}/usr/share/containers/oci/hooks.d
 install -m 644 -t %{buildroot}/usr/share/containers/oci/hooks.d oci-nvidia-hook.json
 
 %post
-mkdir -p %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
-cp -af %{_bindir}/nvidia-container-runtime-hook %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
+if [ $1 -gt 1 ]; then  # only on package upgrade
+  mkdir -p %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
+  cp -af %{_bindir}/nvidia-container-runtime-hook %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
+fi
 
 %posttrans
 if [ ! -e %{_bindir}/nvidia-container-runtime-hook ]; then
-  # reparing lost file nvidia-container-runtime-hook
+  # repairing lost file nvidia-container-runtime-hook
   cp -avf %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit/nvidia-container-runtime-hook %{_bindir}
 fi
 rm -rf %{_localstatedir}/lib/rpm-state/nvidia-container-toolkit
 ln -sf %{_bindir}/nvidia-container-runtime-hook %{_bindir}/nvidia-container-toolkit
 
+# Generate the default config; If this file already exists no changes are made.
+%{_bindir}/nvidia-ctk --quiet config default --in-place --config=%{_sysconfdir}/nvidia-container-runtime/config.toml
+
 %postun
 if [ "$1" = 0 ]; then  # package is uninstalled, not upgraded
   if [ -L %{_bindir}/nvidia-container-toolkit ]; then rm -f %{_bindir}/nvidia-container-toolkit; fi

pkg/config/engine/config.go

@@ -0,0 +1,61 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package engine
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Config represents a runtime config
+type Config string
+
+// Write writes the specified contents to a config file.
+func (c Config) Write(output []byte) (int, error) {
+	path := string(c)
+	if path == "" {
+		n, err := os.Stdout.Write(output)
+		if err == nil {
+			os.Stdout.WriteString("\n")
+		}
+		return n, err
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	if dir := filepath.Dir(path); dir != "" {
+		err := os.MkdirAll(dir, 0755)
+		if err != nil {
+			return 0, fmt.Errorf("unable to create directory %v: %v", dir, err)
+		}
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	return f.Write(output)
+}

pkg/config/engine/containerd/config_v2.go

@@ -18,8 +18,8 @@ package containerd
 
 import (
 	"fmt"
-	"os"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/pelletier/go-toml"
 )
 
@@ -133,34 +133,11 @@ func (c *Config) RemoveRuntime(name string) error {
 // Save writes the config to the specified path
 func (c Config) Save(path string) (int64, error) {
 	config := c.Tree
-	output, err := config.ToTomlString()
+	output, err := config.Marshal()
 	if err != nil {
 		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
 	}
 
-	if path == "" {
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return int64(len(output)), nil
-	}
-
-	if len(output) == 0 {
-		err := os.Remove(path)
-		if err != nil {
-			return 0, fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		return 0, nil
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
-	}
-	defer f.Close()
-
-	n, err := f.WriteString(output)
-	if err != nil {
-		return 0, fmt.Errorf("unable to write output: %v", err)
-	}
-
+	n, err := engine.Config(path).Write(output)
 	return int64(n), err
 }

pkg/config/engine/containerd/option.go

@@ -108,26 +108,23 @@ func (b *builder) build() (engine.Interface, error) {
 
 // loadConfig loads the containerd config from disk
 func (b *builder) loadConfig(config string) (*Config, error) {
-	b.logger.Infof("Loading config: %v", config)
-
 	info, err := os.Stat(config)
 	if os.IsExist(err) && info.IsDir() {
 		return nil, fmt.Errorf("config file is a directory")
 	}
 
-	configFile := config
 	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		b.logger.Infof("Config file does not exist, creating new one")
+		b.logger.Infof("Config file does not exist; using empty config")
+		config = "/dev/null"
+	} else {
+		b.logger.Infof("Loading config from %v", config)
 	}
 
-	tomlConfig, err := toml.LoadFile(configFile)
+	tomlConfig, err := toml.LoadFile(config)
 	if err != nil {
 		return nil, err
 	}
 
-	b.logger.Infof("Successfully loaded config")
-
 	cfg := Config{
 		Tree: tomlConfig,
 	}

pkg/config/engine/crio/crio.go

@@ -18,7 +18,6 @@ package crio
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/pelletier/go-toml"
@@ -103,34 +102,11 @@ func (c *Config) RemoveRuntime(name string) error {
 // Save writes the config to the specified path
 func (c Config) Save(path string) (int64, error) {
 	config := (toml.Tree)(c)
-	output, err := config.ToTomlString()
+	output, err := config.Marshal()
 	if err != nil {
 		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
 	}
 
-	if path == "" {
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return int64(len(output)), nil
-	}
-
-	if len(output) == 0 {
-		err := os.Remove(path)
-		if err != nil {
-			return 0, fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		return 0, nil
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
-	}
-	defer f.Close()
-
-	n, err := f.WriteString(output)
-	if err != nil {
-		return 0, fmt.Errorf("unable to write output: %v", err)
-	}
-
+	n, err := engine.Config(path).Write(output)
 	return int64(n), err
 }

pkg/config/engine/crio/option.go

@@ -67,13 +67,14 @@ func (b *builder) loadConfig(config string) (*Config, error) {
 		return nil, fmt.Errorf("config file is a directory")
 	}
 
-	configFile := config
 	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		b.logger.Infof("Config file does not exist, creating new one")
+		b.logger.Infof("Config file does not exist; using empty config")
+		config = "/dev/null"
+	} else {
+		b.logger.Infof("Loading config from %v", config)
 	}
 
-	cfg, err := toml.LoadFile(configFile)
+	cfg, err := toml.LoadFile(config)
 	if err != nil {
 		return nil, err
 	}

pkg/config/engine/docker/docker.go

@@ -19,7 +19,6 @@ package docker
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
@@ -122,29 +121,6 @@ func (c Config) Save(path string) (int64, error) {
 		return 0, fmt.Errorf("unable to convert to JSON: %v", err)
 	}
 
-	if path == "" {
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return int64(len(output)), nil
-	}
-
-	if len(output) == 0 {
-		err := os.Remove(path)
-		if err != nil {
-			return 0, fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		return 0, nil
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
-	}
-	defer f.Close()
-
-	n, err := f.WriteString(string(output))
-	if err != nil {
-		return 0, fmt.Errorf("unable to write output: %v", err)
-	}
-
-	return int64(n), nil
+	n, err := engine.Config(path).Write(output)
+	return int64(n), err
 }

pkg/config/engine/docker/option.go

@@ -58,10 +58,8 @@ func (b *builder) build() (*Config, error) {
 }
 
 // loadConfig loads the docker config from disk
-func (b *builder) loadConfig(configFilePath string) (*Config, error) {
-	b.logger.Infof("Loading docker config from %v", configFilePath)
-
-	info, err := os.Stat(configFilePath)
+func (b *builder) loadConfig(config string) (*Config, error) {
+	info, err := os.Stat(config)
 	if os.IsExist(err) && info.IsDir() {
 		return nil, fmt.Errorf("config file is a directory")
 	}
@@ -69,11 +67,12 @@ func (b *builder) loadConfig(configFilePath string) (*Config, error) {
 	cfg := make(Config)
 
 	if os.IsNotExist(err) {
-		b.logger.Infof("Config file does not exist, creating new one")
+		b.logger.Infof("Config file does not exist; using empty config")
 		return &cfg, nil
 	}
 
-	readBytes, err := ioutil.ReadFile(configFilePath)
+	b.logger.Infof("Loading config from %v", config)
+	readBytes, err := ioutil.ReadFile(config)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read config: %v", err)
 	}
@@ -82,7 +81,5 @@ func (b *builder) loadConfig(configFilePath string) (*Config, error) {
 	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
 		return nil, err
 	}
-
-	b.logger.Infof("Successfully loaded config")
 	return &cfg, nil
 }

pkg/nvcdi/driver-nvml.go

@@ -18,6 +18,7 @@ package nvcdi
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+	"golang.org/x/sys/unix"
 )
 
 // NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
@@ -55,7 +57,10 @@ func newDriverVersionDiscoverer(logger logger.Interface, driverRoot string, nvid
 		return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
 	}
 
-	firmwares := NewDriverFirmwareDiscoverer(logger, driverRoot, version)
+	firmwares, err := NewDriverFirmwareDiscoverer(logger, driverRoot, version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for GSP firmware: %v", err)
+	}
 
 	binaries := NewDriverBinariesDiscoverer(logger, driverRoot)
 
@@ -96,18 +101,65 @@ func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvid
 	return d, nil
 }
 
+func getUTSRelease() (string, error) {
+	utsname := &unix.Utsname{}
+	if err := unix.Uname(utsname); err != nil {
+		return "", err
+	}
+	return unix.ByteSliceToString(utsname.Release[:]), nil
+}
+
+func getFirmwareSearchPaths(logger logger.Interface) ([]string, error) {
+
+	var firmwarePaths []string
+	if p := getCustomFirmwareClassPath(logger); p != "" {
+		logger.Debugf("using custom firmware class path: %s", p)
+		firmwarePaths = append(firmwarePaths, p)
+	}
+
+	utsRelease, err := getUTSRelease()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get UTS_RELEASE: %v", err)
+	}
+
+	standardPaths := []string{
+		filepath.Join("/lib/firmware/updates/", utsRelease),
+		filepath.Join("/lib/firmware/updates/"),
+		filepath.Join("/lib/firmware/", utsRelease),
+		filepath.Join("/lib/firmware/"),
+	}
+
+	return append(firmwarePaths, standardPaths...), nil
+}
+
+// getCustomFirmwareClassPath returns the custom firmware class path if it exists.
+func getCustomFirmwareClassPath(logger logger.Interface) string {
+	customFirmwareClassPath, err := os.ReadFile("/sys/module/firmware_class/parameters/path")
+	if err != nil {
+		logger.Warningf("failed to get custom firmware class path: %v", err)
+		return ""
+	}
+
+	return strings.TrimSpace(string(customFirmwareClassPath))
+}
+
 // NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version.
-func NewDriverFirmwareDiscoverer(logger logger.Interface, driverRoot string, version string) discover.Discover {
-	gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp*.bin")
+func NewDriverFirmwareDiscoverer(logger logger.Interface, driverRoot string, version string) (discover.Discover, error) {
+	gspFirmwareSearchPaths, err := getFirmwareSearchPaths(logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get firmware search paths: %v", err)
+	}
+	gspFirmwarePaths := filepath.Join("nvidia", version, "gsp*.bin")
 	return discover.NewMounts(
 		logger,
 		lookup.NewFileLocator(
 			lookup.WithLogger(logger),
 			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(gspFirmwareSearchPaths...),
 		),
 		driverRoot,
-		[]string{gspFirmwarePath},
-	)
+		[]string{gspFirmwarePaths},
+	), nil
 }
 
 // NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver.

pkg/nvcdi/driver-wsl.go

@@ -67,7 +67,6 @@ func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, nvi
 	if len(searchPaths) > 1 {
 		logger.Warningf("Found multiple driver store paths: %v", searchPaths)
 	}
-	driverStorePath := searchPaths[0]
 	searchPaths = append(searchPaths, "/usr/lib/wsl/lib")
 
 	libraries := discover.NewMounts(
@@ -83,12 +82,11 @@ func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, nvi
 		requiredDriverStoreFiles,
 	)
 
-	// On WSL2 the driver store location is used unchanged.
-	// For this reason we need to create a symlink from /usr/bin/nvidia-smi to the nvidia-smi binary in the driver store.
-	target := filepath.Join(driverStorePath, "nvidia-smi")
-	link := "/usr/bin/nvidia-smi"
-	links := []string{fmt.Sprintf("%s::%s", target, link)}
-	symlinkHook := discover.CreateCreateSymlinkHook(nvidiaCTKPath, links)
+	symlinkHook := nvidiaSMISimlinkHook{
+		logger:        logger,
+		mountsFrom:    libraries,
+		nvidiaCTKPath: nvidiaCTKPath,
+	}
 
 	ldcacheHook, _ := discover.NewLDCacheUpdateHook(logger, libraries, nvidiaCTKPath)
 
@@ -100,3 +98,39 @@ func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, nvi
 
 	return d, nil
 }
+
+type nvidiaSMISimlinkHook struct {
+	discover.None
+	logger        logger.Interface
+	mountsFrom    discover.Discover
+	nvidiaCTKPath string
+}
+
+// Hooks returns a hook that creates a symlink to nvidia-smi in the driver store.
+// On WSL2 the driver store location is used unchanged, for this reason we need
+// to create a symlink from /usr/bin/nvidia-smi to the nvidia-smi binary in the
+// driver store.
+func (m nvidiaSMISimlinkHook) Hooks() ([]discover.Hook, error) {
+	mounts, err := m.mountsFrom.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover mounts: %w", err)
+	}
+
+	var target string
+	for _, mount := range mounts {
+		if filepath.Base(mount.Path) == "nvidia-smi" {
+			target = mount.Path
+			break
+		}
+	}
+
+	if target == "" {
+		m.logger.Warningf("Failed to find nvidia-smi in mounts: %v", mounts)
+		return nil, nil
+	}
+	link := "/usr/bin/nvidia-smi"
+	links := []string{fmt.Sprintf("%s::%s", target, link)}
+	symlinkHook := discover.CreateCreateSymlinkHook(m.nvidiaCTKPath, links)
+
+	return symlinkHook.Hooks()
+}

pkg/nvcdi/driver-wsl_test.go

@@ -0,0 +1,163 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/stretchr/testify/require"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestNvidiaSMISymlinkHook(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	errMounts := errors.New("mounts error")
+
+	testCases := []struct {
+		description   string
+		mounts        discover.Discover
+		expectedError error
+		expectedHooks []discover.Hook
+	}{
+		{
+			description: "mounts error is returned",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					return nil, errMounts
+				},
+			},
+			expectedError: errMounts,
+		},
+		{
+			description: "no mounts returns no hooks",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					return nil, nil
+				},
+			},
+		},
+		{
+			description: "no nvidia-smi returns no hooks",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					mounts := []discover.Mount{
+						{Path: "/not-nvidia-smi"},
+						{Path: "/also-not-nvidia-smi"},
+					}
+					return mounts, nil
+				},
+			},
+		},
+		{
+			description: "nvidia-smi must be in path",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					mounts := []discover.Mount{
+						{Path: "/not-nvidia-smi", HostPath: "nvidia-smi"},
+						{Path: "/also-not-nvidia-smi", HostPath: "not-nvidia-smi"},
+					}
+					return mounts, nil
+				},
+			},
+		},
+		{
+			description: "nvidia-smi returns hook",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					mounts := []discover.Mount{
+						{Path: "nvidia-smi"},
+					}
+					return mounts, nil
+				},
+			},
+			expectedHooks: []discover.Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "nvidia-ctk",
+					Args: []string{"nvidia-ctk", "hook", "create-symlinks",
+						"--link", "nvidia-smi::/usr/bin/nvidia-smi"},
+				},
+			},
+		},
+		{
+			description: "checks basename",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					mounts := []discover.Mount{
+						{Path: "/some/path/nvidia-smi"},
+						{Path: "/nvidia-smi/but-not"},
+					}
+					return mounts, nil
+				},
+			},
+			expectedHooks: []discover.Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "nvidia-ctk",
+					Args: []string{"nvidia-ctk", "hook", "create-symlinks",
+						"--link", "/some/path/nvidia-smi::/usr/bin/nvidia-smi"},
+				},
+			},
+		},
+		{
+			description: "returns first match",
+			mounts: &discover.DiscoverMock{
+				MountsFunc: func() ([]discover.Mount, error) {
+					mounts := []discover.Mount{
+						{Path: "/some/path/nvidia-smi"},
+						{Path: "/another/path/nvidia-smi"},
+					}
+					return mounts, nil
+				},
+			},
+			expectedHooks: []discover.Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "nvidia-ctk",
+					Args: []string{"nvidia-ctk", "hook", "create-symlinks",
+						"--link", "/some/path/nvidia-smi::/usr/bin/nvidia-smi"},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			m := nvidiaSMISimlinkHook{
+				logger:        logger,
+				mountsFrom:    tc.mounts,
+				nvidiaCTKPath: "nvidia-ctk",
+			}
+
+			devices, err := m.Devices()
+			require.NoError(t, err)
+			require.Empty(t, devices)
+
+			mounts, err := m.Mounts()
+			require.NoError(t, err)
+			require.Empty(t, mounts)
+
+			hooks, err := m.Hooks()
+			require.ErrorIs(t, err, tc.expectedError)
+			require.Equal(t, tc.expectedHooks, hooks)
+		})
+	}
+}

pkg/nvcdi/full-gpu-nvml.go

@@ -39,7 +39,7 @@ func (l *nvmllib) GetGPUDeviceSpecs(i int, d device.Device) (*specs.Device, erro
 		return nil, fmt.Errorf("failed to get edits for device: %v", err)
 	}
 
-	name, err := l.deviceNamer.GetDeviceName(i, d)
+	name, err := l.deviceNamer.GetDeviceName(i, convert{d})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get device name: %v", err)
 	}

pkg/nvcdi/lib-csv.go

@@ -53,8 +53,13 @@ func (l *csvlib) GetAllDeviceSpecs() ([]specs.Device, error) {
 		return nil, fmt.Errorf("failed to create container edits for CSV files: %v", err)
 	}
 
+	name, err := l.deviceNamer.GetDeviceName(0, uuidUnsupported{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get device name: %v", err)
+	}
+
 	deviceSpec := specs.Device{
-		Name:           "all",
+		Name:           name,
 		ContainerEdits: *e.ContainerEdits,
 	}
 	return []specs.Device{deviceSpec}, nil

pkg/nvcdi/mig-device-nvml.go

@@ -36,7 +36,7 @@ func (l *nvmllib) GetMIGDeviceSpecs(i int, d device.Device, j int, mig device.Mi
 		return nil, fmt.Errorf("failed to get edits for device: %v", err)
 	}
 
-	name, err := l.deviceNamer.GetMigDeviceName(i, d, j, mig)
+	name, err := l.deviceNamer.GetMigDeviceName(i, convert{d}, j, convert{mig})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get device name: %v", err)
 	}

pkg/nvcdi/namer.go

@@ -17,16 +17,21 @@
 package nvcdi
 
 import (
+	"errors"
 	"fmt"
 
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
 )
 
+// UUIDer is an interface for getting UUIDs.
+type UUIDer interface {
+	GetUUID() (string, error)
+}
+
 // DeviceNamer is an interface for getting device names
 type DeviceNamer interface {
-	GetDeviceName(int, device.Device) (string, error)
-	GetMigDeviceName(int, device.Device, int, device.MigDevice) (string, error)
+	GetDeviceName(int, UUIDer) (string, error)
+	GetMigDeviceName(int, UUIDer, int, UUIDer) (string, error)
 }
 
 // Supported device naming strategies
@@ -61,29 +66,57 @@ func NewDeviceNamer(strategy string) (DeviceNamer, error) {
 }
 
 // GetDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameIndex) GetDeviceName(i int, d device.Device) (string, error) {
+func (s deviceNameIndex) GetDeviceName(i int, _ UUIDer) (string, error) {
 	return fmt.Sprintf("%s%d", s.gpuPrefix, i), nil
 }
 
 // GetMigDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameIndex) GetMigDeviceName(i int, d device.Device, j int, mig device.MigDevice) (string, error) {
+func (s deviceNameIndex) GetMigDeviceName(i int, _ UUIDer, j int, _ UUIDer) (string, error) {
 	return fmt.Sprintf("%s%d:%d", s.migPrefix, i, j), nil
 }
 
 // GetDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameUUID) GetDeviceName(i int, d device.Device) (string, error) {
-	uuid, ret := d.GetUUID()
-	if ret != nvml.SUCCESS {
-		return "", fmt.Errorf("failed to get device UUID: %v", ret)
+func (s deviceNameUUID) GetDeviceName(i int, d UUIDer) (string, error) {
+	uuid, err := d.GetUUID()
+	if err != nil {
+		return "", fmt.Errorf("failed to get device UUID: %v", err)
 	}
 	return uuid, nil
 }
 
 // GetMigDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameUUID) GetMigDeviceName(i int, d device.Device, j int, mig device.MigDevice) (string, error) {
-	uuid, ret := mig.GetUUID()
-	if ret != nvml.SUCCESS {
-		return "", fmt.Errorf("failed to get device UUID: %v", ret)
+func (s deviceNameUUID) GetMigDeviceName(i int, _ UUIDer, j int, mig UUIDer) (string, error) {
+	uuid, err := mig.GetUUID()
+	if err != nil {
+		return "", fmt.Errorf("failed to get device UUID: %v", err)
 	}
 	return uuid, nil
 }
+
+//go:generate moq -stub -out namer_nvml_mock.go . nvmlUUIDer
+type nvmlUUIDer interface {
+	GetUUID() (string, nvml.Return)
+}
+
+type convert struct {
+	nvmlUUIDer
+}
+
+type uuidUnsupported struct{}
+
+func (m convert) GetUUID() (string, error) {
+	if m.nvmlUUIDer == nil {
+		return uuidUnsupported{}.GetUUID()
+	}
+	uuid, ret := m.nvmlUUIDer.GetUUID()
+	if ret != nvml.SUCCESS {
+		return "", ret
+	}
+	return uuid, nil
+}
+
+var errUUIDUnsupported = errors.New("GetUUID is not supported")
+
+func (m uuidUnsupported) GetUUID() (string, error) {
+	return "", errUUIDUnsupported
+}

pkg/nvcdi/namer_nvml_mock.go

@@ -0,0 +1,72 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package nvcdi
+
+import (
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+	"sync"
+)
+
+// Ensure, that nvmlUUIDerMock does implement nvmlUUIDer.
+// If this is not the case, regenerate this file with moq.
+var _ nvmlUUIDer = &nvmlUUIDerMock{}
+
+// nvmlUUIDerMock is a mock implementation of nvmlUUIDer.
+//
+//	func TestSomethingThatUsesnvmlUUIDer(t *testing.T) {
+//
+//		// make and configure a mocked nvmlUUIDer
+//		mockednvmlUUIDer := &nvmlUUIDerMock{
+//			GetUUIDFunc: func() (string, nvml.Return) {
+//				panic("mock out the GetUUID method")
+//			},
+//		}
+//
+//		// use mockednvmlUUIDer in code that requires nvmlUUIDer
+//		// and then make assertions.
+//
+//	}
+type nvmlUUIDerMock struct {
+	// GetUUIDFunc mocks the GetUUID method.
+	GetUUIDFunc func() (string, nvml.Return)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// GetUUID holds details about calls to the GetUUID method.
+		GetUUID []struct {
+		}
+	}
+	lockGetUUID sync.RWMutex
+}
+
+// GetUUID calls GetUUIDFunc.
+func (mock *nvmlUUIDerMock) GetUUID() (string, nvml.Return) {
+	callInfo := struct {
+	}{}
+	mock.lockGetUUID.Lock()
+	mock.calls.GetUUID = append(mock.calls.GetUUID, callInfo)
+	mock.lockGetUUID.Unlock()
+	if mock.GetUUIDFunc == nil {
+		var (
+			sOut      string
+			returnOut nvml.Return
+		)
+		return sOut, returnOut
+	}
+	return mock.GetUUIDFunc()
+}
+
+// GetUUIDCalls gets all the calls that were made to GetUUID.
+// Check the length with:
+//
+//	len(mockednvmlUUIDer.GetUUIDCalls())
+func (mock *nvmlUUIDerMock) GetUUIDCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockGetUUID.RLock()
+	calls = mock.calls.GetUUID
+	mock.lockGetUUID.RUnlock()
+	return calls
+}

pkg/nvcdi/namer_test.go

@@ -0,0 +1,67 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+func TestConvert(t *testing.T) {
+	testCases := []struct {
+		description   string
+		nvml          nvmlUUIDer
+		expectedError error
+		expecteUUID   string
+	}{
+		{
+			description:   "empty UUIDer returns error",
+			expectedError: errUUIDUnsupported,
+			expecteUUID:   "",
+		},
+		{
+			description: "nvmlUUIDer returns UUID",
+			nvml: &nvmlUUIDerMock{
+				GetUUIDFunc: func() (string, nvml.Return) {
+					return "SOME_UUID", nvml.SUCCESS
+				},
+			},
+			expectedError: nil,
+			expecteUUID:   "SOME_UUID",
+		},
+		{
+			description: "nvmlUUIDer returns error",
+			nvml: &nvmlUUIDerMock{
+				GetUUIDFunc: func() (string, nvml.Return) {
+					return "SOME_UUID", nvml.ERROR_UNKNOWN
+				},
+			},
+			expectedError: nvml.ERROR_UNKNOWN,
+			expecteUUID:   "",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			uuid, err := convert{tc.nvml}.GetUUID()
+			require.ErrorIs(t, err, tc.expectedError)
+			require.Equal(t, tc.expecteUUID, uuid)
+		})
+	}
+}

scripts/archive-packages.sh

@@ -17,7 +17,10 @@
 
 function assert_usage() {
     echo "Incorrect arguments: $*"
-    echo "$(basename ${BASH_SOURCE[0]}) ARTIFACTORY_REPO GIT_REFERENCE"
+    echo "$(basename ${BASH_SOURCE[0]}) ARTIFACTORY_REPO [GIT_REFERENCE]"
+    echo "  ARTIFACTORY_REPO: URL to Artifactory repository"
+    echo "  GIT_REFERENCE: Git reference to use for the package version"
+    echo "               (if not specified, PACKAGE_IMAGE_TAG must be set)"
     exit 1
 }
 
@@ -26,7 +29,7 @@ set -e
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
 PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
 
-if [[ $# -le 1 ]]; then
+if [[ $# -lt 1 ]]; then
     assert_usage "$@"
 fi
 

scripts/release-packages.sh

@@ -102,9 +102,20 @@ function sync() {
     ubuntu*) pkg_type=deb
         ;;
     *) echo "ERROR: unexpected distribution ${src_dist}"
+       exit 1
         ;;
     esac
 
+    if [[ $# -ge 4 && $4 == "package_type" ]] ; then
+        if [[ "${src_dist}" != "ubuntu18.04" && "${src_dist}" != "centos7" ]]; then
+            echo "Package type repos require ubuntu18.04 or centos7 as the source"
+            echo "skipping"
+            return
+        fi
+        dst_dist=$pkg_type
+    fi
+
+
     local arch=${target##*-}
     local dst_arch=${arch}
     case ${src_dist} in
@@ -174,11 +185,13 @@ git -C ${PACKAGE_REPO_ROOT} clean -fdx ${REPO}
 
 for target in ${targets[@]}; do
     sync ${target} ${PACKAGE_CACHE}/packages ${PACKAGE_REPO_ROOT}/${REPO}
+    # We also create a `package_type` repo; internally we skip this for non-ubuntu18.04 or centos7 distributions
+    sync ${target} ${PACKAGE_CACHE}/packages ${PACKAGE_REPO_ROOT}/${REPO} "package_type"
 done
 
 git -C ${PACKAGE_REPO_ROOT} add ${REPO}
 
-if [[ ${REPO} == "stable" ]]; then
+if [[ "${REPO}" == "stable" ]]; then
 # Stable release
 git -C ${PACKAGE_REPO_ROOT} commit -s -F- <<EOF
 Add packages for NVIDIA Container Toolkit ${VERSION} release

test/container/containerd_test.sh

@@ -27,7 +27,7 @@ testing::containerd::dind::setup() {
 		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
 		-v "${shared_dir}/run/docker/containerd:/run/docker/containerd" \
 		--name "${containerd_dind_ctr}" \
-		docker:stable-dind -H unix://${containerd_dind_socket}
+		docker:dind -H unix://${containerd_dind_socket}
 }
 
 testing::containerd::dind::exec() {

test/container/docker_test.sh

@@ -25,7 +25,7 @@ testing::docker::dind::setup() {
 		-v "${shared_dir}/run/nvidia:/run/nvidia" \
 		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
 		--name "${docker_dind_ctr}" \
-		docker:stable-dind -H unix://${docker_dind_socket}
+		docker:dind -H unix://${docker_dind_socket}
 }
 
 testing::docker::dind::exec() {

test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so

@@ -1 +0,0 @@
-# This is a dummy lib file to test nvidia-runtime.experimental

test/container/toolkit_test.sh

@@ -46,13 +46,6 @@ testing::toolkit::install() {
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-hook.real"
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
 
-	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental.real"
-
-	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-	grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
 
 	# Ensure that the config file has the required contents.

tools/container/README.md

@@ -15,28 +15,23 @@ docker setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`.
 
 Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
 
-**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
 
 | Flags                                                       | Installed Runtimes              | Default Runtime       |
 |-------------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--runtime-name nvidia`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--runtime-name NAME`                                       | `NAME`, `nvidia-experimental`   | `NAME`                |
-| `--runtime-name nvidia-experimental`                        | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
-| `--set-as-default`                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-name nvidia`                    | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-name NAME`                      | `NAME`, `nvidia-experimental`   | `NAME`                |
-| `--set-as-default --runtime-name nvidia-experimental`       | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
-| `--set-as-default=false`                                    | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default=false --runtime-name NAME`                | `NAME`, `nvidia-experimental`   | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia`              | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| **NONE SPECIFIED**                                          | `nvidia`                        | `nvidia`              |
+| `--runtime-name nvidia`                                     | `nvidia`                        | `nvidia`              |
+| `--runtime-name NAME`                                       | `NAME`                          | `NAME`                |
+| `--set-as-default`                                          | `nvidia`                        | `nvidia`              |
+| `--set-as-default --runtime-name nvidia`                    | `nvidia`                        | `nvidia`              |
+| `--set-as-default --runtime-name NAME`                      | `NAME`                          | `NAME`                |
+| `--set-as-default=false`                                    | `nvidia`                        | **NOT SET**           |
+| `--set-as-default=false --runtime-name NAME`                | `NAME`                          | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia`              | `nvidia`                        | **NOT SET**           |
 
 These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
 
@@ -48,7 +43,7 @@ containerd setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`.
 
 Adding the `--set-as-default` flag as follows:
 ```bash
@@ -59,19 +54,15 @@ containerd setup \
 ```
 will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
 
-**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
 
 | Flags                                                  | Installed Runtime Classes       | Default Runtime Class |
 |--------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                     | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--runtime-class NAME`                                 | `NAME`, `nvidia-experimental`   | **NOT SET**           |
-| `--runtime-class nvidia`                               | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--runtime-class nvidia-experimental`                  | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-class NAME`                | `NAME`, `nvidia-experimental`   | `NAME`                |
-| `--set-as-default --runtime-class nvidia`              | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| **NONE SPECIFIED**                                     | `nvidia`                        | **NOT SET**           |
+| `--runtime-class NAME`                                 | `NAME`                          | **NOT SET**           |
+| `--runtime-class nvidia`                               | `nvidia`                        | **NOT SET**           |
+| `--set-as-default`                                     | `nvidia`                        | `nvidia`              |
+| `--set-as-default --runtime-class NAME`                | `NAME`                          | `NAME`                |
+| `--set-as-default --runtime-class nvidia`              | `nvidia`                        | `nvidia`              |
 
 These combinations also hold for the environment variables that map to the command line flags.

tools/container/containerd/config_v1_test.go

@@ -56,13 +56,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
 			expectedDefaultRuntimeName:   nil,
 			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
 		},
-		{
-			legacyConfig:                 true,
-			setAsDefault:                 true,
-			runtimeName:                  "nvidia-experimental",
-			expectedDefaultRuntimeName:   nil,
-			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime.experimental",
-		},
 		{
 			legacyConfig:                 false,
 			setAsDefault:                 false,
@@ -82,13 +75,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
 			expectedDefaultRuntimeName:   "NAME",
 			expectedDefaultRuntimeBinary: nil,
 		},
-		{
-			legacyConfig:                 false,
-			setAsDefault:                 true,
-			runtimeName:                  "nvidia-experimental",
-			expectedDefaultRuntimeName:   "nvidia-experimental",
-			expectedDefaultRuntimeBinary: nil,
-		},
 	}
 
 	for i, tc := range testCases {
@@ -163,17 +149,6 @@ func TestUpdateV1Config(t *testing.T) {
 										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -221,75 +196,6 @@ func TestUpdateV1Config(t *testing.T) {
 										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(1),
-				"plugins": map[string]interface{}{
-					"cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -387,18 +293,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
 										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -459,90 +353,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
 										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(1),
-				"plugins": map[string]interface{}{
-					"cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"runc": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/runc-binary",
-									},
-								},
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -624,10 +434,9 @@ func TestRevertV1Config(t *testing.T) {
 					"cri": map[string]interface{}{
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
-								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
-								"nvidia-cdi":          runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
-								"nvidia-legacy":       runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
+								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
 							},
 						},
 					},
@@ -641,10 +450,9 @@ func TestRevertV1Config(t *testing.T) {
 					"cri": map[string]interface{}{
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
-								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
-								"nvidia-cdi":          runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
-								"nvidia-legacy":       runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
+								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
 							},
 							"default_runtime":      defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
 							"default_runtime_name": "nvidia",

tools/container/containerd/config_v2_test.go

@@ -49,11 +49,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: nil,
 		},
-		{
-			setAsDefault:               false,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: nil,
-		},
 		{
 			setAsDefault:               true,
 			runtimeName:                "nvidia",
@@ -64,11 +59,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: "NAME",
 		},
-		{
-			setAsDefault:               true,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: "nvidia-experimental",
-		},
 	}
 
 	for i, tc := range testCases {
@@ -124,16 +114,6 @@ func TestUpdateV2Config(t *testing.T) {
 										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -178,70 +158,6 @@ func TestUpdateV2Config(t *testing.T) {
 										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(2),
-				"plugins": map[string]interface{}{
-					"io.containerd.grpc.v1.cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -337,17 +253,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -405,85 +310,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(2),
-				"plugins": map[string]interface{}{
-					"io.containerd.grpc.v1.cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"runc": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/runc-binary",
-									},
-								},
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -562,8 +388,7 @@ func TestRevertV2Config(t *testing.T) {
 					"io.containerd.grpc.v1.cri": map[string]interface{}{
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
-								"nvidia":              runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
+								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
 							},
 						},
 					},
@@ -577,8 +402,7 @@ func TestRevertV2Config(t *testing.T) {
 					"io.containerd.grpc.v1.cri": map[string]interface{}{
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
-								"nvidia":              runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
+								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
 							},
 							"default_runtime_name": "nvidia",
 						},

tools/container/containerd/containerd.go

@@ -105,29 +105,29 @@ func main() {
 			Usage:       "Path to the containerd config file",
 			Value:       defaultConfig,
 			Destination: &options.Config,
-			EnvVars:     []string{"CONTAINERD_CONFIG", "RUNTIME_CONFIG"},
+			EnvVars:     []string{"RUNTIME_CONFIG", "CONTAINERD_CONFIG"},
 		},
 		&cli.StringFlag{
 			Name:        "socket",
 			Usage:       "Path to the containerd socket file",
 			Value:       defaultSocket,
 			Destination: &options.Socket,
-			EnvVars:     []string{"CONTAINERD_SOCKET", "RUNTIME_SOCKET"},
+			EnvVars:     []string{"RUNTIME_SOCKET", "CONTAINERD_SOCKET"},
 		},
 		&cli.StringFlag{
 			Name:        "restart-mode",
 			Usage:       "Specify how containerd should be restarted;  If 'none' is selected, it will not be restarted [signal | systemd | none]",
 			Value:       defaultRestartMode,
 			Destination: &options.RestartMode,
-			EnvVars:     []string{"CONTAINERD_RESTART_MODE", "RUNTIME_RESTART_MODE"},
+			EnvVars:     []string{"RUNTIME_RESTART_MODE", "CONTAINERD_RESTART_MODE"},
 		},
 		&cli.StringFlag{
-			Name:        "runtime-class",
-			Aliases:     []string{"runtime-name", "nvidia-runtime-name"},
+			Name:        "runtime-name",
+			Aliases:     []string{"nvidia-runtime-name", "runtime-class"},
 			Usage:       "The name of the runtime class to set for the nvidia-container-runtime",
 			Value:       defaultRuntimeClass,
 			Destination: &options.RuntimeName,
-			EnvVars:     []string{"CONTAINERD_RUNTIME_CLASS", "NVIDIA_RUNTIME_NAME"},
+			EnvVars:     []string{"NVIDIA_RUNTIME_NAME", "CONTAINERD_RUNTIME_CLASS"},
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-runtime-dir",
@@ -141,7 +141,7 @@ func main() {
 			Usage:       "Set nvidia-container-runtime as the default runtime",
 			Value:       defaultSetAsDefault,
 			Destination: &options.SetAsDefault,
-			EnvVars:     []string{"CONTAINERD_SET_AS_DEFAULT", "NVIDIA_RUNTIME_SET_AS_DEFAULT"},
+			EnvVars:     []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "CONTAINERD_SET_AS_DEFAULT"},
 			Hidden:      true,
 		},
 		&cli.StringFlag{

tools/container/crio/crio.go

@@ -105,14 +105,14 @@ func main() {
 			Usage:       "Path to the cri-o config file",
 			Value:       defaultConfig,
 			Destination: &options.Config,
-			EnvVars:     []string{"CRIO_CONFIG", "RUNTIME_CONFIG"},
+			EnvVars:     []string{"RUNTIME_CONFIG", "CRIO_CONFIG"},
 		},
 		&cli.StringFlag{
 			Name:        "socket",
 			Usage:       "Path to the crio socket file",
 			Value:       "",
 			Destination: &options.Socket,
-			EnvVars:     []string{"CRIO_SOCKET", "RUNTIME_SOCKET"},
+			EnvVars:     []string{"RUNTIME_SOCKET", "CRIO_SOCKET"},
 			// Note: We hide this option since restarting cri-o via a socket is not supported.
 			Hidden: true,
 		},
@@ -121,15 +121,15 @@ func main() {
 			Usage:       "Specify how cri-o should be restarted;  If 'none' is selected, it will not be restarted [systemd | none]",
 			Value:       defaultRestartMode,
 			Destination: &options.RestartMode,
-			EnvVars:     []string{"CRIO_RESTART_MODE", "RUNTIME_RESTART_MODE"},
+			EnvVars:     []string{"RUNTIME_RESTART_MODE", "CRIO_RESTART_MODE"},
 		},
 		&cli.StringFlag{
-			Name:        "runtime-class",
-			Aliases:     []string{"runtime-name", "nvidia-runtime-name"},
+			Name:        "runtime-name",
+			Aliases:     []string{"nvidia-runtime-name", "runtime-class"},
 			Usage:       "The name of the runtime class to set for the nvidia-container-runtime",
 			Value:       defaultRuntimeClass,
 			Destination: &options.RuntimeName,
-			EnvVars:     []string{"CRIO_RUNTIME_CLASS", "NVIDIA_RUNTIME_NAME"},
+			EnvVars:     []string{"NVIDIA_RUNTIME_NAME", "CRIO_RUNTIME_CLASS"},
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-runtime-dir",
@@ -143,7 +143,7 @@ func main() {
 			Usage:       "Set nvidia-container-runtime as the default runtime",
 			Value:       defaultSetAsDefault,
 			Destination: &options.SetAsDefault,
-			EnvVars:     []string{"CRIO_SET_AS_DEFAULT", "NVIDIA_RUNTIME_SET_AS_DEFAULT"},
+			EnvVars:     []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "CRIO_SET_AS_DEFAULT"},
 			Hidden:      true,
 		},
 		&cli.StringFlag{

tools/container/docker/docker.go

@@ -99,21 +99,21 @@ func main() {
 			Usage:       "Path to docker config file",
 			Value:       defaultConfig,
 			Destination: &options.Config,
-			EnvVars:     []string{"DOCKER_CONFIG", "RUNTIME_CONFIG"},
+			EnvVars:     []string{"RUNTIME_CONFIG", "DOCKER_CONFIG"},
 		},
 		&cli.StringFlag{
 			Name:        "socket",
 			Usage:       "Path to the docker socket file",
 			Value:       defaultSocket,
 			Destination: &options.Socket,
-			EnvVars:     []string{"DOCKER_SOCKET", "RUNTIME_SOCKET"},
+			EnvVars:     []string{"RUNTIME_SOCKET", "DOCKER_SOCKET"},
 		},
 		&cli.StringFlag{
 			Name:        "restart-mode",
 			Usage:       "Specify how docker should be restarted; If 'none' is selected it will not be restarted [signal | systemd | none ]",
 			Value:       defaultRestartMode,
 			Destination: &options.RestartMode,
-			EnvVars:     []string{"DOCKER_RESTART_MODE", "RUNTIME_RESTART_MODE"},
+			EnvVars:     []string{"RUNTIME_RESTART_MODE", "DOCKER_RESTART_MODE"},
 		},
 		&cli.StringFlag{
 			Name:        "host-root",
@@ -127,11 +127,11 @@ func main() {
 		},
 		&cli.StringFlag{
 			Name:        "runtime-name",
-			Aliases:     []string{"runtime-class", "nvidia-runtime-name"},
+			Aliases:     []string{"nvidia-runtime-name", "runtime-class"},
 			Usage:       "Specify the name of the `nvidia` runtime. If set-as-default is selected, the runtime is used as the default runtime.",
 			Value:       defaultRuntimeName,
 			Destination: &options.RuntimeName,
-			EnvVars:     []string{"DOCKER_RUNTIME_NAME", "NVIDIA_RUNTIME_NAME"},
+			EnvVars:     []string{"NVIDIA_RUNTIME_NAME", "DOCKER_RUNTIME_NAME"},
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-runtime-dir",
@@ -142,10 +142,10 @@ func main() {
 		},
 		&cli.BoolFlag{
 			Name:        "set-as-default",
-			Usage:       "Set the `nvidia` runtime as the default runtime. If --runtime-name is specified as `nvidia-experimental` the experimental runtime is set as the default runtime instead",
+			Usage:       "Set the `nvidia` runtime as the default runtime.",
 			Value:       defaultSetAsDefault,
 			Destination: &options.SetAsDefault,
-			EnvVars:     []string{"DOCKER_SET_AS_DEFAULT", "NVIDIA_RUNTIME_SET_AS_DEFAULT"},
+			EnvVars:     []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"},
 			Hidden:      true,
 		},
 	}

tools/container/docker/docker_test.go

@@ -43,11 +43,6 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: "NAME",
 		},
-		{
-			setAsDefault:               true,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: "nvidia-experimental",
-		},
 		{
 			setAsDefault:               true,
 			runtimeName:                "nvidia",
@@ -92,10 +87,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -117,35 +108,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
-					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
-						"args": []string{},
-					},
-					"nvidia-legacy": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config:       map[string]interface{}{},
-			setAsDefault: false,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -173,10 +135,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -207,10 +165,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -235,38 +189,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
-					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
-						"args": []string{},
-					},
-					"nvidia-legacy": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"default-runtime": "runc",
-			},
-			setAsDefault: true,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"default-runtime": "nvidia-experimental",
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -299,10 +221,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -361,7 +279,7 @@ func TestRevertConfig(t *testing.T) {
 		{
 			config: map[string]interface{}{
 				"runtimes": map[string]interface{}{
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
@@ -376,25 +294,6 @@ func TestRevertConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
-				},
-			},
-			expectedConfig: map[string]interface{}{},
-		},
-		{
-			config: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},

tools/container/operator/operator.go

@@ -19,8 +19,7 @@ package operator
 import "path/filepath"
 
 const (
-	defaultRuntimeName      = "nvidia"
-	experimentalRuntimeName = "nvidia-experimental"
+	defaultRuntimeName = "nvidia"
 
 	defaultRoot = "/usr/bin"
 )
@@ -59,7 +58,7 @@ func GetRuntimes(opts ...Option) Runtimes {
 	runtimes := make(Runtimes)
 	runtimes.add(c.nvidiaRuntime())
 
-	modes := []string{"experimental", "cdi", "legacy"}
+	modes := []string{"cdi", "legacy"}
 	for _, mode := range modes {
 		runtimes.add(c.modeRuntime(mode))
 	}
@@ -85,9 +84,8 @@ func (r *Runtimes) add(runtime Runtime) {
 // If name is equal to one of the predefined runtimes, `nvidia` is used as the runtime name instead.
 func (c config) nvidiaRuntime() Runtime {
 	predefinedRuntimes := map[string]struct{}{
-		"nvidia-experimental": {},
-		"nvidia-cdi":          {},
-		"nvidia-legacy":       {},
+		"nvidia-cdi":    {},
+		"nvidia-legacy": {},
 	}
 	name := c.nvidiaRuntimeName
 	if _, isPredefinedRuntime := predefinedRuntimes[name]; isPredefinedRuntime {

tools/container/operator/operator_test.go

@@ -37,10 +37,6 @@ func TestOptions(t *testing.T) {
 					name: "nvidia",
 					Path: "/usr/bin/nvidia-container-runtime",
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -60,10 +56,6 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime",
 					SetAsDefault: true,
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -84,10 +76,6 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime",
 					SetAsDefault: true,
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -108,10 +96,6 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime",
 					SetAsDefault: true,
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -130,56 +114,6 @@ func TestOptions(t *testing.T) {
 					name: "NAME",
 					Path: "/usr/bin/nvidia-container-runtime",
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
-				"nvidia-cdi": Runtime{
-					name: "nvidia-cdi",
-					Path: "/usr/bin/nvidia-container-runtime.cdi",
-				},
-				"nvidia-legacy": Runtime{
-					name: "nvidia-legacy",
-					Path: "/usr/bin/nvidia-container-runtime.legacy",
-				},
-			},
-		},
-		{
-			setAsDefault:           true,
-			nvidiaRuntimeName:      "nvidia-experimental",
-			expectedDefaultRuntime: "nvidia-experimental",
-			expectedRuntimes: Runtimes{
-				"nvidia": Runtime{
-					name: "nvidia",
-					Path: "/usr/bin/nvidia-container-runtime",
-				},
-				"nvidia-experimental": Runtime{
-					name:         "nvidia-experimental",
-					Path:         "/usr/bin/nvidia-container-runtime.experimental",
-					SetAsDefault: true,
-				},
-				"nvidia-cdi": Runtime{
-					name: "nvidia-cdi",
-					Path: "/usr/bin/nvidia-container-runtime.cdi",
-				},
-				"nvidia-legacy": Runtime{
-					name: "nvidia-legacy",
-					Path: "/usr/bin/nvidia-container-runtime.legacy",
-				},
-			},
-		},
-		{
-			setAsDefault:      false,
-			nvidiaRuntimeName: "nvidia-experimental",
-			expectedRuntimes: Runtimes{
-				"nvidia": Runtime{
-					name: "nvidia",
-					Path: "/usr/bin/nvidia-container-runtime",
-				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",

tools/container/toolkit/runtime.go

@@ -19,16 +19,12 @@ package main
 import (
 	"fmt"
 	"path/filepath"
-	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
-	log "github.com/sirupsen/logrus"
 )
 
 const (
 	nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
-
-	nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental"
 )
 
 // installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
@@ -36,9 +32,6 @@ const (
 func installContainerRuntimes(toolkitDir string, driverRoot string) error {
 	runtimes := operator.GetRuntimes()
 	for _, runtime := range runtimes {
-		if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource {
-			continue
-		}
 		r := newNvidiaContainerRuntimeInstaller(runtime.Path)
 
 		_, err := r.install(toolkitDir)
@@ -46,30 +39,6 @@ func installContainerRuntimes(toolkitDir string, driverRoot string) error {
 			return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
 		}
 	}
-
-	// Install the experimental runtime and treat failures as non-fatal.
-	err := installExperimentalRuntime(toolkitDir, driverRoot)
-	if err != nil {
-		log.Warningf("Could not install experimental runtime: %v", err)
-	}
-
-	return nil
-}
-
-// installExperimentalRuntime ensures that the experimental NVIDIA Container runtime is installed
-func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
-	libraryRoot, err := findLibraryRoot(driverRoot)
-	if err != nil {
-		log.Warningf("Error finding library path for root %v: %v", driverRoot, err)
-	}
-	log.Infof("Using library root %v", libraryRoot)
-
-	e := newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot)
-	_, err = e.install(toolkitDir)
-	if err != nil {
-		return fmt.Errorf("error installing experimental NVIDIA Container Runtime: %v", err)
-	}
-
 	return nil
 }
 
@@ -87,22 +56,6 @@ func newNvidiaContainerRuntimeInstaller(source string) *executable {
 	return newRuntimeInstaller(source, target, nil)
 }
 
-func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {
-	source := nvidiaExperimentalContainerRuntimeSource
-	wrapperName := filepath.Base(source)
-	dotfileName := wrapperName + ".real"
-	target := executableTarget{
-		dotfileName: dotfileName,
-		wrapperName: wrapperName,
-	}
-
-	env := make(map[string]string)
-	if libraryRoot != "" {
-		env["LD_LIBRARY_PATH"] = strings.Join([]string{libraryRoot, "$LD_LIBRARY_PATH"}, ":")
-	}
-	return newRuntimeInstaller(source, target, env)
-}
-
 func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {
 	preLines := []string{
 		"",

tools/container/toolkit/runtime_test.go

@@ -55,36 +55,3 @@ func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
 	exepectedContents := strings.Join(expectedLines, "\n")
 	require.Equal(t, exepectedContents, buf.String())
 }
-
-func TestExperimentalContainerRuntimeInstallerWrapper(t *testing.T) {
-	r := newNvidiaContainerRuntimeExperimentalInstaller("/some/root/usr/lib64")
-
-	const shebang = "#! /bin/sh"
-	const destFolder = "/dest/folder"
-	const dotfileName = "source.real"
-
-	buf := &bytes.Buffer{}
-
-	err := r.writeWrapperTo(buf, destFolder, dotfileName)
-	require.NoError(t, err)
-
-	expectedLines := []string{
-		shebang,
-		"",
-		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
-		"if [ \"${?}\" != \"0\" ]; then",
-		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
-		"	exec runc \"$@\"",
-		"fi",
-		"",
-		"LD_LIBRARY_PATH=/some/root/usr/lib64:$LD_LIBRARY_PATH \\",
-		"PATH=/dest/folder:$PATH \\",
-		"XDG_CONFIG_HOME=/dest/folder/.config \\",
-		"source.real \\",
-		"\t\"$@\"",
-		"",
-	}
-
-	exepectedContents := strings.Join(expectedLines, "\n")
-	require.Equal(t, exepectedContents, buf.String())
-}

vendor/github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s/objectmeta.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Adapted from k8s.io/apimachinery/pkg/api/validation:
+// https://github.com/kubernetes/apimachinery/blob/7687996c715ee7d5c8cf1e3215e607eb065a4221/pkg/api/validation/objectmeta.go
+
+package k8s
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/multierror"
+)
+
+// TotalAnnotationSizeLimitB defines the maximum size of all annotations in characters.
+const TotalAnnotationSizeLimitB int = 256 * (1 << 10) // 256 kB
+
+// ValidateAnnotations validates that a set of annotations are correctly defined.
+func ValidateAnnotations(annotations map[string]string, path string) error {
+	errors := multierror.New()
+	for k := range annotations {
+		// The rule is QualifiedName except that case doesn't matter, so convert to lowercase before checking.
+		for _, msg := range IsQualifiedName(strings.ToLower(k)) {
+			errors = multierror.Append(errors, fmt.Errorf("%v.%v is invalid: %v", path, k, msg))
+		}
+	}
+	if err := ValidateAnnotationsSize(annotations); err != nil {
+		errors = multierror.Append(errors, fmt.Errorf("%v is too long: %v", path, err))
+	}
+	return errors
+}
+
+// ValidateAnnotationsSize validates that a set of annotations is not too large.
+func ValidateAnnotationsSize(annotations map[string]string) error {
+	var totalSize int64
+	for k, v := range annotations {
+		totalSize += (int64)(len(k)) + (int64)(len(v))
+	}
+	if totalSize > (int64)(TotalAnnotationSizeLimitB) {
+		return fmt.Errorf("annotations size %d is larger than limit %d", totalSize, TotalAnnotationSizeLimitB)
+	}
+	return nil
+}

vendor/github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s/validation.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Adapted from k8s.io/apimachinery/pkg/util/validation:
+// https://github.com/kubernetes/apimachinery/blob/7687996c715ee7d5c8cf1e3215e607eb065a4221/pkg/util/validation/validation.go
+
+package k8s
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+const qnameCharFmt string = "[A-Za-z0-9]"
+const qnameExtCharFmt string = "[-A-Za-z0-9_.]"
+const qualifiedNameFmt string = "(" + qnameCharFmt + qnameExtCharFmt + "*)?" + qnameCharFmt
+const qualifiedNameErrMsg string = "must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character"
+const qualifiedNameMaxLength int = 63
+
+var qualifiedNameRegexp = regexp.MustCompile("^" + qualifiedNameFmt + "$")
+
+// IsQualifiedName tests whether the value passed is what Kubernetes calls a
+// "qualified name".  This is a format used in various places throughout the
+// system.  If the value is not valid, a list of error strings is returned.
+// Otherwise an empty list (or nil) is returned.
+func IsQualifiedName(value string) []string {
+	var errs []string
+	parts := strings.Split(value, "/")
+	var name string
+	switch len(parts) {
+	case 1:
+		name = parts[0]
+	case 2:
+		var prefix string
+		prefix, name = parts[0], parts[1]
+		if len(prefix) == 0 {
+			errs = append(errs, "prefix part "+EmptyError())
+		} else if msgs := IsDNS1123Subdomain(prefix); len(msgs) != 0 {
+			errs = append(errs, prefixEach(msgs, "prefix part ")...)
+		}
+	default:
+		return append(errs, "a qualified name "+RegexError(qualifiedNameErrMsg, qualifiedNameFmt, "MyName", "my.name", "123-abc")+
+			" with an optional DNS subdomain prefix and '/' (e.g. 'example.com/MyName')")
+	}
+
+	if len(name) == 0 {
+		errs = append(errs, "name part "+EmptyError())
+	} else if len(name) > qualifiedNameMaxLength {
+		errs = append(errs, "name part "+MaxLenError(qualifiedNameMaxLength))
+	}
+	if !qualifiedNameRegexp.MatchString(name) {
+		errs = append(errs, "name part "+RegexError(qualifiedNameErrMsg, qualifiedNameFmt, "MyName", "my.name", "123-abc"))
+	}
+	return errs
+}
+
+const labelValueFmt string = "(" + qualifiedNameFmt + ")?"
+const labelValueErrMsg string = "a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character"
+
+// LabelValueMaxLength is a label's max length
+const LabelValueMaxLength int = 63
+
+var labelValueRegexp = regexp.MustCompile("^" + labelValueFmt + "$")
+
+// IsValidLabelValue tests whether the value passed is a valid label value.  If
+// the value is not valid, a list of error strings is returned.  Otherwise an
+// empty list (or nil) is returned.
+func IsValidLabelValue(value string) []string {
+	var errs []string
+	if len(value) > LabelValueMaxLength {
+		errs = append(errs, MaxLenError(LabelValueMaxLength))
+	}
+	if !labelValueRegexp.MatchString(value) {
+		errs = append(errs, RegexError(labelValueErrMsg, labelValueFmt, "MyValue", "my_value", "12345"))
+	}
+	return errs
+}
+
+const dns1123LabelFmt string = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"
+const dns1123LabelErrMsg string = "a lowercase RFC 1123 label must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character"
+
+// DNS1123LabelMaxLength is a label's max length in DNS (RFC 1123)
+const DNS1123LabelMaxLength int = 63
+
+var dns1123LabelRegexp = regexp.MustCompile("^" + dns1123LabelFmt + "$")
+
+// IsDNS1123Label tests for a string that conforms to the definition of a label in
+// DNS (RFC 1123).
+func IsDNS1123Label(value string) []string {
+	var errs []string
+	if len(value) > DNS1123LabelMaxLength {
+		errs = append(errs, MaxLenError(DNS1123LabelMaxLength))
+	}
+	if !dns1123LabelRegexp.MatchString(value) {
+		errs = append(errs, RegexError(dns1123LabelErrMsg, dns1123LabelFmt, "my-name", "123-abc"))
+	}
+	return errs
+}
+
+const dns1123SubdomainFmt string = dns1123LabelFmt + "(\\." + dns1123LabelFmt + ")*"
+const dns1123SubdomainErrorMsg string = "a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character"
+
+// DNS1123SubdomainMaxLength is a subdomain's max length in DNS (RFC 1123)
+const DNS1123SubdomainMaxLength int = 253
+
+var dns1123SubdomainRegexp = regexp.MustCompile("^" + dns1123SubdomainFmt + "$")
+
+// IsDNS1123Subdomain tests for a string that conforms to the definition of a
+// subdomain in DNS (RFC 1123).
+func IsDNS1123Subdomain(value string) []string {
+	var errs []string
+	if len(value) > DNS1123SubdomainMaxLength {
+		errs = append(errs, MaxLenError(DNS1123SubdomainMaxLength))
+	}
+	if !dns1123SubdomainRegexp.MatchString(value) {
+		errs = append(errs, RegexError(dns1123SubdomainErrorMsg, dns1123SubdomainFmt, "example.com"))
+	}
+	return errs
+}
+
+const dns1035LabelFmt string = "[a-z]([-a-z0-9]*[a-z0-9])?"
+const dns1035LabelErrMsg string = "a DNS-1035 label must consist of lower case alphanumeric characters or '-', start with an alphabetic character, and end with an alphanumeric character"
+
+// DNS1035LabelMaxLength is a label's max length in DNS (RFC 1035)
+const DNS1035LabelMaxLength int = 63
+
+var dns1035LabelRegexp = regexp.MustCompile("^" + dns1035LabelFmt + "$")
+
+// IsDNS1035Label tests for a string that conforms to the definition of a label in
+// DNS (RFC 1035).
+func IsDNS1035Label(value string) []string {
+	var errs []string
+	if len(value) > DNS1035LabelMaxLength {
+		errs = append(errs, MaxLenError(DNS1035LabelMaxLength))
+	}
+	if !dns1035LabelRegexp.MatchString(value) {
+		errs = append(errs, RegexError(dns1035LabelErrMsg, dns1035LabelFmt, "my-name", "abc-123"))
+	}
+	return errs
+}
+
+// wildcard definition - RFC 1034 section 4.3.3.
+// examples:
+// - valid: *.bar.com, *.foo.bar.com
+// - invalid: *.*.bar.com, *.foo.*.com, *bar.com, f*.bar.com, *
+const wildcardDNS1123SubdomainFmt = "\\*\\." + dns1123SubdomainFmt
+const wildcardDNS1123SubdomainErrMsg = "a wildcard DNS-1123 subdomain must start with '*.', followed by a valid DNS subdomain, which must consist of lower case alphanumeric characters, '-' or '.' and end with an alphanumeric character"
+
+// IsWildcardDNS1123Subdomain tests for a string that conforms to the definition of a
+// wildcard subdomain in DNS (RFC 1034 section 4.3.3).
+func IsWildcardDNS1123Subdomain(value string) []string {
+	wildcardDNS1123SubdomainRegexp := regexp.MustCompile("^" + wildcardDNS1123SubdomainFmt + "$")
+
+	var errs []string
+	if len(value) > DNS1123SubdomainMaxLength {
+		errs = append(errs, MaxLenError(DNS1123SubdomainMaxLength))
+	}
+	if !wildcardDNS1123SubdomainRegexp.MatchString(value) {
+		errs = append(errs, RegexError(wildcardDNS1123SubdomainErrMsg, wildcardDNS1123SubdomainFmt, "*.example.com"))
+	}
+	return errs
+}
+
+// MaxLenError returns a string explanation of a "string too long" validation
+// failure.
+func MaxLenError(length int) string {
+	return fmt.Sprintf("must be no more than %d characters", length)
+}
+
+// RegexError returns a string explanation of a regex validation failure.
+func RegexError(msg string, fmt string, examples ...string) string {
+	if len(examples) == 0 {
+		return msg + " (regex used for validation is '" + fmt + "')"
+	}
+	msg += " (e.g. "
+	for i := range examples {
+		if i > 0 {
+			msg += " or "
+		}
+		msg += "'" + examples[i] + "', "
+	}
+	msg += "regex used for validation is '" + fmt + "')"
+	return msg
+}
+
+// EmptyError returns a string explanation of a "must not be empty" validation
+// failure.
+func EmptyError() string {
+	return "must be non-empty"
+}
+
+func prefixEach(msgs []string, prefix string) []string {
+	for i := range msgs {
+		msgs[i] = prefix + msgs[i]
+	}
+	return msgs
+}
+
+// InclusiveRangeError returns a string explanation of a numeric "must be
+// between" validation failure.
+func InclusiveRangeError(lo, hi int) string {
+	return fmt.Sprintf(`must be between %d and %d, inclusive`, lo, hi)
+}

vendor/github.com/container-orchestrated-devices/container-device-interface/internal/validation/validate.go

@@ -0,0 +1,56 @@
+/*
+   Copyright © The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package validation
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s"
+)
+
+// ValidateSpecAnnotations checks whether spec annotations are valid.
+func ValidateSpecAnnotations(name string, any interface{}) error {
+	if any == nil {
+		return nil
+	}
+
+	switch v := any.(type) {
+	case map[string]interface{}:
+		annotations := make(map[string]string)
+		for k, v := range v {
+			if s, ok := v.(string); ok {
+				annotations[k] = s
+			} else {
+				return fmt.Errorf("invalid annotation %v.%v; %v is not a string", name, k, any)
+			}
+		}
+		return validateSpecAnnotations(name, annotations)
+	}
+
+	return nil
+}
+
+// validateSpecAnnotations checks whether spec annotations are valid.
+func validateSpecAnnotations(name string, annotations map[string]string) error {
+	path := "annotations"
+	if name != "" {
+		path = strings.Join([]string{name, path}, ".")
+	}
+
+	return k8s.ValidateAnnotations(annotations, path)
+}

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/annotations.go

@@ -20,6 +20,8 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
 )
 
 const (
@@ -101,22 +103,22 @@ func AnnotationKey(pluginName, deviceID string) (string, error) {
 		return "", fmt.Errorf("invalid plugin+deviceID %q, too long", name)
 	}
 
-	if c := rune(name[0]); !isAlphaNumeric(c) {
+	if c := rune(name[0]); !parser.IsAlphaNumeric(c) {
 		return "", fmt.Errorf("invalid name %q, first '%c' should be alphanumeric",
 			name, c)
 	}
 	if len(name) > 2 {
 		for _, c := range name[1 : len(name)-1] {
 			switch {
-			case isAlphaNumeric(c):
+			case parser.IsAlphaNumeric(c):
 			case c == '_' || c == '-' || c == '.':
 			default:
-				return "", fmt.Errorf("invalid name %q, invalid charcter '%c'",
+				return "", fmt.Errorf("invalid name %q, invalid character '%c'",
 					name, c)
 			}
 		}
 	}
-	if c := rune(name[len(name)-1]); !isAlphaNumeric(c) {
+	if c := rune(name[len(name)-1]); !parser.IsAlphaNumeric(c) {
 		return "", fmt.Errorf("invalid name %q, last '%c' should be alphanumeric",
 			name, c)
 	}

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go

@@ -238,7 +238,7 @@ func (d *DeviceNode) Validate() error {
 	}
 	for _, bit := range d.Permissions {
 		if bit != 'r' && bit != 'w' && bit != 'm' {
-			return fmt.Errorf("device %q: invalid persmissions %q",
+			return fmt.Errorf("device %q: invalid permissions %q",
 				d.Path, d.Permissions)
 		}
 	}

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go

@@ -19,6 +19,8 @@ package cdi
 import (
 	"fmt"
 
+	"github.com/container-orchestrated-devices/container-device-interface/internal/validation"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
 	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -50,7 +52,7 @@ func (d *Device) GetSpec() *Spec {
 
 // GetQualifiedName returns the qualified name for this device.
 func (d *Device) GetQualifiedName() string {
-	return QualifiedName(d.spec.GetVendor(), d.spec.GetClass(), d.Name)
+	return parser.QualifiedName(d.spec.GetVendor(), d.spec.GetClass(), d.Name)
 }
 
 // ApplyEdits applies the device-speific container edits to an OCI Spec.
@@ -68,6 +70,13 @@ func (d *Device) validate() error {
 	if err := ValidateDeviceName(d.Name); err != nil {
 		return err
 	}
+	name := d.Name
+	if d.spec != nil {
+		name = d.GetQualifiedName()
+	}
+	if err := validation.ValidateSpecAnnotations(name, d.Annotations); err != nil {
+		return err
+	}
 	edits := d.edits()
 	if edits.isEmpty() {
 		return fmt.Errorf("invalid device, empty device edits")

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/doc.go

@@ -137,7 +137,7 @@
 // were loaded from. The later a directory occurs in the list of CDI
 // directories to scan, the higher priority Spec files loaded from that
 // directory are assigned to. When two or more Spec files define the
-// same device, conflict is resolved by chosing the definition from the
+// same device, conflict is resolved by choosing the definition from the
 // Spec file with the highest priority.
 //
 // The default CDI directory configuration is chosen to encourage
@@ -197,7 +197,7 @@
 //     return registry.SpecDB().WriteSpec(spec, specName)
 // }
 //
-// Similary, generating and later cleaning up transient Spec files can be
+// Similarly, generating and later cleaning up transient Spec files can be
 // done with code fragments similar to the following. These transient Spec
 // files are temporary Spec files with container-specific parametrization.
 // They are typically created before the associated container is created

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/qualified-device.go

@@ -17,27 +17,32 @@
 package cdi
 
 import (
-	"fmt"
-	"strings"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
 )
 
 // QualifiedName returns the qualified name for a device.
 // The syntax for a qualified device names is
-//   "<vendor>/<class>=<name>".
-// A valid vendor name may contain the following runes:
-//   'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'.
-// A valid class name may contain the following runes:
-//   'A'-'Z', 'a'-'z', '0'-'9', '-', '_'.
-// A valid device name may containe the following runes:
-//   'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':'
+//
+//	"<vendor>/<class>=<name>".
+//
+// A valid vendor and class name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'.
+//
+// A valid device name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':'
+//
+// Deprecated: use parser.QualifiedName instead
 func QualifiedName(vendor, class, name string) string {
-	return vendor + "/" + class + "=" + name
+	return parser.QualifiedName(vendor, class, name)
 }
 
 // IsQualifiedName tests if a device name is qualified.
+//
+// Deprecated: use parser.IsQualifiedName instead
 func IsQualifiedName(device string) bool {
-	_, _, _, err := ParseQualifiedName(device)
-	return err == nil
+	return parser.IsQualifiedName(device)
 }
 
 // ParseQualifiedName splits a qualified name into device vendor, class,
@@ -45,66 +50,33 @@ func IsQualifiedName(device string) bool {
 // of the split components fail to pass syntax validation, vendor and
 // class are returned as empty, together with the verbatim input as the
 // name and an error describing the reason for failure.
+//
+// Deprecated: use parser.ParseQualifiedName instead
 func ParseQualifiedName(device string) (string, string, string, error) {
-	vendor, class, name := ParseDevice(device)
-
-	if vendor == "" {
-		return "", "", device, fmt.Errorf("unqualified device %q, missing vendor", device)
-	}
-	if class == "" {
-		return "", "", device, fmt.Errorf("unqualified device %q, missing class", device)
-	}
-	if name == "" {
-		return "", "", device, fmt.Errorf("unqualified device %q, missing device name", device)
-	}
-
-	if err := ValidateVendorName(vendor); err != nil {
-		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
-	}
-	if err := ValidateClassName(class); err != nil {
-		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
-	}
-	if err := ValidateDeviceName(name); err != nil {
-		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
-	}
-
-	return vendor, class, name, nil
+	return parser.ParseQualifiedName(device)
 }
 
 // ParseDevice tries to split a device name into vendor, class, and name.
 // If this fails, for instance in the case of unqualified device names,
 // ParseDevice returns an empty vendor and class together with name set
 // to the verbatim input.
+//
+// Deprecated: use parser.ParseDevice instead
 func ParseDevice(device string) (string, string, string) {
-	if device == "" || device[0] == '/' {
-		return "", "", device
-	}
-
-	parts := strings.SplitN(device, "=", 2)
-	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
-		return "", "", device
-	}
-
-	name := parts[1]
-	vendor, class := ParseQualifier(parts[0])
-	if vendor == "" {
-		return "", "", device
-	}
-
-	return vendor, class, name
+	return parser.ParseDevice(device)
 }
 
 // ParseQualifier splits a device qualifier into vendor and class.
 // The syntax for a device qualifier is
-//     "<vendor>/<class>"
+//
+//	"<vendor>/<class>"
+//
 // If parsing fails, an empty vendor and the class set to the
 // verbatim input is returned.
+//
+// Deprecated: use parser.ParseQualifier instead
 func ParseQualifier(kind string) (string, string) {
-	parts := strings.SplitN(kind, "/", 2)
-	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
-		return "", kind
-	}
-	return parts[0], parts[1]
+	return parser.ParseQualifier(kind)
 }
 
 // ValidateVendorName checks the validity of a vendor name.
@@ -112,54 +84,21 @@ func ParseQualifier(kind string) (string, string) {
 //   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
 //   - digits ('0'-'9')
 //   - underscore, dash, and dot ('_', '-', and '.')
+//
+// Deprecated: use parser.ValidateVendorName instead
 func ValidateVendorName(vendor string) error {
-	if vendor == "" {
-		return fmt.Errorf("invalid (empty) vendor name")
-	}
-	if !isLetter(rune(vendor[0])) {
-		return fmt.Errorf("invalid vendor %q, should start with letter", vendor)
-	}
-	for _, c := range string(vendor[1 : len(vendor)-1]) {
-		switch {
-		case isAlphaNumeric(c):
-		case c == '_' || c == '-' || c == '.':
-		default:
-			return fmt.Errorf("invalid character '%c' in vendor name %q",
-				c, vendor)
-		}
-	}
-	if !isAlphaNumeric(rune(vendor[len(vendor)-1])) {
-		return fmt.Errorf("invalid vendor %q, should end with a letter or digit", vendor)
-	}
-
-	return nil
+	return parser.ValidateVendorName(vendor)
 }
 
 // ValidateClassName checks the validity of class name.
 // A class name may contain the following ASCII characters:
 //   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
 //   - digits ('0'-'9')
-//   - underscore and dash ('_', '-')
+//   - underscore, dash, and dot ('_', '-', and '.')
+//
+// Deprecated: use parser.ValidateClassName instead
 func ValidateClassName(class string) error {
-	if class == "" {
-		return fmt.Errorf("invalid (empty) device class")
-	}
-	if !isLetter(rune(class[0])) {
-		return fmt.Errorf("invalid class %q, should start with letter", class)
-	}
-	for _, c := range string(class[1 : len(class)-1]) {
-		switch {
-		case isAlphaNumeric(c):
-		case c == '_' || c == '-':
-		default:
-			return fmt.Errorf("invalid character '%c' in device class %q",
-				c, class)
-		}
-	}
-	if !isAlphaNumeric(rune(class[len(class)-1])) {
-		return fmt.Errorf("invalid class %q, should end with a letter or digit", class)
-	}
-	return nil
+	return parser.ValidateClassName(class)
 }
 
 // ValidateDeviceName checks the validity of a device name.
@@ -167,39 +106,8 @@ func ValidateClassName(class string) error {
 //   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
 //   - digits ('0'-'9')
 //   - underscore, dash, dot, colon ('_', '-', '.', ':')
+//
+// Deprecated: use parser.ValidateDeviceName instead
 func ValidateDeviceName(name string) error {
-	if name == "" {
-		return fmt.Errorf("invalid (empty) device name")
-	}
-	if !isAlphaNumeric(rune(name[0])) {
-		return fmt.Errorf("invalid class %q, should start with a letter or digit", name)
-	}
-	if len(name) == 1 {
-		return nil
-	}
-	for _, c := range string(name[1 : len(name)-1]) {
-		switch {
-		case isAlphaNumeric(c):
-		case c == '_' || c == '-' || c == '.' || c == ':':
-		default:
-			return fmt.Errorf("invalid character '%c' in device name %q",
-				c, name)
-		}
-	}
-	if !isAlphaNumeric(rune(name[len(name)-1])) {
-		return fmt.Errorf("invalid name %q, should end with a letter or digit", name)
-	}
-	return nil
-}
-
-func isLetter(c rune) bool {
-	return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z')
-}
-
-func isDigit(c rune) bool {
-	return '0' <= c && c <= '9'
-}
-
-func isAlphaNumeric(c rune) bool {
-	return isLetter(c) || isDigit(c)
+	return parser.ValidateDeviceName(name)
 }

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/registry.go

@@ -23,14 +23,12 @@ import (
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 )
 
-//
 // Registry keeps a cache of all CDI Specs installed or generated on
 // the host. Registry is the primary interface clients should use to
 // interact with CDI.
 //
 // The most commonly used Registry functions are for refreshing the
 // registry and injecting CDI devices into an OCI Spec.
-//
 type Registry interface {
 	RegistryResolver
 	RegistryRefresher

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go

@@ -28,6 +28,7 @@ import (
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 	"sigs.k8s.io/yaml"
 
+	"github.com/container-orchestrated-devices/container-device-interface/internal/validation"
 	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
 )
 
@@ -131,6 +132,7 @@ func (s *Spec) write(overwrite bool) error {
 
 	if filepath.Ext(s.path) == ".yaml" {
 		data, err = yaml.Marshal(s.Spec)
+		data = append([]byte("---\n"), data...)
 	} else {
 		data, err = json.Marshal(s.Spec)
 	}
@@ -207,7 +209,7 @@ func (s *Spec) validate() (map[string]*Device, error) {
 
 	minVersion, err := MinimumRequiredVersion(s.Spec)
 	if err != nil {
-		return nil, fmt.Errorf("could not determine minumum required version: %v", err)
+		return nil, fmt.Errorf("could not determine minimum required version: %v", err)
 	}
 	if newVersion(minVersion).IsGreaterThan(newVersion(s.Version)) {
 		return nil, fmt.Errorf("the spec version must be at least v%v", minVersion)
@@ -219,6 +221,9 @@ func (s *Spec) validate() (map[string]*Device, error) {
 	if err := ValidateClassName(s.class); err != nil {
 		return nil, err
 	}
+	if err := validation.ValidateSpecAnnotations(s.Kind, s.Annotations); err != nil {
+		return nil, err
+	}
 	if err := s.edits().Validate(); err != nil {
 		return nil, err
 	}
@@ -306,7 +311,7 @@ func GenerateSpecName(vendor, class string) string {
 // match the vendor and class of the CDI Spec. transientID should be
 // unique among all CDI users on the same host that might generate
 // transient Spec files using the same vendor/class combination. If
-// the external entity to which the lifecycle of the tranient Spec
+// the external entity to which the lifecycle of the transient Spec
 // is tied to has a unique ID of its own, then this is usually a
 // good choice for transientID.
 //

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/version.go

@@ -21,6 +21,7 @@ import (
 
 	"golang.org/x/mod/semver"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
 	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
 )
 
@@ -37,6 +38,7 @@ const (
 	v030 version = "v0.3.0"
 	v040 version = "v0.4.0"
 	v050 version = "v0.5.0"
+	v060 version = "v0.6.0"
 
 	// vEarliest is the earliest supported version of the CDI specification
 	vEarliest version = v030
@@ -51,9 +53,10 @@ var validSpecVersions = requiredVersionMap{
 	v030: nil,
 	v040: requiresV040,
 	v050: requiresV050,
+	v060: requiresV060,
 }
 
-// MinimumRequiredVersion determines the minumum spec version for the input spec.
+// MinimumRequiredVersion determines the minimum spec version for the input spec.
 func MinimumRequiredVersion(spec *cdi.Spec) (string, error) {
 	minVersion := validSpecVersions.requiredVersion(spec)
 	return minVersion.String(), nil
@@ -115,13 +118,38 @@ func (r requiredVersionMap) requiredVersion(spec *cdi.Spec) version {
 	return minVersion
 }
 
+// requiresV060 returns true if the spec uses v0.6.0 features
+func requiresV060(spec *cdi.Spec) bool {
+	// The v0.6.0 spec allows annotations to be specified at a spec level
+	for range spec.Annotations {
+		return true
+	}
+
+	// The v0.6.0 spec allows annotations to be specified at a device level
+	for _, d := range spec.Devices {
+		for range d.Annotations {
+			return true
+		}
+	}
+
+	// The v0.6.0 spec allows dots "." in Kind name label (class)
+	vendor, class := parser.ParseQualifier(spec.Kind)
+	if vendor != "" {
+		if strings.ContainsRune(class, '.') {
+			return true
+		}
+	}
+
+	return false
+}
+
 // requiresV050 returns true if the spec uses v0.5.0 features
 func requiresV050(spec *cdi.Spec) bool {
 	var edits []*cdi.ContainerEdits
 
 	for _, d := range spec.Devices {
 		// The v0.5.0 spec allowed device names to start with a digit instead of requiring a letter
-		if len(d.Name) > 0 && !isLetter(rune(d.Name[0])) {
+		if len(d.Name) > 0 && !parser.IsLetter(rune(d.Name[0])) {
 			return true
 		}
 		edits = append(edits, &d.ContainerEdits)

vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/parser/parser.go

@@ -0,0 +1,212 @@
+/*
+   Copyright © The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package parser
+
+import (
+	"fmt"
+	"strings"
+)
+
+// QualifiedName returns the qualified name for a device.
+// The syntax for a qualified device names is
+//
+//	"<vendor>/<class>=<name>".
+//
+// A valid vendor and class name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'.
+//
+// A valid device name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':'
+func QualifiedName(vendor, class, name string) string {
+	return vendor + "/" + class + "=" + name
+}
+
+// IsQualifiedName tests if a device name is qualified.
+func IsQualifiedName(device string) bool {
+	_, _, _, err := ParseQualifiedName(device)
+	return err == nil
+}
+
+// ParseQualifiedName splits a qualified name into device vendor, class,
+// and name. If the device fails to parse as a qualified name, or if any
+// of the split components fail to pass syntax validation, vendor and
+// class are returned as empty, together with the verbatim input as the
+// name and an error describing the reason for failure.
+func ParseQualifiedName(device string) (string, string, string, error) {
+	vendor, class, name := ParseDevice(device)
+
+	if vendor == "" {
+		return "", "", device, fmt.Errorf("unqualified device %q, missing vendor", device)
+	}
+	if class == "" {
+		return "", "", device, fmt.Errorf("unqualified device %q, missing class", device)
+	}
+	if name == "" {
+		return "", "", device, fmt.Errorf("unqualified device %q, missing device name", device)
+	}
+
+	if err := ValidateVendorName(vendor); err != nil {
+		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
+	}
+	if err := ValidateClassName(class); err != nil {
+		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
+	}
+	if err := ValidateDeviceName(name); err != nil {
+		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
+	}
+
+	return vendor, class, name, nil
+}
+
+// ParseDevice tries to split a device name into vendor, class, and name.
+// If this fails, for instance in the case of unqualified device names,
+// ParseDevice returns an empty vendor and class together with name set
+// to the verbatim input.
+func ParseDevice(device string) (string, string, string) {
+	if device == "" || device[0] == '/' {
+		return "", "", device
+	}
+
+	parts := strings.SplitN(device, "=", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", device
+	}
+
+	name := parts[1]
+	vendor, class := ParseQualifier(parts[0])
+	if vendor == "" {
+		return "", "", device
+	}
+
+	return vendor, class, name
+}
+
+// ParseQualifier splits a device qualifier into vendor and class.
+// The syntax for a device qualifier is
+//
+//	"<vendor>/<class>"
+//
+// If parsing fails, an empty vendor and the class set to the
+// verbatim input is returned.
+func ParseQualifier(kind string) (string, string) {
+	parts := strings.SplitN(kind, "/", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", kind
+	}
+	return parts[0], parts[1]
+}
+
+// ValidateVendorName checks the validity of a vendor name.
+// A vendor name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+func ValidateVendorName(vendor string) error {
+	err := validateVendorOrClassName(vendor)
+	if err != nil {
+		err = fmt.Errorf("invalid vendor. %w", err)
+	}
+	return err
+}
+
+// ValidateClassName checks the validity of class name.
+// A class name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+func ValidateClassName(class string) error {
+	err := validateVendorOrClassName(class)
+	if err != nil {
+		err = fmt.Errorf("invalid class. %w", err)
+	}
+	return err
+}
+
+// validateVendorOrClassName checks the validity of vendor or class name.
+// A name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+func validateVendorOrClassName(name string) error {
+	if name == "" {
+		return fmt.Errorf("empty name")
+	}
+	if !IsLetter(rune(name[0])) {
+		return fmt.Errorf("%q, should start with letter", name)
+	}
+	for _, c := range string(name[1 : len(name)-1]) {
+		switch {
+		case IsAlphaNumeric(c):
+		case c == '_' || c == '-' || c == '.':
+		default:
+			return fmt.Errorf("invalid character '%c' in name %q",
+				c, name)
+		}
+	}
+	if !IsAlphaNumeric(rune(name[len(name)-1])) {
+		return fmt.Errorf("%q, should end with a letter or digit", name)
+	}
+
+	return nil
+}
+
+// ValidateDeviceName checks the validity of a device name.
+// A device name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, dot, colon ('_', '-', '.', ':')
+func ValidateDeviceName(name string) error {
+	if name == "" {
+		return fmt.Errorf("invalid (empty) device name")
+	}
+	if !IsAlphaNumeric(rune(name[0])) {
+		return fmt.Errorf("invalid class %q, should start with a letter or digit", name)
+	}
+	if len(name) == 1 {
+		return nil
+	}
+	for _, c := range string(name[1 : len(name)-1]) {
+		switch {
+		case IsAlphaNumeric(c):
+		case c == '_' || c == '-' || c == '.' || c == ':':
+		default:
+			return fmt.Errorf("invalid character '%c' in device name %q",
+				c, name)
+		}
+	}
+	if !IsAlphaNumeric(rune(name[len(name)-1])) {
+		return fmt.Errorf("invalid name %q, should end with a letter or digit", name)
+	}
+	return nil
+}
+
+// IsLetter reports whether the rune is a letter.
+func IsLetter(c rune) bool {
+	return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z')
+}
+
+// IsDigit reports whether the rune is a digit.
+func IsDigit(c rune) bool {
+	return '0' <= c && c <= '9'
+}
+
+// IsAlphaNumeric reports whether the rune is a letter or digit.
+func IsAlphaNumeric(c rune) bool {
+	return IsLetter(c) || IsDigit(c)
+}

vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go

@@ -3,21 +3,24 @@ package specs
 import "os"
 
 // CurrentVersion is the current version of the Spec.
-const CurrentVersion = "0.5.0"
+const CurrentVersion = "0.6.0"
 
 // Spec is the base configuration for CDI
 type Spec struct {
 	Version string `json:"cdiVersion"`
 	Kind    string `json:"kind"`
-
-	Devices        []Device       `json:"devices"`
-	ContainerEdits ContainerEdits `json:"containerEdits,omitempty"`
+	// Annotations add meta information per CDI spec. Note these are CDI-specific and do not affect container metadata.
+	Annotations    map[string]string `json:"annotations,omitempty"`
+	Devices        []Device          `json:"devices"`
+	ContainerEdits ContainerEdits    `json:"containerEdits,omitempty"`
 }
 
 // Device is a "Device" a container runtime can add to a container
 type Device struct {
-	Name           string         `json:"name"`
-	ContainerEdits ContainerEdits `json:"containerEdits"`
+	Name string `json:"name"`
+	// Annotations add meta information per device. Note these are CDI-specific and do not affect container metadata.
+	Annotations    map[string]string `json:"annotations,omitempty"`
+	ContainerEdits ContainerEdits    `json:"containerEdits"`
 }
 
 // ContainerEdits are edits a container runtime must make to the OCI spec to expose the device.

vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/oci.go

@@ -22,7 +22,7 @@ func ApplyOCIEditsForDevice(config *spec.Spec, cdi *Spec, dev string) error {
 	return fmt.Errorf("CDI: device %q not found for spec %q", dev, cdi.Kind)
 }
 
-// ApplyOCIEdits applies the OCI edits the CDI spec declares globablly
+// ApplyOCIEdits applies the OCI edits the CDI spec declares globally
 func ApplyOCIEdits(config *spec.Spec, cdi *Spec) error {
 	return ApplyEditsToOCISpec(config, &cdi.ContainerEdits)
 }

vendor/modules.txt

@@ -6,10 +6,13 @@ github.com/BurntSushi/toml/internal
 ## explicit; go 1.15
 github.com/NVIDIA/go-nvml/pkg/dl
 github.com/NVIDIA/go-nvml/pkg/nvml
-# github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
+# github.com/container-orchestrated-devices/container-device-interface v0.6.0
 ## explicit; go 1.17
 github.com/container-orchestrated-devices/container-device-interface/internal/multierror
+github.com/container-orchestrated-devices/container-device-interface/internal/validation
+github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s
 github.com/container-orchestrated-devices/container-device-interface/pkg/cdi
+github.com/container-orchestrated-devices/container-device-interface/pkg/parser
 github.com/container-orchestrated-devices/container-device-interface/specs-go
 # github.com/cpuguy83/go-md2man/v2 v2.0.2
 ## explicit; go 1.11

versions.mk

@@ -14,7 +14,7 @@
 
 LIB_NAME := nvidia-container-toolkit
 LIB_VERSION := 1.14.0
-LIB_TAG := rc.1
+LIB_TAG := rc.2
 
 # The package version is the combination of the library version and tag.
 # If the tag is specified the two components are joined with a tilde (~).
@@ -30,8 +30,8 @@ NVIDIA_CONTAINER_RUNTIME_VERSION := 3.14.0
 # Specify the expected libnvidia-container0 version for arm64-based ubuntu builds.
 LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack
 
-CUDA_VERSION := 12.1.1
-GOLANG_VERSION := 1.20.3
+CUDA_VERSION := 12.2.0
+GOLANG_VERSION := 1.20.5
 
 GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "")
 GIT_COMMIT_SHORT ?= $(shell git rev-parse --short HEAD 2> /dev/null || echo "")