Merge branch 'prep-release' into 'main'

Update changelog and libnvidia-container for release

See merge request nvidia/container-toolkit/container-toolkit!152

.common-ci.yml

@@ -190,7 +190,7 @@ test-packaging:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
 
 # Define an external release step that pushes an image to an external repository.
-# This includes a devlopment image off master.
+# This includes a devlopment image off main.
 .release:external:
   extends:
     - .release

.gitlab-ci.yml

@@ -96,7 +96,7 @@ unit-tests:
   stage: package-build
   timeout: 2h 30m
   script:
-    - ./scripts/release.sh ${DIST}-${ARCH}
+    - ./scripts/build-packages.sh ${DIST}-${ARCH}
 
   artifacts:
     name: ${ARTIFACTS_NAME}

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
 	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
+	branch = main
 [submodule "third_party/nvidia-container-runtime"]
 	path = third_party/nvidia-container-runtime
 	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git

.nvidia-ci.yml

@@ -27,8 +27,8 @@ default:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_TLS_CERTDIR: "/certs"
-  # Release "devel"-tagged images off the master branch
-  RELEASE_DEVEL_BRANCH: "master"
+  # Release "devel"-tagged images off the main branch
+  RELEASE_DEVEL_BRANCH: "main"
   DEVEL_RELEASE_IMAGE_VERSION: "devel"
   # On the multi-arch builder we don't need the qemu setup.
   SKIP_QEMU_SETUP: "1"
@@ -232,15 +232,6 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
-.release:dockerhub:
-  extends:
-    - .release:external
-  variables:
-    OUT_REGISTRY_USER: "${REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${DOCKERHUB_REGISTRY}"
-    OUT_IMAGE_NAME: "${REGISTRY_IMAGE}"
-
 release:staging-ubuntu18.04:
   extends:
     - .release:staging
@@ -281,29 +272,3 @@ release:ngc-ubi8:
   extends:
     - .release:ngc
     - .dist-ubi8
-
-# Release to Dockerhub
-release:dockerhub-centos7:
-  extends:
-    - .release:dockerhub
-    - .dist-centos7
-
-release:dockerhub-centos8:
-  extends:
-    - .release:dockerhub
-    - .dist-centos8
-
-release:dockerhub-ubuntu18.04:
-  extends:
-    - .release:dockerhub
-    - .dist-ubuntu18.04
-
-release:dockerhub-ubuntu20.04:
-  extends:
-    - .release:dockerhub
-    - .dist-ubuntu20.04
-
-release:dockerhub-ubi8:
-  extends:
-    - .release:dockerhub
-    - .dist-ubi8

DEVELOPMENT.md

@@ -13,7 +13,7 @@ The `nvidia-container-toolkit` resides in this repo directly.
 
 In oder to build the packages, the following command is executed
 ```sh
-./scripts/build-all-components.sh TARGET
+./scripts/build-packages.sh TARGET
 ```
 where `TARGET` is a make target that is valid for each of the sub-components.
 
@@ -21,6 +21,8 @@ These include:
 * `ubuntu18.04-amd64`
 * `centos8-x86_64`
 
+If no `TARGET` is specified, all valid release targets are built.
+
 The packages are generated in the `dist` folder.
 
 ## Testing local changes
@@ -37,9 +39,23 @@ The [test/release](./test/release/) folder contains documentation on how the ins
 
 ## Releasing
 
-A utility script [`scripts/release.sh`](./scripts/release.sh) is provided to build
-packages required for release. If run without arguments, all supported distribution-architecture combinations are built. A specific distribution-architecture pair can also be provided
-```sh
-./scripts/release.sh ubuntu18.04-amd64
+In order to release packages required for a release, a utility script
+[`scripts/release-packages.sh`](./scripts/release-packages.sh) is provided.
+This script can be executed as follows:
+
+```bash
+GPG_LOCAL_USER="GPG_USER" \
+MASTER_KEY_PATH=/path/to/gpg-master.key \
+SUB_KEY_PATH=/path/to/gpg-subkey.key \
+    ./scripts/release-packages.sh REPO PACKAGE_REPO_ROOT [REFERENCE]
 ```
-where the `amd64` builds for `ubuntu18.04` are provided as an example.
+
+Where `REPO` is one of `stable` or `experimental`, `PACKAGE_REPO_ROOT` is the local path to the `libnvidia-container` repository checked out to the `gh-pages` branch, and `REFERENCE` is the git SHA that is to be released. If reference is not specified `HEAD` is assumed.
+
+This scripts performs the following basic functions:
+* Pulls the package image defined by the `REFERENCE` git SHA from the staging registry,
+* Copies the required packages to the package repository at `PACKAGE_REPO_ROOT/REPO`,
+* Signs the packages using the specified GPG keys
+
+While the last two are performed, commits are added to the package repository. These can be pushed to the relevant repository.
+

Makefile

@@ -38,8 +38,6 @@ EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES))
 CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
-$(info CMD_TARGETS=$(CMD_TARGETS))
-
 CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
 MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
 
@@ -48,6 +46,12 @@ TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 DOCKER_TARGETS := $(patsubst %,docker-%, $(TARGETS))
 .PHONY: $(TARGETS) $(DOCKER_TARGETS)
 
+ifeq ($(VERSION),)
+CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+else
+CLI_VERSION = $(VERSION)
+endif
+
 GOOS ?= linux
 
 binaries: cmds
@@ -56,7 +60,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-s -w -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.gitCommit=$(GIT_COMMIT) -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...

README.md

@@ -1,6 +1,6 @@
 # NVIDIA Container Toolkit
 
-[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/master/LICENSE)
+[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/main/LICENSE)
 [![Documentation](https://img.shields.io/badge/documentation-wiki-blue.svg?style=flat-square)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html)
 [![Package repository](https://img.shields.io/badge/packages-repository-b956e8.svg?style=flat-square)](https://nvidia.github.io/libnvidia-container)
 

build/container/Dockerfile.ubuntu

@@ -42,6 +42,9 @@ RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" .
 
 FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
+# Remove the CUDA repository configurations to avoid issues with rotated GPG keys
+RUN rm -f /etc/apt/sources.list.d/cuda.list
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libcap2 \

cmd/nvidia-container-runtime/logger.go

@@ -20,60 +20,220 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
 
 	"github.com/sirupsen/logrus"
-	"github.com/tsaikd/KDGoLib/logrusutil"
 )
 
 // Logger adds a way to manage output to a log file to a logrus.Logger
 type Logger struct {
 	*logrus.Logger
-	previousOutput io.Writer
-	logFile        *os.File
+	previousLogger *logrus.Logger
+	logFiles       []*os.File
 }
 
-// NewLogger constructs a Logger with a preddefined formatter
+// NewLogger creates an empty logger
 func NewLogger() *Logger {
-	logrusLogger := logrus.New()
-
-	formatter := &logrusutil.ConsoleLogFormatter{
-		TimestampFormat: "2006/01/02 15:04:07",
-		Flag:            logrusutil.Ltime,
+	return &Logger{
+		Logger: logrus.New(),
 	}
-
-	logger := &Logger{
-		Logger: logrusLogger,
-	}
-	logger.SetFormatter(formatter)
-
-	return logger
 }
 
-// LogToFile opens the specified file for appending and sets the logger to
-// output to the opened file. A reference to the file pointer is stored to
-// allow this to be closed.
-func (l *Logger) LogToFile(filename string) error {
-	logFile, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return fmt.Errorf("error opening debug log file: %v", err)
+// UpdateLogger constructs a Logger with a preddefined formatter
+func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, error) {
+	configFromArgs := parseArgs(argv)
+
+	level, logLevelError := configFromArgs.getLevel(logLevel)
+
+	var logFiles []*os.File
+	var argLogFileError error
+
+	// We don't create log files if the version argument is supplied
+	if !configFromArgs.version {
+		configLogFile, err := createLogFile(filename)
+		if err != nil {
+			return logger, fmt.Errorf("error opening debug log file: %v", err)
+		}
+		if configLogFile != nil {
+			logFiles = append(logFiles, configLogFile)
+		}
+
+		argLogFile, err := createLogFile(configFromArgs.file)
+		if argLogFile != nil {
+			logFiles = append(logFiles, argLogFile)
+		}
+		argLogFileError = err
 	}
 
-	l.logFile = logFile
-	l.previousOutput = l.Out
-	l.SetOutput(logFile)
-
-	return nil
-}
-
-// CloseFile closes the log file (if any) and resets the logger output to what it
-// was before LogToFile was called.
-func (l *Logger) CloseFile() error {
-	if l.logFile == nil {
-		return nil
+	l := &Logger{
+		Logger:         logrus.New(),
+		previousLogger: logger.Logger,
+		logFiles:       logFiles,
 	}
-	logFile := l.logFile
-	l.SetOutput(l.previousOutput)
-	l.logFile = nil
 
-	return logFile.Close()
+	l.SetLevel(level)
+	if level == logrus.DebugLevel {
+		logrus.SetReportCaller(true)
+		// Shorten function and file names reported by the logger, by
+		// trimming common "github.com/opencontainers/runc" prefix.
+		// This is only done for text formatter.
+		_, file, _, _ := runtime.Caller(0)
+		prefix := filepath.Dir(file) + "/"
+		logrus.SetFormatter(&logrus.TextFormatter{
+			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+				function := strings.TrimPrefix(f.Function, prefix) + "()"
+				fileLine := strings.TrimPrefix(f.File, prefix) + ":" + strconv.Itoa(f.Line)
+				return function, fileLine
+			},
+		})
+	}
+
+	if configFromArgs.format == "json" {
+		l.SetFormatter(new(logrus.JSONFormatter))
+	}
+
+	if len(logFiles) == 0 {
+		l.SetOutput(io.Discard)
+	} else if len(logFiles) == 1 {
+		l.SetOutput(logFiles[0])
+	} else if len(logFiles) > 1 {
+		var writers []io.Writer
+		for _, f := range logFiles {
+			writers = append(writers, f)
+		}
+		l.SetOutput(io.MultiWriter(writers...))
+	}
+
+	if logLevelError != nil {
+		l.Warn(logLevelError)
+	}
+
+	if argLogFileError != nil {
+		l.Warnf("Failed to open log file: %v", argLogFileError)
+	}
+
+	return l, nil
+}
+
+// Reset closes the log file (if any) and resets the logger output to what it
+// was before UpdateLogger was called.
+func (l *Logger) Reset() error {
+	defer func() {
+		previous := l.previousLogger
+		if previous == nil {
+			previous = logrus.New()
+		}
+		logger = &Logger{Logger: previous}
+	}()
+
+	var errs []error
+	for _, f := range l.logFiles {
+		err := f.Close()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	var err error
+	for _, e := range errs {
+		if err == nil {
+			err = e
+			continue
+		}
+		return fmt.Errorf("%v; %w", e, err)
+	}
+
+	return err
+}
+
+func createLogFile(filename string) (*os.File, error) {
+	if filename != "" && filename != os.DevNull {
+		return os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	}
+
+	return nil, nil
+}
+
+type loggerConfig struct {
+	file    string
+	format  string
+	debug   bool
+	version bool
+}
+
+func (c loggerConfig) getLevel(logLevel string) (logrus.Level, error) {
+	if c.debug {
+		return logrus.DebugLevel, nil
+	}
+
+	if logLevel, err := logrus.ParseLevel(logLevel); err == nil {
+		return logLevel, nil
+	}
+
+	return logrus.InfoLevel, fmt.Errorf("invalid log-level '%v'", logLevel)
+}
+
+// Informed by Taken from https://github.com/opencontainers/runc/blob/7fd8b57001f5bfa102e89cb434d96bf71f7c1d35/main.go#L182
+func parseArgs(args []string) loggerConfig {
+	c := loggerConfig{}
+
+	expected := map[string]*string{
+		"log-format": &c.format,
+		"log":        &c.file,
+	}
+
+	found := make(map[string]bool)
+
+	for i := 0; i < len(args); i++ {
+		if len(found) == 4 {
+			break
+		}
+
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		trimmed := strings.TrimLeft(parts[0], "-")
+		// If this is not a flag we continue
+		if parts[0] == trimmed {
+			continue
+		}
+
+		// Check the version flag
+		if trimmed == "version" {
+			c.version = true
+			found["version"] = true
+			// For the version flag we don't process any other flags
+			continue
+		}
+
+		// Check the debug flag
+		if trimmed == "debug" {
+			c.debug = true
+			found["debug"] = true
+			continue
+		}
+
+		destination, exists := expected[trimmed]
+		if !exists {
+			continue
+		}
+
+		var value string
+		if len(parts) == 2 {
+			value = parts[2]
+		} else if i+1 < len(args) {
+			value = args[i+1]
+			i++
+		} else {
+			continue
+		}
+
+		*destination = value
+		found[trimmed] = true
+	}
+
+	return c
 }

cmd/nvidia-container-runtime/main.go

@@ -3,11 +3,20 @@ package main
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
+// version must be set by go build's -X main.version= option in the Makefile.
+var version = "unknown"
+
+// gitCommit will be the hash that the binary was built from
+// and will be populated by the Makefile
+var gitCommit = ""
+
 var logger = NewLogger()
 
 func main() {
@@ -21,34 +30,55 @@ func main() {
 // run is an entry point that allows for idiomatic handling of errors
 // when calling from the main function.
 func run(argv []string) (rerr error) {
-	logger.Debugf("Running %v", argv)
+	printVersion := hasVersionFlag(argv)
+	if printVersion {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
+	}
+
 	cfg, err := config.GetConfig()
 	if err != nil {
 		return fmt.Errorf("error loading config: %v", err)
 	}
 
-	err = logger.LogToFile(cfg.NVIDIAContainerRuntimeConfig.DebugFilePath)
+	logger, err = UpdateLogger(
+		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
+		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
+		argv,
+	)
 	if err != nil {
-		return fmt.Errorf("error opening debug log file: %v", err)
-	}
-	defer func() {
-		// We capture and log a returning error before closing the log file.
-		if rerr != nil {
-			logger.Errorf("%v", rerr)
-		}
-		logger.CloseFile()
-	}()
-
-	if logLevel, err := logrus.ParseLevel(cfg.NVIDIAContainerRuntimeConfig.LogLevel); err == nil {
-		logger.SetLevel(logLevel)
-	} else {
-		logger.Warnf("Invalid log-level '%v'; using '%v'", cfg.NVIDIAContainerRuntimeConfig.LogLevel, logger.Level.String())
+		return fmt.Errorf("failed to set up logger: %v", err)
 	}
+	defer logger.Reset()
 
+	logger.Debugf("Command line arguments: %v", argv)
 	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
 	if err != nil {
 		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
 	}
 
+	if printVersion {
+		fmt.Print("\n")
+	}
 	return runtime.Exec(argv)
 }
+
+// TODO: This should be refactored / combined with parseArgs in logger.
+func hasVersionFlag(args []string) bool {
+	for i := 0; i < len(args); i++ {
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		trimmed := strings.TrimLeft(parts[0], "-")
+		// If this is not a flag we continue
+		if parts[0] == trimmed {
+			continue
+		}
+
+		// Check the version flag
+		if trimmed == "version" {
+			return true
+		}
+	}
+
+	return false
+}

cmd/nvidia-container-runtime/main_test.go

@@ -24,6 +24,10 @@ const (
 	unmodifiedSpecFileSuffix = "test/input/test_spec.json"
 )
 
+const (
+	runcExecutableName = "runc"
+)
+
 type testConfig struct {
 	root    string
 	binPath string
@@ -80,11 +84,6 @@ func TestBadInput(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle")
-	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
-	output, err := cmdRun.CombinedOutput()
-	require.Errorf(t, err, "runtime should return an error", "output=%v", string(output))
-
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	err = cmdCreate.Run()

cmd/nvidia-container-runtime/modifier/csv.go

@@ -0,0 +1,166 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/cuda"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// csvMode represents the modifications as performed by the csv runtime mode
+type csvMode struct {
+	logger     *logrus.Logger
+	discoverer discover.Discover
+}
+
+const (
+	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
+	visibleDevicesVoid   = "void"
+
+	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
+)
+
+// NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
+// The modifications are defined by CSV MountSpecs.
+func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	rawSpec, err := ociSpec.Load()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
+	// if no modification is required.
+	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
+	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
+		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
+		return nil, nil
+	}
+	logger.Infof("Constructing modifier from config: %+v", *cfg)
+
+	config := &discover.Config{
+		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
+		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
+	}
+
+	// TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the
+	// visible devices are checked.
+	image, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := checkRequirements(logger, &image); err != nil {
+		return nil, fmt.Errorf("requirements not met: %v", err)
+	}
+
+	csvFiles, err := csv.GetFileList(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.MountSpecPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
+	}
+
+	nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
+	if nvidiaRequireJetpack != "csv-mounts=all" {
+		csvFiles = csv.BaseFilesOnly(csvFiles)
+	}
+
+	csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
+	}
+
+	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
+	}
+
+	createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
+	}
+
+	d := discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
+
+	return newModifierFromDiscoverer(logger, d)
+}
+
+// newModifierFromDiscoverer created a modifier that aplies the discovered
+// modifications to an OCI spec if require by the runtime wrapper.
+func newModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	m := csvMode{
+		logger:     logger,
+		discoverer: d,
+	}
+	return &m, nil
+}
+
+// Modify applies the required modifications to the incomming OCI spec. These modifications
+// are applied in-place.
+func (m csvMode) Modify(spec *specs.Spec) error {
+	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
+	if err != nil {
+		return fmt.Errorf("failed to remove existing hooks: %v", err)
+	}
+
+	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
+	if err != nil {
+		return fmt.Errorf("failed to get required container edits: %v", err)
+	}
+
+	return specEdits.Modify(spec)
+}
+
+func checkRequirements(logger *logrus.Logger, image *image.CUDA) error {
+	if image.HasDisableRequire() {
+		// TODO: We could print the real value here instead
+		logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true)
+		return nil
+	}
+
+	imageRequirements, err := image.GetRequirements()
+	if err != nil {
+		//  TODO: Should we treat this as a failure, or just issue a warning?
+		return fmt.Errorf("failed to get image requirements: %v", err)
+	}
+
+	r := requirements.New(logger, imageRequirements)
+
+	cudaVersion, err := cuda.Version()
+	if err != nil {
+		logger.Warnf("Failed to get CUDA version: %v", err)
+	} else {
+		r.AddVersionProperty(requirements.CUDA, cudaVersion)
+	}
+
+	compteCapability, err := cuda.ComputeCapability(0)
+	if err != nil {
+		logger.Warnf("Failed to get CUDA Compute Capability: %v", err)
+	} else {
+		r.AddVersionProperty(requirements.ARCH, compteCapability)
+	}
+
+	return r.Assert()
+}

cmd/nvidia-container-runtime/modifier/csv_test.go

@@ -28,7 +28,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestNewExperimentalModifier(t *testing.T) {
+func TestNewCSVModifier(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
 	testCases := []struct {
@@ -42,8 +42,8 @@ func TestNewExperimentalModifier(t *testing.T) {
 		{
 			description: "spec load error returns error",
 			spec: &oci.SpecMock{
-				LoadFunc: func() error {
-					return fmt.Errorf("load failed")
+				LoadFunc: func() (*specs.Spec, error) {
+					return nil, fmt.Errorf("load failed")
 				},
 			},
 			expectedError: fmt.Errorf("load failed"),
@@ -63,42 +63,6 @@ func TestNewExperimentalModifier(t *testing.T) {
 			visibleDevices: "void",
 			expectedNil:    true,
 		},
-		{
-			description: "empty config raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
-			},
-			visibleDevices: "all",
-			expectedError:  fmt.Errorf("invalid discover mode"),
-		},
-		{
-			description: "non-legacy discover mode raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					DiscoverMode: "non-legacy",
-				},
-			},
-			visibleDevices: "all",
-			expectedError:  fmt.Errorf("invalid discover mode"),
-		},
-		{
-			description: "legacy discover mode returns modifier",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					DiscoverMode: "legacy",
-				},
-			},
-			visibleDevices: "all",
-		},
-		{
-			description: "csv discover mode returns modifier",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					DiscoverMode: "csv",
-				},
-			},
-			visibleDevices: "all",
-		},
 	}
 
 	for _, tc := range testCases {
@@ -115,7 +79,7 @@ func TestNewExperimentalModifier(t *testing.T) {
 				}
 			}
 
-			m, err := NewExperimentalModifier(logger, tc.cfg, spec)
+			m, err := NewCSVModifier(logger, tc.cfg, spec)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {
@@ -304,7 +268,7 @@ func TestExperimentalModifier(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			m, err := newExperimentalModifierFromDiscoverer(logger, tc.discover)
+			m, err := newModifierFromDiscoverer(logger, tc.discover)
 			require.NoError(t, err)
 
 			err = m.Modify(tc.spec)
@@ -318,32 +282,3 @@ func TestExperimentalModifier(t *testing.T) {
 		})
 	}
 }
-
-func TestResolveDiscoverMode(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description  string
-		mode         string
-		expectedMode string
-	}{
-		{
-			description:  "non-auto resolves to input",
-			mode:         "not-auto",
-			expectedMode: "not-auto",
-		},
-		// TODO: The following test is brittle in that it will break on Tegra-based systems.
-		// {
-		// 	description:  "auto resolves to legacy",
-		// 	mode:         "auto",
-		// 	expectedMode: "legacy",
-		// },
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			mode := resolveAutoDiscoverMode(logger, tc.mode)
-			require.EqualValues(t, tc.expectedMode, mode)
-		})
-	}
-}

cmd/nvidia-container-runtime/modifier/experimental.go

@@ -1,178 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
-)
-
-// experiemental represents the modifications required by the experimental runtime
-type experimental struct {
-	logger     *logrus.Logger
-	discoverer discover.Discover
-}
-
-const (
-	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
-	visibleDevicesVoid   = "void"
-
-	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
-)
-
-// NewExperimentalModifier creates a modifier that applies the experimental
-// modications to an OCI spec if required by the runtime wrapper.
-func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	if err := ociSpec.Load(); err != nil {
-		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
-	// if no modification is required.
-	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
-	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
-		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
-		return nil, nil
-	}
-	logger.Infof("Constructing modifier from config: %+v", cfg)
-
-	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
-	}
-
-	var d discover.Discover
-
-	switch resolveAutoDiscoverMode(logger, cfg.NVIDIAContainerRuntimeConfig.DiscoverMode) {
-	case "legacy":
-		legacyDiscoverer, err := discover.NewLegacyDiscoverer(logger, config)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create legacy discoverer: %v", err)
-		}
-		d = legacyDiscoverer
-	case "csv":
-		csvFiles, err := csv.GetFileList(csv.DefaultMountSpecPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
-		}
-
-		nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
-		if nvidiaRequireJetpack != "csv-mounts=all" {
-			csvFiles = csv.BaseFilesOnly(csvFiles)
-		}
-
-		csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
-		}
-
-		ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
-		}
-
-		createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, config)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
-		}
-
-		d = discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
-	default:
-		return nil, fmt.Errorf("invalid discover mode: %v", cfg.NVIDIAContainerRuntimeConfig.DiscoverMode)
-	}
-
-	return newExperimentalModifierFromDiscoverer(logger, d)
-}
-
-// newExperimentalModifierFromDiscoverer created a modifier that aplies the discovered
-// modifications to an OCI spec if require by the runtime wrapper.
-func newExperimentalModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
-	m := experimental{
-		logger:     logger,
-		discoverer: d,
-	}
-	return &m, nil
-}
-
-// Modify applies the required modifications to the incomming OCI spec. These modifications
-// are applied in-place.
-func (m experimental) Modify(spec *specs.Spec) error {
-	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
-	if err != nil {
-		return fmt.Errorf("failed to remove existing hooks: %v", err)
-	}
-
-	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
-	if err != nil {
-		return fmt.Errorf("failed to get required container edits: %v", err)
-	}
-
-	return specEdits.Modify(spec)
-}
-
-// resolveAutoDiscoverMode determines the correct discover mode for the specified platform if set to "auto"
-func resolveAutoDiscoverMode(logger *logrus.Logger, mode string) (rmode string) {
-	if mode != "auto" {
-		return mode
-	}
-	defer func() {
-		logger.Infof("Auto-detected discover mode as '%v'", rmode)
-	}()
-
-	isTegra, reason := isTegraSystem()
-	logger.Debugf("Is Tegra-based system? %v: %v", isTegra, reason)
-
-	if isTegra {
-		return "csv"
-	}
-
-	return "legacy"
-}
-
-// isTegraSystem returns true if the system is detected as a Tegra-based system
-func isTegraSystem() (bool, string) {
-	const tegraReleaseFile = "/etc/nv_tegra_release"
-	const tegraFamilyFile = "/sys/devices/soc0/family"
-
-	if info, err := os.Stat(tegraReleaseFile); err == nil && !info.IsDir() {
-		return true, fmt.Sprintf("%v found", tegraReleaseFile)
-	}
-
-	if info, err := os.Stat(tegraFamilyFile); err != nil || !info.IsDir() {
-		return false, fmt.Sprintf("%v not found", tegraFamilyFile)
-	}
-
-	contents, err := os.ReadFile(tegraFamilyFile)
-	if err != nil {
-		return false, fmt.Sprintf("could not read %v", tegraFamilyFile)
-	}
-
-	if strings.HasPrefix(strings.ToLower(string(contents)), "tegra") {
-		return true, fmt.Sprintf("%v has 'tegra' prefix", tegraFamilyFile)
-	}
-
-	return false, fmt.Sprintf("%v has no 'tegra' prefix", tegraFamilyFile)
-}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -21,30 +21,30 @@ import (
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 	"github.com/sirupsen/logrus"
 )
 
-const (
-	dockerRuncExecutableName = "docker-runc"
-	runcExecutableName       = "runc"
-)
-
 // newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger
 func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) {
+	lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, cfg.NVIDIAContainerRuntimeConfig.Runtimes)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
+	}
+
+	if !oci.HasCreateSubcommand(argv) {
+		logger.Debugf("Skipping modifier for non-create subcommand")
+		return lowLevelRuntime, nil
+	}
+
 	ociSpec, err := oci.NewSpec(logger, argv)
 	if err != nil {
 		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
 	}
 
-	lowLevelRuntimeCandidates := []string{dockerRuncExecutableName, runcExecutableName}
-	lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, lowLevelRuntimeCandidates)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
-	}
-
-	specModifier, err := newSpecModifier(logger, cfg, ociSpec)
+	specModifier, err := newSpecModifier(logger, cfg, ociSpec, argv)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
 	}
@@ -61,10 +61,13 @@ func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv [
 }
 
 // newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
-func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	if !cfg.NVIDIAContainerRuntimeConfig.Experimental {
+func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
+	switch info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode) {
+	case "legacy":
 		return modifier.NewStableRuntimeModifier(logger), nil
+	case "csv":
+		return modifier.NewCSVModifier(logger, cfg, ociSpec)
 	}
 
-	return modifier.NewExperimentalModifier(logger, cfg, ociSpec)
+	return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)
 }

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -38,17 +38,27 @@ func TestFactoryMethod(t *testing.T) {
 		expectedError bool
 	}{
 		{
-			description: "empty config no error",
+			description: "empty config raises error",
 			cfg: &config.Config{
 				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
 			},
+			expectedError: true,
 		},
 		{
-			description: "experimental flag supported",
+			description: "config with runtime raises no error",
 			cfg: &config.Config{
 				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Experimental: true,
-					DiscoverMode: "legacy",
+					Runtimes: []string{"runc"},
+					Mode:     "legacy",
+				},
+			},
+		},
+		{
+			description: "csv mode is supported",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "csv",
 				},
 			},
 			spec: &specs.Spec{
@@ -59,6 +69,43 @@ func TestFactoryMethod(t *testing.T) {
 				},
 			},
 		},
+		{
+			description: "non-legacy discover mode raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "non-legacy",
+				},
+			},
+			expectedError: true,
+		},
+		{
+			description: "legacy discover mode returns modifier",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "legacy",
+				},
+			},
+		},
+		{
+			description: "csv discover mode returns modifier",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "csv",
+				},
+			},
+		},
+		{
+			description: "empty mode raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+				},
+			},
+			expectedError: true,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -69,7 +116,7 @@ func TestFactoryMethod(t *testing.T) {
 			require.NoError(t, err)
 			require.NoError(t, json.NewEncoder(specFile).Encode(tc.spec))
 
-			argv := []string{"--bundle", bundleDir}
+			argv := []string{"--bundle", bundleDir, "create"}
 
 			_, err = newNVIDIAContainerRuntime(logger, tc.cfg, argv)
 			if tc.expectedError {

cmd/nvidia-container-toolkit/container_config.go

@@ -7,9 +7,9 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strconv"
 	"strings"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"golang.org/x/mod/semver"
 )
 
@@ -104,32 +104,6 @@ type HookState struct {
 	BundlePath string `json:"bundlePath"`
 }
 
-func parseCudaVersion(cudaVersion string) (vmaj, vmin, vpatch uint32) {
-	if _, err := fmt.Sscanf(cudaVersion, "%d.%d.%d\n", &vmaj, &vmin, &vpatch); err != nil {
-		vpatch = 0
-		if _, err := fmt.Sscanf(cudaVersion, "%d.%d\n", &vmaj, &vmin); err != nil {
-			vmin = 0
-			if _, err := fmt.Sscanf(cudaVersion, "%d\n", &vmaj); err != nil {
-				log.Panicln("invalid CUDA version:", cudaVersion)
-			}
-		}
-	}
-
-	return
-}
-
-func getEnvMap(e []string) (m map[string]string) {
-	m = make(map[string]string)
-	for _, s := range e {
-		p := strings.SplitN(s, "=", 2)
-		if len(p) != 2 {
-			log.Panicln("environment error")
-		}
-		m[p[0]] = p[1]
-	}
-	return
-}
-
 func loadSpec(path string) (spec *Spec) {
 	f, err := os.Open(path)
 	if err != nil {
@@ -191,12 +165,6 @@ func isPrivileged(s *Spec) bool {
 	return false
 }
 
-func isLegacyCUDAImage(env map[string]string) bool {
-	legacyCudaVersion := env[envCUDAVersion]
-	cudaRequire := env[envNVRequireCUDA]
-	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
-}
-
 func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
 	// Build a list of envvars to consider.
 	envVars := []string{envNVVisibleDevices}
@@ -335,27 +303,11 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr
 	return capabilities
 }
 
-func getRequirements(env map[string]string, legacyImage bool) []string {
-	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
-	var requirements []string
-	for name, value := range env {
-		if strings.HasPrefix(name, envNVRequirePrefix) {
-			requirements = append(requirements, value)
-		}
-	}
-	if legacyImage {
-		vmaj, vmin, _ := parseCudaVersion(env[envCUDAVersion])
-		cudaRequire := fmt.Sprintf("cuda>=%d.%d", vmaj, vmin)
-		requirements = append(requirements, cudaRequire)
-	}
-	return requirements
-}
-
-func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool) *nvidiaConfig {
-	legacyImage := isLegacyCUDAImage(env)
+func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
+	legacyImage := image.IsLegacy()
 
 	var devices string
-	if d := getDevices(hookConfig, env, mounts, privileged, legacyImage); d != nil {
+	if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil {
 		devices = *d
 	} else {
 		// 'nil' devices means this is not a GPU container.
@@ -363,7 +315,7 @@ func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mou
 	}
 
 	var migConfigDevices string
-	if d := getMigConfigDevices(env); d != nil {
+	if d := getMigConfigDevices(image); d != nil {
 		migConfigDevices = *d
 	}
 	if !privileged && migConfigDevices != "" {
@@ -371,19 +323,21 @@ func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mou
 	}
 
 	var migMonitorDevices string
-	if d := getMigMonitorDevices(env); d != nil {
+	if d := getMigMonitorDevices(image); d != nil {
 		migMonitorDevices = *d
 	}
 	if !privileged && migMonitorDevices != "" {
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	driverCapabilities := getDriverCapabilities(env, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
 
-	requirements := getRequirements(env, legacyImage)
+	requirements, err := image.GetRequirements()
+	if err != nil {
+		log.Panicln("failed to get requirements", err)
+	}
 
-	// Don't fail on invalid values.
-	disableRequire, _ := strconv.ParseBool(env[envNVDisableRequire])
+	disableRequire := image.HasDisableRequire()
 
 	return &nvidiaConfig{
 		Devices:            devices,
@@ -409,13 +363,17 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	env := getEnvMap(s.Process.Env)
+	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	if err != nil {
+		log.Panicln(err)
+	}
+
 	privileged := isPrivileged(s)
 	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    env,
-		Nvidia: getNvidiaConfig(&hook, env, s.Mounts, privileged),
+		Env:    image,
+		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
 	}
 }

cmd/nvidia-container-toolkit/hook_test.go

@@ -7,51 +7,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestParseCudaVersionValid(t *testing.T) {
-	var tests = []struct {
-		version  string
-		expected [3]uint32
-	}{
-		{"0", [3]uint32{0, 0, 0}},
-		{"8", [3]uint32{8, 0, 0}},
-		{"7.5", [3]uint32{7, 5, 0}},
-		{"9.0.116", [3]uint32{9, 0, 116}},
-		{"4294967295.4294967295.4294967295", [3]uint32{4294967295, 4294967295, 4294967295}},
-	}
-	for i, c := range tests {
-		vmaj, vmin, vpatch := parseCudaVersion(c.version)
-
-		version := [3]uint32{vmaj, vmin, vpatch}
-
-		require.Equal(t, c.expected, version, "%d: %v", i, c)
-	}
-}
-
-func TestParseCudaVersionInvalid(t *testing.T) {
-	var tests = []string{
-		"foo",
-		"foo.5.10",
-		"9.0.116.50",
-		"9.0.116foo",
-		"7.foo",
-		"9.0.bar",
-		"9.4294967296",
-		"9.0.116.",
-		"9..0",
-		"9.",
-		".5.10",
-		"-9",
-		"+9",
-		"-9.1.116",
-		"-9.-1.-116",
-	}
-	for _, c := range tests {
-		require.Panics(t, func() {
-			parseCudaVersion(c)
-		}, "parseCudaVersion(%v)", c)
-	}
-}
-
 func TestIsPrivileged(t *testing.T) {
 	var tests = []struct {
 		spec     string

cmd/nvidia-container-toolkit/main.go

@@ -6,21 +6,21 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
 	"strings"
 	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 var (
-	debugflag  = flag.Bool("debug", false, "enable debug output")
-	forceflag  = flag.Bool("force", false, "force execution of prestart hook in experimental mode")
-	configflag = flag.String("config", "", "configuration file")
-
-	defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
+	debugflag   = flag.Bool("debug", false, "enable debug output")
+	versionflag = flag.Bool("version", false, "enable version output")
+	configflag  = flag.String("config", "", "configuration file")
 )
 
 func exit() {
@@ -36,28 +36,16 @@ func exit() {
 	os.Exit(0)
 }
 
-func getPATH(config CLIConfig) string {
-	dirs := filepath.SplitList(os.Getenv("PATH"))
-	// directories from the hook environment have higher precedence
-	dirs = append(dirs, defaultPATH...)
-
-	if config.Root != nil {
-		rootDirs := []string{}
-		for _, dir := range dirs {
-			rootDirs = append(rootDirs, path.Join(*config.Root, dir))
-		}
-		// directories with the root prefix have higher precedence
-		dirs = append(rootDirs, dirs...)
-	}
-	return strings.Join(dirs, ":")
-}
-
 func getCLIPath(config CLIConfig) string {
 	if config.Path != nil {
 		return *config.Path
 	}
 
-	if err := os.Setenv("PATH", getPATH(config)); err != nil {
+	var root string
+	if config.Root != nil {
+		root = *config.Root
+	}
+	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -86,7 +74,7 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
-	if hook.NVIDIAContainerRuntime.Experimental && !*forceflag {
+	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
 		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
 	}
 
@@ -172,6 +160,11 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	if *versionflag {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime Hook", info.GetVersionString())
+		return
+	}
+
 	args := flag.Args()
 	if len(args) == 0 {
 		flag.Usage()
@@ -191,3 +184,12 @@ func main() {
 		os.Exit(2)
 	}
 }
+
+// logInterceptor implements the info.Logger interface to allow for logging from this function.
+type logInterceptor struct{}
+
+func (l *logInterceptor) Infof(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
+
+func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
@@ -35,6 +36,7 @@ type command struct {
 type config struct {
 	hostRoot      string
 	filenames     cli.StringSlice
+	links         cli.StringSlice
 	containerSpec string
 }
 
@@ -66,11 +68,15 @@ func (m command) build() *cli.Command {
 			Destination: &cfg.hostRoot,
 		},
 		&cli.StringSliceFlag{
-			Name:        "csv-filenames",
-			Aliases:     []string{"f"},
-			Usage:       "Specify the (CSV) filenames to process",
+			Name:        "csv-filename",
+			Usage:       "Specify a (CSV) filename to process",
 			Destination: &cfg.filenames,
 		},
+		&cli.StringSliceFlag{
+			Name:        "link",
+			Usage:       "Specify a specific link to create. The link is specified as source:target",
+			Destination: &cfg.links,
+		},
 		&cli.StringFlag{
 			Name:        "container-spec",
 			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
@@ -87,14 +93,9 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to load container state: %v", err)
 	}
 
-	spec, err := s.LoadSpec()
+	containerRoot, err := s.GetContainerRoot()
 	if err != nil {
-		return fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	var containerRoot string
-	if spec.Root != nil {
-		containerRoot = spec.Root.Path
+		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
 	csvFiles := cfg.filenames.Value()
@@ -135,40 +136,59 @@ func (m command) run(c *cli.Context, cfg *config) error {
 			m.logger.Debugf("%v is not a symlink", candidate)
 			continue
 		}
-		target, err := changeRoot(cfg.hostRoot, "/", targets[0])
+
+		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
 		if err != nil {
-			m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, cfg.hostRoot, err)
+			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
+		}
+	}
+
+	links := cfg.links.Value()
+	for _, l := range links {
+		parts := strings.Split(l, ":")
+		if len(parts) != 2 {
+			m.logger.Warnf("Invalid link specification %v", l)
 			continue
 		}
 
-		linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate)
+		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
 		if err != nil {
-			m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", candidate, cfg.hostRoot, err)
-			continue
+			m.logger.Warnf("Failed to create link %v: %v", parts, err)
 		}
-
-		if created[linkPath] {
-			m.logger.Debugf("Link %v already created", linkPath)
-			continue
-		}
-		m.logger.Infof("Symlinking %v to %v", linkPath, target)
-		err = os.MkdirAll(filepath.Dir(linkPath), 0755)
-		if err != nil {
-			m.logger.Warnf("Faild to create directory: %v", err)
-			continue
-		}
-		err = os.Symlink(target, linkPath)
-		if err != nil {
-			m.logger.Warnf("Failed to create symlink: %v", err)
-			continue
-		}
-		created[linkPath] = true
 	}
 
 	return nil
 
 }
 
+func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
+	linkPath, err := changeRoot(hostRoot, containerRoot, link)
+	if err != nil {
+		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
+	}
+	if created[linkPath] {
+		m.logger.Debugf("Link %v already created", linkPath)
+		return nil
+	}
+
+	targetPath, err := changeRoot(hostRoot, "/", target)
+	if err != nil {
+		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
+	}
+
+	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)
+	err = os.MkdirAll(filepath.Dir(linkPath), 0755)
+	if err != nil {
+		return fmt.Errorf("failed to create directory: %v", err)
+	}
+	err = os.Symlink(target, linkPath)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+
+	return nil
+}
+
 func changeRoot(current string, new string, path string) (string, error) {
 	if !filepath.IsAbs(path) {
 		return path, nil

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -59,8 +59,8 @@ func (m command) build() *cli.Command {
 
 	c.Flags = []cli.Flag{
 		&cli.StringSliceFlag{
-			Name:        "folders",
-			Usage:       "Specifiy the additional folders to add to /etc/ld.so.conf before updating the ld cache",
+			Name:        "folder",
+			Usage:       "Specifiy a folder to add to /etc/ld.so.conf before updating the ld cache",
 			Destination: &cfg.folders,
 		},
 		&cli.StringFlag{
@@ -79,14 +79,9 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to load container state: %v", err)
 	}
 
-	spec, err := s.LoadSpec()
+	containerRoot, err := s.GetContainerRoot()
 	if err != nil {
-		return fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	var containerRoot string
-	if spec.Root != nil {
-		containerRoot = spec.Root.Path
+		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
 	err = m.createConfig(containerRoot, cfg.folders.Value())

cmd/nvidia-ctk/main.go

@@ -20,12 +20,11 @@ import (
 	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 )
 
-var version string
-
 var logger = log.New()
 
 // config defines the options that can be set for the CLI through config files,
@@ -41,10 +40,11 @@ func main() {
 
 	// Create the top-level CLI
 	c := cli.NewApp()
+	c.Name = "NVIDIA Container Toolkit CLI"
 	c.UseShortOptionHandling = true
 	c.EnableBashCompletion = true
 	c.Usage = "Tools to configure the NVIDIA Container Toolkit"
-	c.Version = version
+	c.Version = info.GetVersionString()
 
 	// Setup the flags for this command
 	c.Flags = []cli.Flag{

config/config.toml.ubuntu

@@ -17,3 +17,10 @@ ldconfig = "@/sbin/ldconfig.real"
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
 #experimental = false
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]

docker/Dockerfile.amazonlinux

@@ -3,6 +3,7 @@ FROM ${BASEIMAGE}
 
 RUN yum install -y \
         ca-certificates \
+        gcc \
         wget \
         git \
         rpm-build \
@@ -40,6 +41,8 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX

docker/Dockerfile.centos

@@ -41,6 +41,8 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX

docker/Dockerfile.debian

@@ -48,6 +48,8 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX

docker/Dockerfile.devel

@@ -14,7 +14,7 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go get -u golang.org/x/lint/golint
-RUN go get -u github.com/matryer/moq
-RUN go get -u github.com/gordonklaus/ineffassign
-RUN go get -u github.com/client9/misspell/cmd/misspell
+RUN go install golang.org/x/lint/golint@latest
+RUN go install github.com/matryer/moq@latest
+RUN go install github.com/gordonklaus/ineffassign@latest
+RUN go install github.com/client9/misspell/cmd/misspell@latest

docker/Dockerfile.opensuse-leap

@@ -39,6 +39,8 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch

docker/Dockerfile.ubuntu

@@ -46,6 +46,8 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX

docker/docker.mk

@@ -131,6 +131,7 @@ docker-build-%:
 	    --build-arg PKG_VERS="$(LIB_VERSION)" \
 	    --build-arg PKG_REV="$(PKG_REV)" \
 	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
+		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \

go.mod

@@ -4,8 +4,9 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v1.0.0
+	github.com/NVIDIA/go-nvml v0.11.6-0
 	github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
-	github.com/containers/podman/v4 v4.0.1
+	github.com/containers/podman/v4 v4.0.3
 	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
 	github.com/pelletier/go-toml v1.9.4
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -107,6 +107,8 @@ github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01
 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
+github.com/NVIDIA/go-nvml v0.11.6-0 h1:tugQzmaX84Y/6+03wZ/MAgcpfSKDkvkAWeuxFNLHmxY=
+github.com/NVIDIA/go-nvml v0.11.6-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
@@ -126,6 +128,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE=
@@ -304,22 +307,23 @@ github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtr
 github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
 github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
 github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE=
-github.com/containers/buildah v1.24.1/go.mod h1:sE7AaoPQYwAB7dleOOKOpzOO3bA8lRUvZRiZcn/RYi0=
-github.com/containers/common v0.47.3/go.mod h1:/VAV4ibC27Lfyb9cxXM4uTYrJFa/7s+utNB052MJdzY=
-github.com/containers/common v0.47.4/go.mod h1:HgX0mFXyB0Tbe2REEIp9x9CxET6iSzmHfwR6S/t2LZc=
+github.com/containers/buildah v1.24.3/go.mod h1:qU6SSznCzbRS3SVllEYoMUMW8kqxAqsggscCMctwfL8=
+github.com/containers/common v0.47.5/go.mod h1:HgX0mFXyB0Tbe2REEIp9x9CxET6iSzmHfwR6S/t2LZc=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.19.1/go.mod h1:ewoo3u+TpJvGmsz64XgzbyTHwHtM94q7mgK/pX+v2SE=
+github.com/containers/image/v5 v5.19.2/go.mod h1:4LnzIMS0IclD+4NAzzbryLGcSmrKoyLJmNaZp16ke6A=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
 github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
 github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
-github.com/containers/podman/v4 v4.0.1 h1:HZzC54S6lh8BDYrJhrZYTwhyZuZOk6vYp4WZU0fWg/M=
-github.com/containers/podman/v4 v4.0.1/go.mod h1:ALboJU+WpxNIufTpPqfT2xHokFRr7iwNE23k+tfwZw8=
+github.com/containers/podman/v4 v4.0.3 h1:p1A6HgNg20kRkB+MoLodZck3z6Bfaf63pJRcKbdgqLE=
+github.com/containers/podman/v4 v4.0.3/go.mod h1:/uS33+/OF7EzmWVDWSVo3uNUhEnp6cUUGXgKR9cLjb0=
 github.com/containers/psgo v1.7.2/go.mod h1:SLpqxsPOHtTqRygjutCPXmeU2PoEFzV3gzJplN4BMx0=
 github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4=
 github.com/containers/storage v1.38.0/go.mod h1:lBzt28gAk5ADZuRtwdndRJyqX22vnRaXmlF+7ktfMYc=
 github.com/containers/storage v1.38.2/go.mod h1:INP0RPLHWBxx+pTsO5uiHlDUGHDFvWZPWprAbAlQWPQ=
+github.com/containers/storage v1.38.3/go.mod h1:INP0RPLHWBxx+pTsO5uiHlDUGHDFvWZPWprAbAlQWPQ=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@@ -375,6 +379,7 @@ github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.3-0.20220208084023-a5c757555091+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -429,6 +434,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fsouza/go-dockerclient v1.7.7/go.mod h1:njNCXvoZj3sLPjf3yO0DPHf1mdLdCPDYPc14GskKA4Y=
+github.com/fsouza/go-dockerclient v1.7.8/go.mod h1:7cvopLQDrW3dJ5mcx2LzWMBfmpv/fq7MZUEPcQlAtLw=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
 github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM=
 github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E=
@@ -448,8 +454,10 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
@@ -562,6 +570,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
 github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -721,6 +730,7 @@ github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52Cu
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/txtarfs v0.0.0-20210218200122-0702f000015a/go.mod h1:izVPOvVRsHiKkeGCT6tYBNWyDVuzj9wAaBb5R9qamfw=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
 github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
@@ -736,6 +746,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
@@ -885,6 +896,7 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo=
 github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
@@ -1009,6 +1021,7 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -1021,6 +1034,7 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -1494,6 +1508,7 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1529,6 +1544,7 @@ golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/config/config.go

@@ -80,22 +80,26 @@ func loadConfigFrom(reader io.Reader) (*Config, error) {
 		return nil, err
 	}
 
-	return getConfigFrom(toml), nil
+	return getConfigFrom(toml)
 }
 
 // getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getConfigFrom(toml *toml.Tree) *Config {
+func getConfigFrom(toml *toml.Tree) (*Config, error) {
 	cfg := getDefaultConfig()
 
 	if toml == nil {
-		return cfg
+		return cfg, nil
 	}
 
 	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
 	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
-	cfg.NVIDIAContainerRuntimeConfig = *getRuntimeConfigFrom(toml)
+	runtimeConfig, err := getRuntimeConfigFrom(toml)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
+	}
+	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
 
-	return cfg
+	return cfg, nil
 }
 
 // getDefaultConfig defines the default values for the config

internal/config/config_test.go

@@ -62,9 +62,14 @@ func TestGetConfig(t *testing.T) {
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/dev/null",
-					Experimental:  false,
-					DiscoverMode:  "auto",
 					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+					},
 				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "nvidia-ctk",
@@ -79,6 +84,9 @@ func TestGetConfig(t *testing.T) {
 				"nvidia-container-runtime.experimental = true",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 				"nvidia-container-runtime.log-level = \"debug\"",
+				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
+				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
@@ -87,9 +95,14 @@ func TestGetConfig(t *testing.T) {
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
-					Experimental:  true,
-					DiscoverMode:  "not-legacy",
 					LogLevel:      "debug",
+					Runtimes:      []string{"/some/runtime"},
+					Mode:          "not-auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+					},
 				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
@@ -106,6 +119,10 @@ func TestGetConfig(t *testing.T) {
 				"experimental = true",
 				"discover-mode = \"not-legacy\"",
 				"log-level = \"debug\"",
+				"runtimes = [\"/some/runtime\",]",
+				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.csv]",
+				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
@@ -115,9 +132,14 @@ func TestGetConfig(t *testing.T) {
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
-					Experimental:  true,
-					DiscoverMode:  "not-legacy",
 					LogLevel:      "debug",
+					Runtimes:      []string{"/some/runtime"},
+					Mode:          "not-auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+					},
 				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",

internal/config/image/cuda_image.go

@@ -0,0 +1,143 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"golang.org/x/mod/semver"
+)
+
+const (
+	envCUDAVersion      = "CUDA_VERSION"
+	envNVRequirePrefix  = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA    = envNVRequirePrefix + "CUDA"
+	envNVDisableRequire = "NVIDIA_DISABLE_REQUIRE"
+)
+
+// CUDA represents a CUDA image that can be used for GPU computing. This wraps
+// a map of environment variable to values that can be used to perform lookups
+// such as requirements.
+type CUDA map[string]string
+
+// NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
+// The process environment is read (if present) to construc the CUDA Image.
+func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
+	if spec == nil || spec.Process == nil {
+		return NewCUDAImageFromEnv(nil)
+	}
+
+	return NewCUDAImageFromEnv(spec.Process.Env)
+}
+
+// NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
+// is a list of strings of the form ENVAR=VALUE.
+func NewCUDAImageFromEnv(env []string) (CUDA, error) {
+	c := make(CUDA)
+
+	for _, e := range env {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid environment variable: %v", e)
+		}
+		c[parts[0]] = parts[1]
+	}
+
+	return c, nil
+}
+
+// IsLegacy returns whether the associated CUDA image is a "legacy" image. An
+// image is considered legacy if it has a CUDA_VERSION environment variable defined
+// and no NVIDIA_REQUIRE_CUDA environment variable defined.
+func (i CUDA) IsLegacy() bool {
+	legacyCudaVersion := i[envCUDAVersion]
+	cudaRequire := i[envNVRequireCUDA]
+	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
+}
+
+// GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
+// variables.
+func (i CUDA) GetRequirements() ([]string, error) {
+	// TODO: We need not process this if disable require is set, but this will be done
+	// in a single follow-up to ensure that the behavioural change is accurately captured.
+	// if i.HasDisableRequire() {
+	// 	return nil, nil
+	// }
+
+	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
+	var requirements []string
+	for name, value := range i {
+		if strings.HasPrefix(name, envNVRequirePrefix) {
+			requirements = append(requirements, value)
+		}
+	}
+	if i.IsLegacy() {
+		v, err := i.legacyVersion()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get version: %v", err)
+		}
+		cudaRequire := fmt.Sprintf("cuda>=%s", v)
+		requirements = append(requirements, cudaRequire)
+	}
+	return requirements, nil
+}
+
+// HasDisableRequire checks for the value of the NVIDIA_DISABLE_REQUIRE. If set
+// to a valid (true) boolean value this can be used to disable the requirement checks
+func (i CUDA) HasDisableRequire() bool {
+	if disable, exists := i[envNVDisableRequire]; exists {
+		// i.logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", disable)
+		d, _ := strconv.ParseBool(disable)
+		return d
+	}
+
+	return false
+}
+
+func (i CUDA) legacyVersion() (string, error) {
+	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
+	if err != nil {
+		return "", fmt.Errorf("invalid CUDA version: %v", err)
+	}
+
+	return majorMinor, nil
+}
+
+func parseMajorMinorVersion(version string) (string, error) {
+	vVersion := "v" + strings.TrimPrefix(version, "v")
+
+	if !semver.IsValid(vVersion) {
+		return "", fmt.Errorf("invalid version string")
+	}
+
+	majorMinor := strings.TrimPrefix(semver.MajorMinor(vVersion), "v")
+	parts := strings.Split(majorMinor, ".")
+
+	var err error
+	_, err = strconv.ParseUint(parts[0], 10, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid major version")
+	}
+	_, err = strconv.ParseUint(parts[1], 10, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid minor version")
+	}
+	return majorMinor, nil
+}

internal/config/image/cuda_image_test.go

@@ -0,0 +1,71 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseMajorMinorVersionValid(t *testing.T) {
+	var tests = []struct {
+		version  string
+		expected string
+	}{
+		{"0", "0.0"},
+		{"8", "8.0"},
+		{"7.5", "7.5"},
+		{"9.0.116", "9.0"},
+		{"4294967295.4294967295.4294967295", "4294967295.4294967295"},
+		{"v11.6", "11.6"},
+	}
+	for _, c := range tests {
+		t.Run(c.version, func(t *testing.T) {
+			version, err := parseMajorMinorVersion(c.version)
+
+			require.NoError(t, err)
+			require.Equal(t, c.expected, version)
+		})
+	}
+}
+
+func TestParseMajorMinorVersionInvalid(t *testing.T) {
+	var tests = []string{
+		"foo",
+		"foo.5.10",
+		"9.0.116.50",
+		"9.0.116foo",
+		"7.foo",
+		"9.0.bar",
+		"9.4294967296",
+		"9.0.116.",
+		"9..0",
+		"9.",
+		".5.10",
+		"-9",
+		"+9",
+		"-9.1.116",
+		"-9.-1.-116",
+	}
+	for _, c := range tests {
+		t.Run(c, func(t *testing.T) {
+			_, err := parseMajorMinorVersion(c)
+			require.Error(t, err)
+		})
+	}
+}

internal/config/runtime.go

@@ -17,42 +17,78 @@
 package config
 
 import (
+	"fmt"
+
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	dockerRuncExecutableName = "docker-runc"
+	runcExecutableName       = "runc"
+
+	auto = "auto"
+)
+
 // RuntimeConfig stores the config options for the NVIDIA Container Runtime
 type RuntimeConfig struct {
-	DebugFilePath string
-	Experimental  bool
-	DiscoverMode  string
+	DebugFilePath string `toml:"debug"`
 	// LogLevel defines the logging level for the application
-	LogLevel string
+	LogLevel string `toml:"log-level"`
+	// Runtimes defines the candidates for the low-level runtime
+	Runtimes []string    `toml:"runtimes"`
+	Mode     string      `toml:"mode"`
+	Modes    modesConfig `toml:"modes"`
+}
+
+// modesConfig defines (optional) per-mode configs
+type modesConfig struct {
+	CSV csvModeConfig `toml:"csv"`
+}
+
+type csvModeConfig struct {
+	MountSpecPath string `toml:"mount-spec-path"`
+}
+
+// dummy allows us to unmarshal only a RuntimeConfig from a *toml.Tree
+type dummy struct {
+	Runtime RuntimeConfig `toml:"nvidia-container-runtime"`
 }
 
 // getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getRuntimeConfigFrom(toml *toml.Tree) *RuntimeConfig {
+func getRuntimeConfigFrom(toml *toml.Tree) (*RuntimeConfig, error) {
 	cfg := GetDefaultRuntimeConfig()
 
 	if toml == nil {
-		return cfg
+		return cfg, nil
 	}
 
-	cfg.DebugFilePath = toml.GetDefault("nvidia-container-runtime.debug", cfg.DebugFilePath).(string)
-	cfg.Experimental = toml.GetDefault("nvidia-container-runtime.experimental", cfg.Experimental).(bool)
-	cfg.DiscoverMode = toml.GetDefault("nvidia-container-runtime.discover-mode", cfg.DiscoverMode).(string)
-	cfg.LogLevel = toml.GetDefault("nvidia-container-runtime.log-level", cfg.LogLevel).(string)
+	d := dummy{
+		Runtime: *cfg,
+	}
 
-	return cfg
+	if err := toml.Unmarshal(&d); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
+	}
+
+	return &d.Runtime, nil
 }
 
 // GetDefaultRuntimeConfig defines the default values for the config
 func GetDefaultRuntimeConfig() *RuntimeConfig {
 	c := RuntimeConfig{
 		DebugFilePath: "/dev/null",
-		Experimental:  false,
-		DiscoverMode:  "auto",
 		LogLevel:      logrus.InfoLevel.String(),
+		Runtimes: []string{
+			dockerRuncExecutableName,
+			runcExecutableName,
+		},
+		Mode: auto,
+		Modes: modesConfig{
+			CSV: csvModeConfig{
+				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+			},
+		},
 	}
 
 	return &c

internal/cuda/cuda.go

@@ -0,0 +1,137 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cuda
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+/*
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
+
+#ifdef _WIN32
+#define CUDAAPI __stdcall
+#else
+#define CUDAAPI
+#endif
+
+typedef int CUdevice;
+
+typedef enum CUdevice_attribute_enum {
+    CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75,
+    CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR = 76
+} CUdevice_attribute;
+
+typedef enum cudaError_enum {
+	CUDA_SUCCESS = 0
+} CUresult;
+
+CUresult CUDAAPI cuInit(unsigned int Flags);
+CUresult CUDAAPI cuDriverGetVersion(int *driverVersion);
+CUresult CUDAAPI cuDeviceGet(CUdevice *device, int ordinal);
+CUresult CUDAAPI cuDeviceGetAttribute(int *pi, CUdevice_attribute attrib, CUdevice dev);
+*/
+import "C"
+
+const (
+	libraryName      = "libcuda.so.1"
+	libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+// cuda stores a reference the cuda dynamic library
+var lib *dl.DynamicLibrary
+
+// Version returns the CUDA version of the driver as a string or an error if this
+// cannot be determined.
+func Version() (string, error) {
+	lib, err := load()
+	if err != nil {
+		return "", err
+	}
+	defer lib.Close()
+
+	if err := lib.Lookup("cuDriverGetVersion"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+
+	var version C.int
+	if result := C.cuDriverGetVersion(&version); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA version: result=%v", result)
+	}
+
+	major := version / 1000
+	minor := version % 100 / 10
+
+	return fmt.Sprintf("%d.%d", major, minor), nil
+}
+
+// ComputeCapability returns the CUDA compute capability of a device with the specified index as a string
+// or an error if this cannot be determined.
+func ComputeCapability(index int) (string, error) {
+	lib, err := load()
+	if err != nil {
+		return "", err
+	}
+	defer lib.Close()
+
+	if err := lib.Lookup("cuInit"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+	if err := lib.Lookup("cuDeviceGet"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+	if err := lib.Lookup("cuDeviceGetAttribute"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+
+	if result := C.cuInit(C.uint(0)); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to initialize CUDA: result=%v", result)
+	}
+
+	var device C.CUdevice
+	// NOTE: We only query the first device
+	if result := C.cuDeviceGet(&device, C.int(index)); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA device %v: result=%v", 0, result)
+	}
+
+	var major C.int
+	if result := C.cuDeviceGetAttribute(&major, C.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA compute capability major for device %v : result=%v", 0, result)
+	}
+
+	var minor C.int
+	if result := C.cuDeviceGetAttribute(&minor, C.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA compute capability minor for device %v: result=%v", 0, result)
+	}
+
+	return fmt.Sprintf("%d.%d", major, minor), nil
+}
+
+func load() (*dl.DynamicLibrary, error) {
+	lib := dl.New(libraryName, libraryLoadFlags)
+	if lib == nil {
+		return nil, fmt.Errorf("error instantiating DynamicLibrary for CUDA")
+	}
+	err := lib.Open()
+	if err != nil {
+		return nil, fmt.Errorf("error opening DynamicLibrary for CUDA: %v", err)
+	}
+
+	return lib, nil
+}

internal/discover/ldconfig.go

@@ -74,7 +74,7 @@ func (d ldconfig) Hooks() ([]Hook, error) {
 
 	args := []string{hookPath, "hook", "update-ldcache"}
 	for _, f := range libDirs {
-		args = append(args, "--folders", f)
+		args = append(args, "--folder", f)
 	}
 	h := Hook{
 		Lifecycle: cdi.CreateContainerHook,

internal/discover/legacy.go

@@ -1,70 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package discover
-
-import (
-	"path/filepath"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	"github.com/sirupsen/logrus"
-)
-
-// NewLegacyDiscoverer creates a discoverer for the experimental runtime
-func NewLegacyDiscoverer(logger *logrus.Logger, cfg *Config) (Discover, error) {
-	d := legacy{
-		logger: logger,
-		lookup: lookup.NewExecutableLocator(logger, cfg.Root),
-	}
-
-	return &d, nil
-}
-
-type legacy struct {
-	None
-	logger *logrus.Logger
-	lookup lookup.Locator
-}
-
-var _ Discover = (*legacy)(nil)
-
-// Hooks returns the "legacy" NVIDIA Container Runtime hook. This hook calls out
-// to the nvidia-container-cli to make modifications to the container as defined
-// in libnvidia-container.
-func (d legacy) Hooks() ([]Hook, error) {
-	hookPath := filepath.Join(config.DefaultExecutableDir, config.NVIDIAContainerRuntimeHookExecutable)
-	targets, err := d.lookup.Locate(config.NVIDIAContainerRuntimeHookExecutable)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", config.NVIDIAContainerRuntimeHookExecutable, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", config.NVIDIAContainerRuntimeHookExecutable)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", config.NVIDIAContainerRuntimeHookExecutable, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Runtime Hook path %v", hookPath)
-
-	args := []string{hookPath, "--force", "prestart"}
-	h := Hook{
-		Lifecycle: cdi.PrestartHook,
-		Path:      hookPath,
-		Args:      args,
-	}
-
-	return []Hook{h}, nil
-}

internal/discover/symlinks.go

@@ -17,6 +17,10 @@
 package discover
 
 import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/sirupsen/logrus"
@@ -28,15 +32,17 @@ type symlinks struct {
 	lookup                  lookup.Locator
 	nvidiaCTKExecutablePath string
 	csvFiles                []string
+	mountsFrom              Discover
 }
 
 // NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container
-func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, cfg *Config) (Discover, error) {
+func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) {
 	d := symlinks{
 		logger:                  logger,
 		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
 		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
 		csvFiles:                csvFiles,
+		mountsFrom:              mounts,
 	}
 
 	return &d, nil
@@ -58,9 +64,15 @@ func (d symlinks) Hooks() ([]Hook, error) {
 
 	args := []string{hookPath, "hook", "create-symlinks"}
 	for _, f := range d.csvFiles {
-		args = append(args, "--csv-filenames", f)
+		args = append(args, "--csv-filename", f)
 	}
 
+	links, err := d.getSpecificLinkArgs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	args = append(args, links...)
+
 	h := Hook{
 		Lifecycle: cdi.CreateContainerHook,
 		Path:      hookPath,
@@ -69,3 +81,45 @@ func (d symlinks) Hooks() ([]Hook, error) {
 
 	return []Hook{h}, nil
 }
+
+// getSpecificLinkArgs returns the required specic links that need to be created
+func (d symlinks) getSpecificLinkArgs() ([]string, error) {
+	mounts, err := d.mountsFrom.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
+	}
+
+	linkProcessed := make(map[string]bool)
+	var links []string
+	for _, m := range mounts {
+		var target string
+		var link string
+
+		lib := filepath.Base(m.Path)
+
+		if strings.HasPrefix(lib, "libcuda.so") {
+			// XXX Many applications wrongly assume that libcuda.so exists (e.g. with dlopen).
+			target = "libcuda.so.1"
+			link = "libcuda.so"
+		} else if strings.HasPrefix(lib, "libGLX_nvidia.so") {
+			// XXX GLVND requires this symlink for indirect GLX support.
+			target = lib
+			link = "libGLX_indirect.so.0"
+		} else if strings.HasPrefix(lib, "libnvidia-opticalflow.so") {
+			// XXX Fix missing symlink for libnvidia-opticalflow.so.
+			target = "libnvidia-opticalflow.so.1"
+			link = "libnvidia-opticalflow.so"
+		} else {
+			continue
+		}
+		if linkProcessed[link] {
+			continue
+		}
+
+		linkPath := filepath.Join(filepath.Dir(m.Path), link)
+		links = append(links, "--link", fmt.Sprintf("%v:%v", target, linkPath))
+		linkProcessed[link] = true
+	}
+
+	return links, nil
+}

internal/info/auto.go

@@ -0,0 +1,46 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+// Logger is a basic interface for logging to allow these functions to be called
+// from code where logrus is not used.
+type Logger interface {
+	Infof(string, ...interface{})
+	Debugf(string, ...interface{})
+}
+
+// ResolveAutoMode determines the correct mode for the platform if set to "auto"
+func ResolveAutoMode(logger Logger, mode string) (rmode string) {
+	if mode != "auto" {
+		return mode
+	}
+	defer func() {
+		logger.Infof("Auto-detected mode as '%v'", rmode)
+	}()
+
+	isTegra, reason := IsTegraSystem()
+	logger.Debugf("Is Tegra-based system? %v: %v", isTegra, reason)
+
+	hasNVML, reason := HasNVML()
+	logger.Debugf("Has NVML? %v: %v", hasNVML, reason)
+
+	if isTegra && !hasNVML {
+		return "csv"
+	}
+
+	return "legacy"
+}

internal/info/auto_test.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveAutoMode(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description  string
+		mode         string
+		expectedMode string
+	}{
+		{
+			description:  "non-auto resolves to input",
+			mode:         "not-auto",
+			expectedMode: "not-auto",
+		},
+		// TODO: The following test is brittle in that it will break on Tegra-based systems.
+		// {
+		// 	description:  "auto resolves to legacy",
+		// 	mode:         "auto",
+		// 	expectedMode: "legacy",
+		// },
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mode := ResolveAutoMode(logger, tc.mode)
+			require.EqualValues(t, tc.expectedMode, mode)
+		})
+	}
+}

internal/info/info.go

@@ -0,0 +1,65 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+// HasNVML returns true if NVML is detected on the sytems
+func HasNVML() (bool, string) {
+	const (
+		nvmlLibraryName      = "libnvidia-ml.so.1"
+		nvmlLibraryLoadFlags = dl.RTLD_LAZY
+	)
+	lib := dl.New(nvmlLibraryName, nvmlLibraryLoadFlags)
+	if err := lib.Open(); err != nil {
+		return false, fmt.Sprintf("could not load NVML: %v", err)
+	}
+	defer lib.Close()
+
+	return true, "found NVML library"
+}
+
+// IsTegraSystem returns true if the system is detected as a Tegra-based system
+func IsTegraSystem() (bool, string) {
+	const tegraReleaseFile = "/etc/nv_tegra_release"
+	const tegraFamilyFile = "/sys/devices/soc0/family"
+
+	if info, err := os.Stat(tegraReleaseFile); err == nil && !info.IsDir() {
+		return true, fmt.Sprintf("%v found", tegraReleaseFile)
+	}
+
+	if info, err := os.Stat(tegraFamilyFile); err != nil || !info.IsDir() {
+		return false, fmt.Sprintf("%v not found", tegraFamilyFile)
+	}
+
+	contents, err := os.ReadFile(tegraFamilyFile)
+	if err != nil {
+		return false, fmt.Sprintf("could not read %v", tegraFamilyFile)
+	}
+
+	if strings.HasPrefix(strings.ToLower(string(contents)), "tegra") {
+		return true, fmt.Sprintf("%v has 'tegra' prefix", tegraFamilyFile)
+	}
+
+	return false, fmt.Sprintf("%v has no 'tegra' prefix", tegraFamilyFile)
+}

internal/info/version.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import "strings"
+
+// version must be set by go build's -X main.version= option in the Makefile.
+var version = "unknown"
+
+// gitCommit will be the hash that the binary was built from
+// and will be populated by the Makefile
+var gitCommit = ""
+
+// GetVersionParts returns the different version components
+func GetVersionParts() []string {
+	v := []string{version}
+
+	if gitCommit != "" {
+		v = append(v, "commit: "+gitCommit)
+	}
+
+	return v
+}
+
+// GetVersionString returns the string representation of the version
+func GetVersionString(more ...string) string {
+	v := append(GetVersionParts(), more...)
+	return strings.Join(v, "\n")
+}

internal/lookup/executable.go

@@ -25,24 +25,13 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-const (
-	envPath = "PATH"
-)
-
-var defaultPaths = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
-
 type executable struct {
 	file
 }
 
 // NewExecutableLocator creates a locator to fine executable files in the path. A logger can also be specified.
 func NewExecutableLocator(logger *log.Logger, root string) Locator {
-	pathEnv := os.Getenv(envPath)
-	paths := filepath.SplitList(pathEnv)
-
-	if root != "" {
-		paths = append(paths, defaultPaths...)
-	}
+	paths := GetPaths(root)
 
 	var prefixes []string
 	for _, dir := range paths {

internal/lookup/path.go

@@ -0,0 +1,69 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	envPath = "PATH"
+)
+
+var (
+	defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
+)
+
+// GetPaths returns a list of paths for a specified root. These are constructed from the
+// PATH environment variable, a default path list, and the supplied root.
+func GetPaths(root string) []string {
+	dirs := filepath.SplitList(os.Getenv(envPath))
+
+	inDirs := make(map[string]bool)
+	for _, d := range dirs {
+		inDirs[d] = true
+	}
+
+	// directories from the environment have higher precedence
+	for _, d := range defaultPATH {
+		if inDirs[d] {
+			// We don't add paths that are already included
+			continue
+		}
+		dirs = append(dirs, d)
+	}
+
+	if root != "" && root != "/" {
+		rootDirs := []string{}
+		for _, dir := range dirs {
+			rootDirs = append(rootDirs, path.Join(root, dir))
+		}
+		// directories with the root prefix have higher precedence
+		dirs = append(rootDirs, dirs...)
+	}
+
+	return dirs
+}
+
+// GetPath returns a colon-separated path value that can be used to set the PATH
+// environment variable
+func GetPath(root string) string {
+	return strings.Join(GetPaths(root), ":")
+}

internal/oci/runtime_low_level.go

@@ -18,8 +18,8 @@ package oci
 
 import (
 	"fmt"
-	"os/exec"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -32,6 +32,7 @@ func NewLowLevelRuntime(logger *log.Logger, candidates []string) (Runtime, error
 		return nil, fmt.Errorf("error locating runtime: %v", err)
 	}
 
+	logger.Infof("Using low-level runtime %v", runtimePath)
 	return NewRuntimeForPath(logger, runtimePath)
 }
 
@@ -42,14 +43,15 @@ func findRuntime(logger *log.Logger, candidates []string) (string, error) {
 		return "", fmt.Errorf("at least one runtime candidate must be specified")
 	}
 
+	locator := lookup.NewExecutableLocator(logger, "/")
 	for _, candidate := range candidates {
-		logger.Infof("Looking for runtime binary '%v'", candidate)
-		runcPath, err := exec.LookPath(candidate)
-		if err == nil {
-			logger.Infof("Found runtime binary '%v'", runcPath)
-			return runcPath, nil
+		logger.Debugf("Looking for runtime binary '%v'", candidate)
+		targets, err := locator.Locate(candidate)
+		if err == nil && len(targets) > 0 {
+			logger.Debugf("Found runtime binary '%v'", targets)
+			return targets[0], nil
 		}
-		logger.Warnf("Runtime binary '%v' not found: %v", candidate, err)
+		logger.Debugf("Runtime binary '%v' not found: %v (targets=%v)", candidate, err, targets)
 	}
 
 	return "", fmt.Errorf("no runtime binary found from candidate list: %v", candidates)

internal/oci/spec.go

@@ -33,7 +33,7 @@ type SpecModifier interface {
 //go:generate moq -stub -out spec_mock.go . Spec
 // Spec defines the operations to be performed on an OCI specification
 type Spec interface {
-	Load() error
+	Load() (*specs.Spec, error)
 	Flush() error
 	Modify(SpecModifier) error
 	LookupEnv(string) (string, bool)
@@ -46,7 +46,7 @@ func NewSpec(logger *logrus.Logger, args []string) (Spec, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error getting bundle directory: %v", err)
 	}
-	logger.Infof("Using bundle directory: %v", bundleDir)
+	logger.Debugf("Using bundle directory: %v", bundleDir)
 
 	ociSpecPath := GetSpecFilePath(bundleDir)
 	logger.Infof("Using OCI specification file path: %v", ociSpecPath)

internal/oci/spec_file.go

@@ -45,19 +45,19 @@ func NewFileSpec(filepath string) Spec {
 
 // Load reads the contents of an OCI spec from file to be referenced internally.
 // The file is opened "read-only"
-func (s *fileSpec) Load() error {
+func (s *fileSpec) Load() (*specs.Spec, error) {
 	specFile, err := os.Open(s.path)
 	if err != nil {
-		return fmt.Errorf("error opening OCI specification file: %v", err)
+		return nil, fmt.Errorf("error opening OCI specification file: %v", err)
 	}
 	defer specFile.Close()
 
 	spec, err := LoadFrom(specFile)
 	if err != nil {
-		return fmt.Errorf("error loading OCI specification from file: %v", err)
+		return nil, fmt.Errorf("error loading OCI specification from file: %v", err)
 	}
 	s.Spec = spec
-	return nil
+	return s.Spec, nil
 }
 
 // LoadFrom reads the contents of the OCI spec from the specified io.Reader.

internal/oci/spec_memory.go

@@ -37,8 +37,8 @@ func NewMemorySpec(spec *specs.Spec) Spec {
 }
 
 // Load is a no-op for the memorySpec spec
-func (s *memorySpec) Load() error {
-	return nil
+func (s *memorySpec) Load() (*specs.Spec, error) {
+	return s.Spec, nil
 }
 
 // Flush is a no-op for the memorySpec spec

internal/oci/spec_mock.go

@@ -4,6 +4,7 @@
 package oci
 
 import (
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"sync"
 )
 
@@ -20,7 +21,7 @@ var _ Spec = &SpecMock{}
 // 			FlushFunc: func() error {
 // 				panic("mock out the Flush method")
 // 			},
-// 			LoadFunc: func() error {
+// 			LoadFunc: func() (*specs.Spec, error) {
 // 				panic("mock out the Load method")
 // 			},
 // 			LookupEnvFunc: func(s string) (string, bool) {
@@ -40,7 +41,7 @@ type SpecMock struct {
 	FlushFunc func() error
 
 	// LoadFunc mocks the Load method.
-	LoadFunc func() error
+	LoadFunc func() (*specs.Spec, error)
 
 	// LookupEnvFunc mocks the LookupEnv method.
 	LookupEnvFunc func(s string) (string, bool)
@@ -103,7 +104,7 @@ func (mock *SpecMock) FlushCalls() []struct {
 }
 
 // Load calls LoadFunc.
-func (mock *SpecMock) Load() error {
+func (mock *SpecMock) Load() (*specs.Spec, error) {
 	callInfo := struct {
 	}{}
 	mock.lockLoad.Lock()
@@ -111,9 +112,10 @@ func (mock *SpecMock) Load() error {
 	mock.lockLoad.Unlock()
 	if mock.LoadFunc == nil {
 		var (
-			errOut error
+			specOut *specs.Spec
+			errOut  error
 		)
-		return errOut
+		return specOut, errOut
 	}
 	return mock.LoadFunc()
 }

internal/oci/state.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -56,7 +57,7 @@ func ReadContainerState(reader io.Reader) (*State, error) {
 }
 
 // LoadSpec loads the OCI spec associated with the container state
-func (s State) LoadSpec() (*specs.Spec, error) {
+func (s *State) LoadSpec() (*specs.Spec, error) {
 	specFilePath := GetSpecFilePath(s.Bundle)
 	specFile, err := os.Open(specFilePath)
 	if err != nil {
@@ -68,6 +69,25 @@ func (s State) LoadSpec() (*specs.Spec, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
 	}
-
 	return spec, nil
 }
+
+// GetContainerRoot returns the root for the container from the associated spec. If the spec is not yet loaded, it is
+// loaded and cached.
+func (s *State) GetContainerRoot() (string, error) {
+	spec, err := s.LoadSpec()
+	if err != nil {
+		return "", err
+	}
+
+	var containerRoot string
+	if spec.Root != nil {
+		containerRoot = spec.Root.Path
+	}
+
+	if filepath.IsAbs(containerRoot) {
+		return containerRoot, nil
+	}
+
+	return filepath.Join(s.Bundle, containerRoot), nil
+}

internal/requirements/constants.go

@@ -0,0 +1,25 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package requirements
+
+// A list of supported requirements / properties
+const (
+	ARCH   = "arch"
+	BRAND  = "brand"
+	CUDA   = "cuda"
+	DRIVER = "driver"
+)

internal/requirements/constraints/binary.go

@@ -0,0 +1,76 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import (
+	"fmt"
+)
+
+// binary represents a binary operation. This can be used to compare a specified
+// property to a value
+type binary struct {
+	left     Property
+	operator string
+	right    string
+}
+
+// String returns the string representation of the binary comparator
+func (c binary) String() string {
+	return fmt.Sprintf("%v%v%v", c.left.Name(), c.operator, c.right)
+}
+
+// Assert compares the property to the required value using the supplied comparator
+func (c binary) Assert() error {
+	satisfied, err := c.eval()
+	if err != nil {
+		return err
+	}
+	if satisfied {
+		return nil
+	}
+
+	// error_setx(err, "unsatisfied condition: %s, please update your driver to a newer version, or use an earlier cuda container", predicate_format);
+	return fmt.Errorf("unsatisfied condition: %v (%v)", c.String(), c.left.String())
+}
+
+func (c binary) eval() (bool, error) {
+	if c.left == nil {
+		return true, nil
+	}
+
+	compare, err := c.left.CompareTo(c.right)
+	if err != nil {
+		return false, err
+	}
+
+	switch string(c.operator) {
+	case equal:
+		return compare == 0, nil
+	case notEqual:
+		return compare != 0, nil
+	case less:
+		return compare < 0, nil
+	case lessEqual:
+		return compare <= 0, nil
+	case greater:
+		return compare > 0, nil
+	case greaterEqual:
+		return compare >= 0, nil
+	}
+
+	return false, fmt.Errorf("invalid operator %v", c.operator)
+}

internal/requirements/constraints/constants.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import "fmt"
+
+const (
+	equal        = "="
+	notEqual     = "!="
+	less         = "<"
+	lessEqual    = "<="
+	greater      = ">"
+	greaterEqual = ">="
+)
+
+// always is a constraint that is always met
+type always struct{}
+
+func (c always) Assert() error {
+	return nil
+}
+
+func (c always) String() string {
+	return "true"
+}
+
+// invalid is an invalid constraint and can never be met
+type invalid string
+
+func (c invalid) Assert() error {
+	return fmt.Errorf("invalid constraint: %v", c.String())
+}
+
+// String returns the string representation of the contraint
+func (c invalid) String() string {
+	return string(c)
+}

internal/requirements/constraints/constraint_mock.go

@@ -0,0 +1,108 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package constraints
+
+import (
+	"sync"
+)
+
+// Ensure, that ConstraintMock does implement Constraint.
+// If this is not the case, regenerate this file with moq.
+var _ Constraint = &ConstraintMock{}
+
+// ConstraintMock is a mock implementation of Constraint.
+//
+// 	func TestSomethingThatUsesConstraint(t *testing.T) {
+//
+// 		// make and configure a mocked Constraint
+// 		mockedConstraint := &ConstraintMock{
+// 			AssertFunc: func() error {
+// 				panic("mock out the Assert method")
+// 			},
+// 			StringFunc: func() string {
+// 				panic("mock out the String method")
+// 			},
+// 		}
+//
+// 		// use mockedConstraint in code that requires Constraint
+// 		// and then make assertions.
+//
+// 	}
+type ConstraintMock struct {
+	// AssertFunc mocks the Assert method.
+	AssertFunc func() error
+
+	// StringFunc mocks the String method.
+	StringFunc func() string
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Assert holds details about calls to the Assert method.
+		Assert []struct {
+		}
+		// String holds details about calls to the String method.
+		String []struct {
+		}
+	}
+	lockAssert sync.RWMutex
+	lockString sync.RWMutex
+}
+
+// Assert calls AssertFunc.
+func (mock *ConstraintMock) Assert() error {
+	callInfo := struct {
+	}{}
+	mock.lockAssert.Lock()
+	mock.calls.Assert = append(mock.calls.Assert, callInfo)
+	mock.lockAssert.Unlock()
+	if mock.AssertFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.AssertFunc()
+}
+
+// AssertCalls gets all the calls that were made to Assert.
+// Check the length with:
+//     len(mockedConstraint.AssertCalls())
+func (mock *ConstraintMock) AssertCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockAssert.RLock()
+	calls = mock.calls.Assert
+	mock.lockAssert.RUnlock()
+	return calls
+}
+
+// String calls StringFunc.
+func (mock *ConstraintMock) String() string {
+	callInfo := struct {
+	}{}
+	mock.lockString.Lock()
+	mock.calls.String = append(mock.calls.String, callInfo)
+	mock.lockString.Unlock()
+	if mock.StringFunc == nil {
+		var (
+			sOut string
+		)
+		return sOut
+	}
+	return mock.StringFunc()
+}
+
+// StringCalls gets all the calls that were made to String.
+// Check the length with:
+//     len(mockedConstraint.StringCalls())
+func (mock *ConstraintMock) StringCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockString.RLock()
+	calls = mock.calls.String
+	mock.lockString.RUnlock()
+	return calls
+}

internal/requirements/constraints/constraints.go

@@ -0,0 +1,24 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+//go:generate moq -stub -out constraint_mock.go . Constraint
+// Constraint represents a constraint that is to be evaluated
+type Constraint interface {
+	String() string
+	Assert() error
+}

internal/requirements/constraints/constraints_test.go

@@ -0,0 +1,17 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints

internal/requirements/constraints/factory.go

@@ -0,0 +1,143 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+type factory struct {
+	logger     *logrus.Logger
+	properties map[string]Property
+}
+
+// New creates a new constraint for the supplied requirements and properties
+func New(logger *logrus.Logger, requirements []string, properties map[string]Property) (Constraint, error) {
+	if len(requirements) == 0 {
+		return &always{}, nil
+	}
+
+	f := factory{
+		logger:     logger,
+		properties: properties,
+	}
+
+	var constraints []Constraint
+	for _, r := range requirements {
+		c, err := f.newConstraintFromRequirement(r)
+		if err != nil {
+			return nil, err
+		}
+		if c == nil {
+			continue
+		}
+		constraints = append(constraints, c)
+	}
+
+	return AND(constraints), nil
+}
+
+// newConstraintFromRequirement takes a requirement string and generates
+// the associated constraint(s). Invalid constraints are ignored.
+// Each requirement can consist of multiple constraints, with space-separated constraints being ORed
+// together and comma-separated constraints being ANDed together.
+func (r factory) newConstraintFromRequirement(requirement string) (Constraint, error) {
+	const (
+		orSeparator  = " "
+		andSeparator = ","
+	)
+	if strings.TrimSpace(requirement) == "" {
+		return nil, nil
+	}
+
+	var terms []Constraint
+	for _, term := range strings.Split(requirement, orSeparator) {
+		var factors []Constraint
+		for _, factor := range strings.Split(term, andSeparator) {
+			f, err := r.parse(factor)
+			if err != nil {
+				return nil, err
+			}
+			if f == nil {
+				r.logger.Debugf("Skipping unsupported constraint: %v", factor)
+				continue
+			}
+			factors = append(factors, f)
+		}
+
+		if len(factors) == 0 {
+			continue
+		}
+
+		if len(factors) == 1 {
+			terms = append(terms, factors[0])
+		} else {
+			terms = append(terms, and(factors))
+		}
+	}
+
+	return OR(terms), nil
+}
+
+// parse constructs a constraint from the specified string.
+// The string is expected to be of the form [PROPERTY][OPERATOR][VALUE]
+func (r factory) parse(condition string) (Constraint, error) {
+	if strings.TrimSpace(condition) == "" {
+		return nil, nil
+	}
+
+	operators := []string{
+		notEqual,
+		lessEqual,
+		greaterEqual,
+		equal,
+		less,
+		greater,
+	}
+
+	propertyEnd := strings.IndexAny(condition, "<>=!")
+	if propertyEnd == -1 {
+		return nil, fmt.Errorf("invalid constraint: %v", condition)
+	}
+
+	property := condition[:propertyEnd]
+	condition = strings.TrimPrefix(condition, property)
+
+	p, ok := r.properties[property]
+	if !ok || p == nil {
+		return nil, nil
+	}
+
+	var op string
+	for _, o := range operators {
+		if strings.HasPrefix(condition, o) {
+			op = o
+			break
+		}
+	}
+	value := strings.TrimPrefix(condition, op)
+
+	c := binary{
+		left:     p,
+		right:    value,
+		operator: op,
+	}
+	return c, p.Validate(value)
+}

internal/requirements/constraints/factory_test.go

@@ -0,0 +1,187 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import (
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParse(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	cuda := NewVersionProperty("cuda", "")
+
+	f := factory{
+		logger: logger,
+		properties: map[string]Property{
+			"cuda": cuda,
+		},
+	}
+
+	testCases := []struct {
+		description   string
+		condition     string
+		expectedError bool
+		expected      Constraint
+	}{
+		{
+			description: "empty is nil",
+			condition:   "",
+			expected:    nil,
+		},
+		{
+			description:   "missing operator is invalid",
+			condition:     "notvalid",
+			expectedError: true,
+		},
+		{
+			description: "invalid property is invalid",
+			condition:   "foo=45",
+			expected:    nil,
+		},
+		{
+			description:   "cuda must be semver",
+			condition:     "cuda=foo",
+			expectedError: true,
+			expected:      binary{cuda, equal, "foo"},
+		},
+		{
+			description: "cuda greater than equal",
+			condition:   "cuda>=11.6",
+			expected:    binary{cuda, greaterEqual, "11.6"},
+		},
+		{
+			description: "cuda greater than",
+			condition:   "cuda>11.6",
+			expected:    binary{cuda, greater, "11.6"},
+		},
+		{
+			description: "cuda less than equal",
+			condition:   "cuda<=11.6",
+			expected:    binary{cuda, lessEqual, "11.6"},
+		},
+		{
+			description: "cuda less than",
+			condition:   "cuda<11.6",
+			expected:    binary{cuda, less, "11.6"},
+		},
+		{
+			description: "cuda equal",
+			condition:   "cuda=11.6",
+			expected:    binary{cuda, equal, "11.6"},
+		},
+		{
+			description: "cuda not equal",
+			condition:   "cuda!=11.6",
+			expected:    binary{cuda, notEqual, "11.6"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			c, err := f.parse(tc.condition)
+			if tc.expectedError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.EqualValues(t, tc.expected, c)
+		})
+	}
+}
+
+func TestNewConstraintFromRequirement(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	cuda := &PropertyMock{}
+	arch := &PropertyMock{}
+
+	f := factory{
+		logger: logger,
+		properties: map[string]Property{
+			"cuda": cuda,
+			"arch": arch,
+		},
+	}
+
+	testCases := []struct {
+		description   string
+		requirement   string
+		expectedError bool
+		expected      Constraint
+	}{
+		{
+			description: "empty is nil",
+			requirement: "",
+			expected:    nil,
+		},
+		{
+			description:   "malformed constraint is invalid",
+			requirement:   "notvalid",
+			expectedError: true,
+			expected:      nil,
+		},
+		{
+			description: "unsupported property is ignored",
+			requirement: "cuda>=11.6 foo=bar",
+			expected:    binary{cuda, greaterEqual, "11.6"},
+		},
+		{
+			description: "space-separated is and",
+			requirement: "cuda>=11.6 arch=5.3",
+			expected: and([]Constraint{
+				binary{cuda, greaterEqual, "11.6"},
+				binary{arch, equal, "5.3"},
+			}),
+		},
+		{
+			description: "comma-separated is or",
+			requirement: "cuda>=11.6,arch=5.3",
+			expected: or([]Constraint{
+				binary{cuda, greaterEqual, "11.6"},
+				binary{arch, equal, "5.3"},
+			}),
+		},
+		{
+			description: "and takes precedence",
+			requirement: "cuda<13.6 cuda>=11.6,arch=5.3",
+			expected: or([]Constraint{
+				binary{cuda, less, "13.6"},
+				and([]Constraint{
+					binary{cuda, greaterEqual, "11.6"},
+					binary{arch, equal, "5.3"},
+				}),
+			}),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			c, err := f.newConstraintFromRequirement(tc.requirement)
+			if tc.expectedError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.EqualValues(t, tc.expected, c)
+		})
+	}
+
+}

internal/requirements/constraints/logical.go

@@ -0,0 +1,91 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import (
+	"fmt"
+	"strings"
+)
+
+// or represents an OR operation on a set of constraints
+type or []Constraint
+
+// and represents an AND (ALL) operation on a set of contraints
+type and []Constraint
+
+// AND constructs a new constraint that is the logical AND of the supplied constraints
+func AND(constraints []Constraint) Constraint {
+	if len(constraints) == 0 {
+		return &always{}
+	}
+	if len(constraints) == 1 {
+		return constraints[0]
+	}
+	return and(constraints)
+}
+
+// OR constructs a new constrant that is the logical OR of the supplied constraints
+func OR(constraints []Constraint) Constraint {
+	if len(constraints) == 0 {
+		return nil
+	}
+	if len(constraints) == 1 {
+		return constraints[0]
+	}
+
+	return or(constraints)
+}
+
+func (operands or) Assert() error {
+	for _, o := range operands {
+		// We stop on the first nil
+		if err := o.Assert(); err == nil {
+			return nil
+		}
+	}
+	return fmt.Errorf("%v not met", operands)
+}
+
+func (operands or) String() string {
+	var terms []string
+
+	for _, o := range operands {
+		terms = append(terms, o.String())
+	}
+
+	return strings.Join(terms, "||")
+}
+
+func (operands and) Assert() error {
+	for _, o := range operands {
+		// We stop on the first Assert
+		if err := o.Assert(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (operands and) String() string {
+	var terms []string
+
+	for _, o := range operands {
+		terms = append(terms, o.String())
+	}
+
+	return strings.Join(terms, "&&")
+}

internal/requirements/constraints/logical_test.go

@@ -0,0 +1,152 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestANDConstraint(t *testing.T) {
+
+	never := ConstraintMock{AssertFunc: func() error { return fmt.Errorf("false") }}
+
+	testCases := []struct {
+		description string
+		constraints []Constraint
+		expected    bool
+	}{
+		{
+			description: "empty is always true",
+			constraints: []Constraint{},
+			expected:    true,
+		},
+		{
+			description: "single true constraint is true",
+			constraints: []Constraint{
+				&always{},
+			},
+			expected: true,
+		},
+		{
+			description: "single false constraint is false",
+			constraints: []Constraint{
+				&never,
+			},
+			expected: false,
+		},
+		{
+			description: "multiple true constraints are true",
+			constraints: []Constraint{
+				&always{}, &always{},
+			},
+			expected: true,
+		},
+		{
+			description: "mixed constraints are false (first is true)",
+			constraints: []Constraint{
+				&always{}, &never,
+			},
+			expected: false,
+		},
+		{
+			description: "mixed constraints are false (last is true)",
+			constraints: []Constraint{
+				&never, &always{},
+			},
+			expected: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			err := and(tc.constraints).Assert()
+			if tc.expected {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+			}
+		})
+	}
+
+}
+
+func TestORConstraint(t *testing.T) {
+
+	never := ConstraintMock{AssertFunc: func() error { return fmt.Errorf("false") }}
+
+	testCases := []struct {
+		description string
+		constraints []Constraint
+		expected    bool
+	}{
+		{
+			description: "empty is always false",
+			constraints: []Constraint{},
+			expected:    false,
+		},
+		{
+			description: "single true constraint is true",
+			constraints: []Constraint{
+				&always{},
+			},
+			expected: true,
+		},
+		{
+			description: "single false constraint is false",
+			constraints: []Constraint{
+				&never,
+			},
+			expected: false,
+		},
+		{
+			description: "multiple true constraints are true",
+			constraints: []Constraint{
+				&always{}, &always{},
+			},
+			expected: true,
+		},
+		{
+			description: "mixed constraints are true (first is true)",
+			constraints: []Constraint{
+				&always{}, &never,
+			},
+			expected: true,
+		},
+		{
+			description: "mixed constraints are true (last is true)",
+			constraints: []Constraint{
+				&never, &always{},
+			},
+			expected: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			err := or(tc.constraints).Assert()
+			if tc.expected {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+			}
+		})
+	}
+
+}

internal/requirements/constraints/property.go

@@ -0,0 +1,129 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package constraints
+
+import (
+	"fmt"
+	"strings"
+
+	"golang.org/x/mod/semver"
+)
+
+//go:generate moq -stub -out property_mock.go . Property
+// Property represents a property that is used to check requirements
+type Property interface {
+	Name() string
+	Value() (string, error)
+	String() string
+	CompareTo(string) (int, error)
+	Validate(string) error
+}
+
+// NewStringProperty creates a string property based on the name-value pair
+func NewStringProperty(name string, value string) Property {
+	p := stringProperty{
+		name:  name,
+		value: value,
+	}
+
+	return p
+}
+
+// NewVersionProperty creates a property representing a semantic version based on the name-value pair
+func NewVersionProperty(name string, value string) Property {
+	p := versionProperty{
+		stringProperty: stringProperty{
+			name:  name,
+			value: value,
+		},
+	}
+
+	return p
+}
+
+// stringProperty represents a property that is used to check requirements
+type stringProperty struct {
+	name  string
+	value string
+}
+
+type versionProperty struct {
+	stringProperty
+}
+
+// Name returns a stringProperty's name
+func (p stringProperty) Name() string {
+	return p.name
+}
+
+// Value returns a stringProperty's value or an error if this cannot be determined
+func (p stringProperty) Value() (string, error) {
+	return p.value, nil
+}
+
+// CompareTo compares two strings to each other
+func (p stringProperty) CompareTo(other string) (int, error) {
+	value := p.value
+
+	if value < other {
+		return -1, nil
+	}
+
+	if value > other {
+		return 1, nil
+	}
+
+	return 0, nil
+}
+
+// Validate returns nil for all input strings
+func (p stringProperty) Validate(string) error {
+	return nil
+}
+
+// String returns the string representation of the name value combination
+func (p stringProperty) String() string {
+	v, err := p.Value()
+	if err != nil {
+		return fmt.Sprintf("invalid %v: %v", p.name, err)
+	}
+	return fmt.Sprintf("%v=%v", p.name, v)
+}
+
+// CompareTo compares two versions to each other as semantic versions
+func (p versionProperty) CompareTo(other string) (int, error) {
+	if err := p.Validate(other); err != nil {
+		return 0, fmt.Errorf("invailid value for %v: %v", p.name, err)
+	}
+
+	vValue := ensurePrefix(p.value, "v")
+	vOther := ensurePrefix(other, "v")
+	return semver.Compare(vValue, vOther), nil
+}
+
+// Validate checks whether the supplied value is a valid semantic version
+func (p versionProperty) Validate(value string) error {
+	if !semver.IsValid(ensurePrefix(value, "v")) {
+		return fmt.Errorf("invailid value %v; expected a valid version string", value)
+	}
+
+	return nil
+}
+
+func ensurePrefix(s string, prefix string) string {
+	return prefix + strings.TrimPrefix(s, prefix)
+}

internal/requirements/constraints/property_mock.go

@@ -0,0 +1,241 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package constraints
+
+import (
+	"sync"
+)
+
+// Ensure, that PropertyMock does implement Property.
+// If this is not the case, regenerate this file with moq.
+var _ Property = &PropertyMock{}
+
+// PropertyMock is a mock implementation of Property.
+//
+// 	func TestSomethingThatUsesProperty(t *testing.T) {
+//
+// 		// make and configure a mocked Property
+// 		mockedProperty := &PropertyMock{
+// 			CompareToFunc: func(s string) (int, error) {
+// 				panic("mock out the CompareTo method")
+// 			},
+// 			NameFunc: func() string {
+// 				panic("mock out the Name method")
+// 			},
+// 			StringFunc: func() string {
+// 				panic("mock out the String method")
+// 			},
+// 			ValidateFunc: func(s string) error {
+// 				panic("mock out the Validate method")
+// 			},
+// 			ValueFunc: func() (string, error) {
+// 				panic("mock out the Value method")
+// 			},
+// 		}
+//
+// 		// use mockedProperty in code that requires Property
+// 		// and then make assertions.
+//
+// 	}
+type PropertyMock struct {
+	// CompareToFunc mocks the CompareTo method.
+	CompareToFunc func(s string) (int, error)
+
+	// NameFunc mocks the Name method.
+	NameFunc func() string
+
+	// StringFunc mocks the String method.
+	StringFunc func() string
+
+	// ValidateFunc mocks the Validate method.
+	ValidateFunc func(s string) error
+
+	// ValueFunc mocks the Value method.
+	ValueFunc func() (string, error)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// CompareTo holds details about calls to the CompareTo method.
+		CompareTo []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Name holds details about calls to the Name method.
+		Name []struct {
+		}
+		// String holds details about calls to the String method.
+		String []struct {
+		}
+		// Validate holds details about calls to the Validate method.
+		Validate []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Value holds details about calls to the Value method.
+		Value []struct {
+		}
+	}
+	lockCompareTo sync.RWMutex
+	lockName      sync.RWMutex
+	lockString    sync.RWMutex
+	lockValidate  sync.RWMutex
+	lockValue     sync.RWMutex
+}
+
+// CompareTo calls CompareToFunc.
+func (mock *PropertyMock) CompareTo(s string) (int, error) {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockCompareTo.Lock()
+	mock.calls.CompareTo = append(mock.calls.CompareTo, callInfo)
+	mock.lockCompareTo.Unlock()
+	if mock.CompareToFunc == nil {
+		var (
+			nOut   int
+			errOut error
+		)
+		return nOut, errOut
+	}
+	return mock.CompareToFunc(s)
+}
+
+// CompareToCalls gets all the calls that were made to CompareTo.
+// Check the length with:
+//     len(mockedProperty.CompareToCalls())
+func (mock *PropertyMock) CompareToCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockCompareTo.RLock()
+	calls = mock.calls.CompareTo
+	mock.lockCompareTo.RUnlock()
+	return calls
+}
+
+// Name calls NameFunc.
+func (mock *PropertyMock) Name() string {
+	callInfo := struct {
+	}{}
+	mock.lockName.Lock()
+	mock.calls.Name = append(mock.calls.Name, callInfo)
+	mock.lockName.Unlock()
+	if mock.NameFunc == nil {
+		var (
+			sOut string
+		)
+		return sOut
+	}
+	return mock.NameFunc()
+}
+
+// NameCalls gets all the calls that were made to Name.
+// Check the length with:
+//     len(mockedProperty.NameCalls())
+func (mock *PropertyMock) NameCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockName.RLock()
+	calls = mock.calls.Name
+	mock.lockName.RUnlock()
+	return calls
+}
+
+// String calls StringFunc.
+func (mock *PropertyMock) String() string {
+	callInfo := struct {
+	}{}
+	mock.lockString.Lock()
+	mock.calls.String = append(mock.calls.String, callInfo)
+	mock.lockString.Unlock()
+	if mock.StringFunc == nil {
+		var (
+			sOut string
+		)
+		return sOut
+	}
+	return mock.StringFunc()
+}
+
+// StringCalls gets all the calls that were made to String.
+// Check the length with:
+//     len(mockedProperty.StringCalls())
+func (mock *PropertyMock) StringCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockString.RLock()
+	calls = mock.calls.String
+	mock.lockString.RUnlock()
+	return calls
+}
+
+// Validate calls ValidateFunc.
+func (mock *PropertyMock) Validate(s string) error {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockValidate.Lock()
+	mock.calls.Validate = append(mock.calls.Validate, callInfo)
+	mock.lockValidate.Unlock()
+	if mock.ValidateFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ValidateFunc(s)
+}
+
+// ValidateCalls gets all the calls that were made to Validate.
+// Check the length with:
+//     len(mockedProperty.ValidateCalls())
+func (mock *PropertyMock) ValidateCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockValidate.RLock()
+	calls = mock.calls.Validate
+	mock.lockValidate.RUnlock()
+	return calls
+}
+
+// Value calls ValueFunc.
+func (mock *PropertyMock) Value() (string, error) {
+	callInfo := struct {
+	}{}
+	mock.lockValue.Lock()
+	mock.calls.Value = append(mock.calls.Value, callInfo)
+	mock.lockValue.Unlock()
+	if mock.ValueFunc == nil {
+		var (
+			sOut   string
+			errOut error
+		)
+		return sOut, errOut
+	}
+	return mock.ValueFunc()
+}
+
+// ValueCalls gets all the calls that were made to Value.
+// Check the length with:
+//     len(mockedProperty.ValueCalls())
+func (mock *PropertyMock) ValueCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockValue.RLock()
+	calls = mock.calls.Value
+	mock.lockValue.RUnlock()
+	return calls
+}

internal/requirements/requirements.go

@@ -0,0 +1,70 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package requirements
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements/constraints"
+	"github.com/sirupsen/logrus"
+)
+
+// Requirements represents a collection of requirements that can be compared to properties
+type Requirements struct {
+	logger       *logrus.Logger
+	requirements []string
+	properties   map[string]constraints.Property
+}
+
+// New creates a new set of requirements
+func New(logger *logrus.Logger, requirements []string) *Requirements {
+	r := Requirements{
+		logger:       logger,
+		requirements: requirements,
+		properties: map[string]constraints.Property{
+			// Set up the supported properties. These are overridden with actual values.
+			CUDA:   constraints.NewVersionProperty(CUDA, ""),
+			ARCH:   constraints.NewVersionProperty(ARCH, ""),
+			DRIVER: constraints.NewVersionProperty(DRIVER, ""),
+			BRAND:  constraints.NewStringProperty(BRAND, ""),
+		},
+	}
+
+	return &r
+}
+
+// AddVersionProperty adds the specified property (name, value pair) to the requirements
+func (r *Requirements) AddVersionProperty(name string, value string) {
+	r.properties[name] = constraints.NewVersionProperty(name, value)
+}
+
+// AddStringProperty adds the specified property (name, value pair) to the requirements
+func (r *Requirements) AddStringProperty(name string, value string) {
+	r.properties[name] = constraints.NewStringProperty(name, value)
+}
+
+// Assert checks the specified requirements
+func (r Requirements) Assert() error {
+	if len(r.requirements) == 0 {
+		return nil
+	}
+
+	r.logger.Debugf("Checking properties %+v against requirements %v", r.properties, r.requirements)
+	c, err := constraints.New(r.logger, r.requirements, r.properties)
+	if err != nil {
+		return err
+	}
+	return c.Assert()
+}

internal/runtime/runtime_modifier.go

@@ -68,7 +68,7 @@ func (r *modifyingRuntimeWrapper) Exec(args []string) error {
 
 // modify loads, modifies, and flushes the OCI specification using the defined Modifier
 func (r *modifyingRuntimeWrapper) modify() error {
-	err := r.ociSpec.Load()
+	_, err := r.ociSpec.Load()
 	if err != nil {
 		return fmt.Errorf("error loading OCI specification for modification: %v", err)
 	}

packaging/debian/changelog

@@ -1,3 +1,20 @@
+nvidia-container-toolkit (1.10.0~rc.2-1) experimental; urgency=medium
+
+  * Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
+  * Switch to debug logging to reduce log verbosity
+  * Support logging to logs requested in command line
+  * Fix bug when launching containers with relative root path (e.g. using containerd)
+  * Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
+  * Fix failure to locate low-level runtime if PATH envvar is unset
+  * Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
+  * Use csv as default mode on Tegra systems without NVML
+  * Add --version flag to all CLIs
+  * [libnvidia-container] Bump libtirpc to 1.3.2
+  * [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
+  * [libnvidia-container] Add libcudadebugger.so to list of compute libraries
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Fri, 13 May 2022 13:41:10 +0200
+
 nvidia-container-toolkit (1.10.0~rc.1-1) experimental; urgency=medium
 
   * Include nvidia-ctk CLI in installed binaries

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -67,6 +67,20 @@ rm -f %{_bindir}/nvidia-container-runtime-hook
 /usr/share/containers/oci/hooks.d/oci-nvidia-hook.json
 
 %changelog
+* Fri May 13 2022 NVIDIA CORPORATION <cudatools@nvidia.com> 1.10.0-0.1.rc.2
+- Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
+- Switch to debug logging to reduce log verbosity
+- Support logging to logs requested in command line
+- Fix bug when launching containers with relative root path (e.g. using containerd)
+- Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
+- Fix failure to locate low-level runtime if PATH envvar is unset
+- Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
+- Use csv as default mode on Tegra systems without NVML
+- Add --version flag to all CLIs
+- [libnvidia-container] Bump libtirpc to 1.3.2
+- [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
+- [libnvidia-container] Add libcudadebugger.so to list of compute libraries
+
 * Thu Mar 24 2022 NVIDIA CORPORATION <cudatools@nvidia.com> 1.10.0-0.1.rc.1
 - Include nvidia-ctk CLI in installed binaries
 - Add experimental option to NVIDIA Container Runtime

scripts/Dockerfile.sign.deb

@@ -0,0 +1,4 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+    apt-get install -y apt-utils gpg xz-utils

scripts/Dockerfile.sign.rpm

@@ -0,0 +1,3 @@
+FROM centos:8
+
+RUN yum install -y createrepo rpm-sign pinentry

scripts/get-component-versions.sh

@@ -54,12 +54,16 @@ nvidia_docker_tag=${nvidia_container_toolkit_tag}
 nvidia_docker_version_tag="${nvidia_docker_version}${nvidia_docker_tag:+~${nvidia_docker_tag}}"
 
 echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version_tag}"
+echo "LIBNVIDIA_CONTAINER_PACKAGE_VERSION=${libnvidia_container_version_tag//\~/-}"
 echo "NVIDIA_CONTAINER_TOOLKIT_VERSION=${nvidia_container_toolkit_version}"
 echo "NVIDIA_CONTAINER_TOOLKIT_TAG=${nvidia_container_toolkit_tag}"
+echo "NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION=${nvidia_container_toolkit_version_tag//\~/-}"
 if [[ "${libnvidia_container_version_tag}" != "${nvidia_container_toolkit_version_tag}" ]]; then
     >&2 echo "WARNING: The libnvidia-container and nvidia-container-toolkit versions do not match"
 fi
 echo "NVIDIA_CONTAINER_RUNTIME_VERSION=${nvidia_container_runtime_version}"
 echo "NVIDIA_CONTAINER_RUNTIME_TAG=${nvidia_container_runtime_tag}"
+echo "NVIDIA_CONTAINER_RUNTIME_PACKAGE_VERSION=${nvidia_container_runtime_version_tag//\~/-}"
 echo "NVIDIA_DOCKER_VERSION=${nvidia_docker_version}"
 echo "NVIDIA_DOCKER_TAG=${nvidia_docker_tag}"
+echo "NVIDIA_DOCKER_PACKAGE_VERSION=${nvidia_docker_version_tag//\~/-}"

scripts/packages-sign-all.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+: "${ALL_DEBS:? Must set ALL_DEBS}"
+: "${ALL_RPMS:? Must set ALL_RPMS}"
+: "${GPG_LOCAL_USER:? Must set GPG_LOCAL_USER}"
+: "${TARGETS:? Must set TARGETS}"
+
+set -x -e
+
+function deb-sign {
+	local last_found
+	for r in ${*}; do
+		if [ -f "./${r}" ]; then
+			last_found=${r}
+		fi
+	done
+	if [[ -z ${last_found} ]]; then
+		echo "WARNING: No expected package found in $(pwd); skipping signing of repo;"
+		return
+	fi
+	apt-ftparchive packages . \
+		| tee Packages \
+		| xz > Packages.xz
+	apt-ftparchive -c repo.conf release . \
+		| gpg --batch --yes --expert --clearsign \
+			--armor \
+			--no-emit-version \
+			--no-comments \
+			--personal-digest-preferences sha512 \
+			--local-user ${GPG_LOCAL_USER} \
+		> InRelease
+}
+
+function rpm-sign {
+	for r in ${*}; do
+		if [ -f "./${r}" ]; then
+			rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}"
+		fi
+	done
+	createrepo -v --no-database -s sha512 --compress-type xz --revision "1.0" .
+	gpg2 --batch --yes --expert --sign --detach-sign \
+		--armor \
+		--no-emit-version \
+		--no-comments --personal-digest-preferences sha512 \
+		--local-user ${GPG_LOCAL_USER} \
+	repodata/repomd.xml
+}
+
+function sign() {
+	local target=$1
+    local dst_root=$2
+
+	local src_dist=${target%-*}
+    local dist=${src_dist/amazonlinux/amzn}
+
+	local pkg_type
+    case ${src_dist} in
+    amazonlinux*) pkg_type=rpm
+        ;;
+    centos*) pkg_type=rpm
+        ;;
+    debian*) pkg_type=deb
+        ;;
+    opensuse-leap*) pkg_type=rpm
+        ;;
+    ubuntu*) pkg_type=deb
+        ;;
+    *) echo "ERROR: unexpected distribution ${src_dist}"
+        ;;
+    esac
+
+    local arch=${target##*-}
+    case ${src_dist} in
+    ubuntu*) arch=${arch//ppc64le/ppc64el}
+    esac
+
+    local dst=${dst_root}/${dist}/${arch}
+
+	if [[ ! -d ${dst} ]]; then
+		echo "Directory ${dst} not found. Skipping"
+		return
+	fi
+
+	cd ${dst}
+	if [[ -f "/etc/debian_version" ]]; then
+		[[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS}
+	else
+		[[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS}
+	fi
+	cd -
+}
+
+for target in ${TARGETS[@]}; do
+    sign ${target} $(pwd)
+done

scripts/release-packages.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function assert_usage() {
+    echo "Incorrect arguments: $*"
+    echo "$(basename ${BASH_SOURCE[0]}) PACKAGE_REPO_ROOT [SHA]"
+    echo "\tPACKAGE_REPO_ROOT: The path to the libnvidia-container repository"
+    echo "\tSHA: The SHA / reference to release. [Default: HEAD]"
+    exit 1
+}
+
+set -e
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+    assert_usage $*
+fi
+
+PACKAGE_REPO_ROOT=$1
+if [[ ! -d ${PACKAGE_REPO_ROOT} ]]; then
+    echo "The specified PACKAGE_REPO_ROOT '${PACKAGE_REPO_ROOT}' must exist"
+    exit 1
+fi
+
+: ${REFERENCE:="HEAD"}
+if [[ $# -ge 2 ]]; then
+    REFERENCE=$2
+fi
+
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+
+TAG=v"${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}"
+SHA=$(git rev-parse --short=8 ${REFERENCE})
+
+REPO="experimental"
+if [[ ${TAG/rc./} == ${TAG} ]]; then
+    REPO="stable"
+fi
+
+PACKAGE_CACHE=release-${TAG}-${REPO}
+
+echo "Fetching packages with SHA '${SHA}' as tag '${TAG}' to ${PACKAGE_CACHE}"
+IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit"
+IMAGE_TAG=${SHA}-packaging
+${SCRIPTS_DIR}/pull-packages.sh \
+    ${IMAGE_NAME}:${IMAGE_TAG} \
+    ${PACKAGE_CACHE}
+
+: ${ALL_RPMS:="$(find ${PACKAGE_CACHE} -name "*.rpm" -exec basename {} \; | sort | uniq)"}
+: ${ALL_DEBS:="$(find ${PACKAGE_CACHE} -name "*.deb" -exec basename {} \; | sort | uniq)"}
+
+
+PACKAGE_REPO_ROOT=$(cd "${PACKAGE_REPO_ROOT}" && pwd)
+echo "Updating ${REPO} repo at ${PACKAGE_REPO_ROOT}"
+
+docker build \
+    -t nvidia/toolkit-deb-pkg-signer \
+    -f ${SCRIPTS_DIR}/Dockerfile.sign.deb \
+        ${SCRIPTS_DIR}
+
+docker build \
+    -t nvidia/toolkit-rpm-pkg-signer \
+    -f ${SCRIPTS_DIR}/Dockerfile.sign.rpm \
+        ${SCRIPTS_DIR}
+
+function sync() {
+    local target=$1
+    local src_root=$2
+    local dst_root=$3
+
+    local src_dist=${target%-*}
+    local dst_dist=${src_dist/amazonlinux/amzn}
+
+    local pkg_type
+    case ${src_dist} in
+    amazonlinux*) pkg_type=rpm
+        ;;
+    centos*) pkg_type=rpm
+        ;;
+    debian*) pkg_type=deb
+        ;;
+    opensuse-leap*) pkg_type=rpm
+        ;;
+    ubuntu*) pkg_type=deb
+        ;;
+    *) echo "ERROR: unexpected distribution ${src_dist}"
+        ;;
+    esac
+
+    local arch=${target##*-}
+    local dst_arch=${arch}
+    case ${src_dist} in
+    ubuntu*) dst_arch=${arch//ppc64le/ppc64el}
+    esac
+
+    local src=${src_root}/${src_dist}/${arch}
+    local dst=${dst_root}/${dst_dist}/${dst_arch}
+
+    if [[ ! -d ${src} || -z $(ls ${src}/*.${pkg_type}) ]]; then
+        echo "Skipping ${src}"
+        return
+    fi
+    mkdir -p ${dst}
+    cp ${src}/libnvidia-container*.${pkg_type} ${dst}
+    cp ${src}/nvidia-container-toolkit*.${pkg_type} ${dst}
+    if [[ ${REPO} == "stable" ]]; then
+        cp ${src}/nvidia-container-runtime*.${pkg_type} ${dst}
+        cp ${src}/nvidia-docker*.${pkg_type} ${dst}
+    fi
+}
+
+# This list represents the distribution-architecture pairs that are actually published
+# to the relevant repositories. This targets forwarded to the build-all-components script
+# can be overridden by specifying command line arguments.
+all=(
+    amazonlinux2-aarch64
+    amazonlinux2-x86_64
+    centos7-ppc64le
+    centos7-x86_64
+    centos8-aarch64
+    centos8-ppc64le
+    centos8-x86_64
+    debian10-amd64
+    debian9-amd64
+    opensuse-leap15.1-x86_64
+    ubuntu16.04-amd64
+    ubuntu16.04-ppc64le
+    ubuntu18.04-amd64
+    ubuntu18.04-arm64
+    ubuntu18.04-ppc64le
+)
+
+targets=${all[@]}
+
+_current_branch=$(git -C ${PACKAGE_REPO_ROOT} rev-parse --abbrev-ref HEAD)
+if [[ x"${_current_branch}" != x"gh-pages" ]]; then
+    echo "It is expected that the gh-pages branch be checked out"
+    exit 1
+fi
+
+: ${UPSTREAM_REMOTE:="origin"}
+_remote_name=$( git remote -v | grep "git@gitlab.com:nvidia/container-toolkit/libnvidia-container.git (push)" | cut -d$'\t' -f1 )
+if [[ x"${_remote_name}" != x"${UPSTREAM_REMOTE}" ]]; then
+    echo "Identified ${_remote_name} as git@gitlab.com:nvidia/container-toolkit/libnvidia-container.git remote."
+    echo "Set UPSTREAM_REMOTE=${_remote_name} instead of ${UPSTREAM_REMOTE}"
+fi
+
+: ${UPSTREAM_REFERENCE:="${UPSTREAM_REMOTE}/gh-pages"}
+git -C ${PACKAGE_REPO_ROOT} reset --hard ${UPSTREAM_REFERENCE}
+git -C ${PACKAGE_REPO_ROOT} clean -fdx ${REPO}
+
+for target in ${targets[@]}; do
+    sync ${target} ${PACKAGE_CACHE} ${PACKAGE_REPO_ROOT}/${REPO}
+done
+
+git -C ${PACKAGE_REPO_ROOT} add ${REPO}
+
+if [[ ${REPO} == "stable" ]]; then
+# Stable release
+git -C ${PACKAGE_REPO_ROOT} commit -s -F- <<EOF
+Add packages for NVIDIA Container Toolkit ${TAG} release
+
+These include:
+* libnvidia-container* ${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}
+* nvidia-container-toolkit ${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}
+* nvidia-container-runtime ${NVIDIA_CONTAINER_RUNTIME_PACKAGE_VERSION}
+* nvidia-docker ${NVIDIA_DOCKER_PACKAGE_VERSION}
+EOF
+else
+# Experimental / release candidate release
+git -C ${PACKAGE_REPO_ROOT} commit -s -F- <<EOF
+Add packages for NVIDIA Container Toolkit ${TAG} ${REPO} release
+
+These include:
+* libnvidia-container* ${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}
+* nvidia-container-toolkit ${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}
+EOF
+fi
+
+: ${MASTER_KEY_PATH:? Path to master key MASTER_KEY_PATH must be set}
+: ${SUB_KEY_PATH:? Path to sub key SUB_KEY_PATH must be set}
+: ${GPG_LOCAL_USER:? GPG_LOCAL_USER must be set}
+: ${GNUPG_CONF:=$(mktemp -d -t nvidia-container-toolkit-package-XXXXXXXXXX)}
+
+function sign() {
+    local pkg_type=$1
+    docker run -it --rm \
+        -e ALL_DEBS="${ALL_DEBS}" \
+        -e ALL_RPMS="${ALL_RPMS}" \
+        -e GPG_LOCAL_USER="${GPG_LOCAL_USER}" \
+        -e TARGETS="${targets}" \
+        -v ${PACKAGE_REPO_ROOT}/${REPO}:/sign-packages \
+        -v ${MASTER_KEY_PATH}:/keys/master.key:ro \
+        -v ${SUB_KEY_PATH}:/keys/sub.key:ro \
+        -v ${SCRIPTS_DIR}:/helpers \
+        -w /sign-packages \
+            nvidia/toolkit-${pkg_type}-pkg-signer \
+        bash -x -c "
+        export GPG_TTY=\$(tty);
+        gpg --import /keys/master.key;
+        gpg --import /keys/sub.key;
+        /helpers/packages-sign-all.sh;
+        "
+
+}
+
+sign deb
+
+git -C ${PACKAGE_REPO_ROOT} add ${REPO}
+git -C ${PACKAGE_REPO_ROOT} commit -s -m "TOFIX: Sign deb packages for ${TAG} in ${REPO}"
+
+sign rpm
+
+git -C ${PACKAGE_REPO_ROOT} add ${REPO}
+git -C ${PACKAGE_REPO_ROOT} commit -s -m "TOFIX: Sign rpm packages for ${TAG} in ${REPO}"
+
+echo "To publish changes, go to ${PACKAGE_REPO_ROOT} and run: "
+echo "   git rebase -i ${UPSTREAM_REFERENCE}"

test/release/Makefile

@@ -56,4 +56,4 @@ $(RUN_TARGETS): run-%: image-%
 
 # Ensure that the local package root exists
 $(RELEASE_TARGETS): release-%: $(LOCAL_PACKAGE_ROOT)/$(*)/$(ARCH)
-	$(PROJECT_ROOT)/scripts/release.sh $(*)-$(ARCH)
+	$(PROJECT_ROOT)/scripts/build-packages.sh $(*)-$(ARCH)

vendor/github.com/NVIDIA/go-nvml/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/NVIDIA/go-nvml/pkg/dl/dl.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dl
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// #cgo LDFLAGS: -ldl
+// #include <dlfcn.h>
+// #include <stdlib.h>
+import "C"
+
+const (
+	RTLD_LAZY = C.RTLD_LAZY
+	RTLD_NOW = C.RTLD_NOW
+	RTLD_GLOBAL = C.RTLD_GLOBAL
+	RTLD_LOCAL = C.RTLD_LOCAL
+	RTLD_NODELETE = C.RTLD_NODELETE
+	RTLD_NOLOAD = C.RTLD_NOLOAD
+	RTLD_DEEPBIND = C.RTLD_DEEPBIND
+)
+
+type DynamicLibrary struct{
+	Name string
+	Flags int
+	handle unsafe.Pointer
+}
+
+func New(name string, flags int) *DynamicLibrary {
+	return &DynamicLibrary{
+		Name: name,
+		Flags: flags,
+		handle: nil,
+    }
+}
+
+func (dl *DynamicLibrary) Open() error {
+	name := C.CString(dl.Name)
+	defer C.free(unsafe.Pointer(name))
+
+	handle := C.dlopen(name, C.int(dl.Flags))
+	if handle == C.NULL {
+		return fmt.Errorf("%s", C.GoString(C.dlerror()))
+	}
+	dl.handle = handle
+	return nil
+}
+
+func (dl *DynamicLibrary) Close() error {
+	err := C.dlclose(dl.handle)
+	if err != 0 {
+		return fmt.Errorf("%s", C.GoString(C.dlerror()))
+	}
+	return nil
+}
+
+func (dl *DynamicLibrary) Lookup(symbol string) error {
+	sym := C.CString(symbol)
+	defer C.free(unsafe.Pointer(sym))
+
+	C.dlerror() // Clear out any previous errors
+	C.dlsym(dl.handle, sym)
+	err := C.dlerror()
+	if unsafe.Pointer(err) == C.NULL {
+		return nil
+	}
+	return fmt.Errorf("%s", C.GoString(err))
+}

vendor/modules.txt

@@ -2,15 +2,16 @@
 ## explicit
 github.com/BurntSushi/toml
 github.com/BurntSushi/toml/internal
+# github.com/NVIDIA/go-nvml v0.11.6-0
+## explicit
+github.com/NVIDIA/go-nvml/pkg/dl
 # github.com/blang/semver v3.5.1+incompatible
 github.com/blang/semver
 # github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
 ## explicit
 github.com/container-orchestrated-devices/container-device-interface/pkg/cdi
 github.com/container-orchestrated-devices/container-device-interface/specs-go
-# github.com/containers/podman/v2 v2.2.1
-## explicit
-# github.com/containers/podman/v4 v4.0.1
+# github.com/containers/podman/v4 v4.0.3
 ## explicit
 github.com/containers/podman/v4/pkg/hooks/1.0.0
 # github.com/cpuguy83/go-md2man/v2 v2.0.1

versions.mk

@@ -14,7 +14,7 @@
 
 LIB_NAME := nvidia-container-toolkit
 LIB_VERSION := 1.10.0
-LIB_TAG := rc.1
+LIB_TAG := rc.2
 
 # Specify the nvidia-docker2 and nvidia-container-runtime package versions.
 # Note: The tag is automatically specified to match LIB_TAG.
@@ -25,4 +25,6 @@ NVIDIA_CONTAINER_RUNTIME_VERSION := 3.10.0
 LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack
 
 CUDA_VERSION := 11.6.0
-GOLANG_VERSION := 1.16.4
+GOLANG_VERSION := 1.17.8
+
+GIT_COMMIT ?= $(shell git describe --dirty --long --always 2> /dev/null || echo "")