Rename -toolkit executable to -runtime-hook

This change renames the nvidia-container-toolkit executable
to nvidia-container-runtime-hook. Here nvidia-container-toolkit
is created as a symlink to nvidia-container-runtime-hook.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

cmd/nvidia-container-runtime-hook/capabilities.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"log"
+	"strings"
+)
+
+const (
+	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
+	defaultDriverCapabilities = DriverCapabilities("utility,compute")
+
+	none = DriverCapabilities("")
+	all  = DriverCapabilities("all")
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}
+
+// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
+// variable. Operations include default values, filtering, and handling meta values such as "all"
+type DriverCapabilities string
+
+// Intersection returns intersection between two sets of capabilities.
+func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
+	if capabilities == all {
+		return d
+	}
+	if d == all {
+		return capabilities
+	}
+
+	lookup := make(map[string]bool)
+	for _, c := range d.list() {
+		lookup[c] = true
+	}
+	var found []string
+	for _, c := range capabilities.list() {
+		if lookup[c] {
+			found = append(found, c)
+		}
+	}
+
+	intersection := DriverCapabilities(strings.Join(found, ","))
+	return intersection
+}
+
+// String returns the string representation of the driver capabilities
+func (d DriverCapabilities) String() string {
+	return string(d)
+}
+
+// list returns the driver capabilities as a list
+func (d DriverCapabilities) list() []string {
+	var caps []string
+	for _, c := range strings.Split(string(d), ",") {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) == 0 {
+			continue
+		}
+		caps = append(caps, trimmed)
+	}
+
+	return caps
+}

cmd/nvidia-container-runtime-hook/capabilities_test.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDriverCapabilitiesIntersection(t *testing.T) {
+	testCases := []struct {
+		capabilities          DriverCapabilities
+		supportedCapabilities DriverCapabilities
+		expectedIntersection  DriverCapabilities
+	}{
+		{
+			capabilities:          none,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          allDriverCapabilities,
+			supportedCapabilities: all,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: all,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: DriverCapabilities("cap1"),
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("cap0,cap1"),
+			supportedCapabilities: DriverCapabilities("cap1,cap0"),
+			expectedIntersection:  DriverCapabilities("cap0,cap1"),
+		},
+		{
+			capabilities:          defaultDriverCapabilities,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  defaultDriverCapabilities,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+		{
+			capabilities:          DriverCapabilities("cap1"),
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
+			require.EqualValues(t, tc.expectedIntersection, intersection)
+		})
+	}
+}
+
+func TestDriverCapabilitiesList(t *testing.T) {
+	testCases := []struct {
+		capabilities DriverCapabilities
+		expected     []string
+	}{
+		{
+			capabilities: DriverCapabilities(""),
+		},
+		{
+			capabilities: DriverCapabilities("  "),
+		},
+		{
+			capabilities: DriverCapabilities(","),
+		},
+		{
+			capabilities: DriverCapabilities(",cap"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap,"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap0,,cap1"),
+			expected:     []string{"cap0", "cap1"},
+		},
+		{
+			capabilities: DriverCapabilities("cap1,cap0,cap3"),
+			expected:     []string{"cap1", "cap0", "cap3"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			require.EqualValues(t, tc.expected, tc.capabilities.list())
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"golang.org/x/mod/semver"
+)
+
+var envSwarmGPU *string
+
+const (
+	envCUDAVersion          = "CUDA_VERSION"
+	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
+	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
+	envNVMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
+	envNVMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
+	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+const (
+	deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
+)
+
+type nvidiaConfig struct {
+	Devices            string
+	MigConfigDevices   string
+	MigMonitorDevices  string
+	DriverCapabilities string
+	Requirements       []string
+	DisableRequire     bool
+}
+
+type containerConfig struct {
+	Pid    int
+	Rootfs string
+	Env    map[string]string
+	Nvidia *nvidiaConfig
+}
+
+// Root from OCI runtime spec
+// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L94-L100
+type Root struct {
+	Path string `json:"path"`
+}
+
+// Process from OCI runtime spec
+// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L57
+type Process struct {
+	Env          []string         `json:"env,omitempty"`
+	Capabilities *json.RawMessage `json:"capabilities,omitempty" platform:"linux"`
+}
+
+// LinuxCapabilities from OCI runtime spec
+// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L61
+type LinuxCapabilities struct {
+	Bounding    []string `json:"bounding,omitempty" platform:"linux"`
+	Effective   []string `json:"effective,omitempty" platform:"linux"`
+	Inheritable []string `json:"inheritable,omitempty" platform:"linux"`
+	Permitted   []string `json:"permitted,omitempty" platform:"linux"`
+	Ambient     []string `json:"ambient,omitempty" platform:"linux"`
+}
+
+// Mount from OCI runtime spec
+// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L103
+type Mount struct {
+	Destination string   `json:"destination"`
+	Type        string   `json:"type,omitempty" platform:"linux,solaris"`
+	Source      string   `json:"source,omitempty"`
+	Options     []string `json:"options,omitempty"`
+}
+
+// Spec from OCI runtime spec
+// We use pointers to structs, similarly to the latest version of runtime-spec:
+// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L5-L28
+type Spec struct {
+	Version *string  `json:"ociVersion"`
+	Process *Process `json:"process,omitempty"`
+	Root    *Root    `json:"root,omitempty"`
+	Mounts  []Mount  `json:"mounts,omitempty"`
+}
+
+// HookState holds state information about the hook
+type HookState struct {
+	Pid int `json:"pid,omitempty"`
+	// After 17.06, runc is using the runtime spec:
+	// github.com/docker/runc/blob/17.06/libcontainer/configs/config.go#L262-L263
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/state.go#L3-L17
+	Bundle string `json:"bundle"`
+	// Before 17.06, runc used a custom struct that didn't conform to the spec:
+	// github.com/docker/runc/blob/17.03.x/libcontainer/configs/config.go#L245-L252
+	BundlePath string `json:"bundlePath"`
+}
+
+func loadSpec(path string) (spec *Spec) {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Panicln("could not open OCI spec:", err)
+	}
+	defer f.Close()
+
+	if err = json.NewDecoder(f).Decode(&spec); err != nil {
+		log.Panicln("could not decode OCI spec:", err)
+	}
+	if spec.Version == nil {
+		log.Panicln("Version is empty in OCI spec")
+	}
+	if spec.Process == nil {
+		log.Panicln("Process is empty in OCI spec")
+	}
+	if spec.Root == nil {
+		log.Panicln("Root is empty in OCI spec")
+	}
+	return
+}
+
+func isPrivileged(s *Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	var caps []string
+	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
+	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
+	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
+	if (rc1cmp == 1 || rc1cmp == 0) && (rc5cmp == -1) {
+		err := json.Unmarshal(*s.Process.Capabilities, &caps)
+		if err != nil {
+			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		}
+		// Otherwise, parse s.Process.Capabilities as:
+		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	} else {
+		var lc LinuxCapabilities
+		err := json.Unmarshal(*s.Process.Capabilities, &lc)
+		if err != nil {
+			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		}
+		// We only make sure that the bounding capabibility set has
+		// CAP_SYS_ADMIN. This allows us to make sure that the container was
+		// actually started as '--privileged', but also allow non-root users to
+		// access the privileged NVIDIA capabilities.
+		caps = lc.Bounding
+	}
+
+	for _, c := range caps {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getDevicesFromEnvvar(image image.CUDA) *string {
+	// Build a list of envvars to consider.
+	envVars := []string{envNVVisibleDevices}
+	if envSwarmGPU != nil {
+		// The Swarm envvar has higher precedence.
+		envVars = append([]string{*envSwarmGPU}, envVars...)
+	}
+
+	devices := image.DevicesFromEnvvars(envVars...)
+	if len(devices) == 0 {
+		return nil
+	}
+
+	devicesString := strings.Join(devices, ",")
+
+	return &devicesString
+}
+
+func getDevicesFromMounts(mounts []Mount) *string {
+	var devices []string
+	for _, m := range mounts {
+		root := filepath.Clean(deviceListAsVolumeMountsRoot)
+		source := filepath.Clean(m.Source)
+		destination := filepath.Clean(m.Destination)
+
+		// Only consider mounts who's host volume is /dev/null
+		if source != "/dev/null" {
+			continue
+		}
+		// Only consider container mount points that begin with 'root'
+		if len(destination) < len(root) {
+			continue
+		}
+		if destination[:len(root)] != root {
+			continue
+		}
+		// Grab the full path beyond 'root' and add it to the list of devices
+		device := destination[len(root):]
+		if len(device) > 0 && device[0] == '/' {
+			device = device[1:]
+		}
+		if len(device) == 0 {
+			continue
+		}
+		devices = append(devices, device)
+	}
+
+	if devices == nil {
+		return nil
+	}
+
+	ret := strings.Join(devices, ",")
+	return &ret
+}
+
+func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
+	// If enabled, try and get the device list from volume mounts first
+	if hookConfig.AcceptDeviceListAsVolumeMounts {
+		devices := getDevicesFromMounts(mounts)
+		if devices != nil {
+			return devices
+		}
+	}
+
+	// Fallback to reading from the environment variable if privileges are correct
+	devices := getDevicesFromEnvvar(image)
+	if devices == nil {
+		return nil
+	}
+	if privileged || hookConfig.AcceptEnvvarUnprivileged {
+		return devices
+	}
+
+	configName := hookConfig.getConfigOption("AcceptEnvvarUnprivileged")
+	log.Printf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES (privileged=%v, %v=%v) ", privileged, configName, hookConfig.AcceptEnvvarUnprivileged)
+
+	return nil
+}
+
+func getMigConfigDevices(env map[string]string) *string {
+	if devices, ok := env[envNVMigConfigDevices]; ok {
+		return &devices
+	}
+	return nil
+}
+
+func getMigMonitorDevices(env map[string]string) *string {
+	if devices, ok := env[envNVMigMonitorDevices]; ok {
+		return &devices
+	}
+	return nil
+}
+
+func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
+	// We use the default driver capabilities by default. This is filtered to only include the
+	// supported capabilities
+	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
+
+	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
+
+	if !capsEnvSpecified && legacyImage {
+		// Environment variable unset with legacy image: set all capabilities.
+		return supportedDriverCapabilities
+	}
+
+	if capsEnvSpecified && len(capsEnv) > 0 {
+		// If the envvironment variable is specified and is non-empty, use the capabilities value
+		envCapabilities := DriverCapabilities(capsEnv)
+		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
+		if envCapabilities != all && capabilities != envCapabilities {
+			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
+		}
+	}
+
+	return capabilities
+}
+
+func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
+	legacyImage := image.IsLegacy()
+
+	var devices string
+	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
+		devices = *d
+	} else {
+		// 'nil' devices means this is not a GPU container.
+		return nil
+	}
+
+	var migConfigDevices string
+	if d := getMigConfigDevices(image); d != nil {
+		migConfigDevices = *d
+	}
+	if !privileged && migConfigDevices != "" {
+		log.Panicln("cannot set MIG_CONFIG_DEVICES in non privileged container")
+	}
+
+	var migMonitorDevices string
+	if d := getMigMonitorDevices(image); d != nil {
+		migMonitorDevices = *d
+	}
+	if !privileged && migMonitorDevices != "" {
+		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
+	}
+
+	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+
+	requirements, err := image.GetRequirements()
+	if err != nil {
+		log.Panicln("failed to get requirements", err)
+	}
+
+	disableRequire := image.HasDisableRequire()
+
+	return &nvidiaConfig{
+		Devices:            devices,
+		MigConfigDevices:   migConfigDevices,
+		MigMonitorDevices:  migMonitorDevices,
+		DriverCapabilities: driverCapabilities,
+		Requirements:       requirements,
+		DisableRequire:     disableRequire,
+	}
+}
+
+func getContainerConfig(hook HookConfig) (config containerConfig) {
+	var h HookState
+	d := json.NewDecoder(os.Stdin)
+	if err := d.Decode(&h); err != nil {
+		log.Panicln("could not decode container state:", err)
+	}
+
+	b := h.Bundle
+	if len(b) == 0 {
+		b = h.BundlePath
+	}
+
+	s := loadSpec(path.Join(b, "config.json"))
+
+	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	privileged := isPrivileged(s)
+	envSwarmGPU = hook.SwarmResource
+	return containerConfig{
+		Pid:    h.Pid,
+		Rootfs: s.Root.Path,
+		Env:    image,
+		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
+	}
+}

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -0,0 +1,991 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetNvidiaConfig(t *testing.T) {
+	var tests = []struct {
+		description    string
+		env            map[string]string
+		privileged     bool
+		hookConfig     *HookConfig
+		expectedConfig *nvidiaConfig
+		expectedPanic  bool
+	}{
+		{
+			description:    "No environment, unprivileged",
+			env:            map[string]string{},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description:    "No environment, privileged",
+			env:            map[string]string{},
+			privileged:     true,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, no devices, no capabilities, no requirements",
+			env: map[string]string{
+				envCUDAVersion: "9.0",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: allDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices 'all', no capabilities, no requirements",
+			env: map[string]string{
+				envCUDAVersion:      "9.0",
+				envNVVisibleDevices: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: allDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices 'empty', no capabilities, no requirements",
+			env: map[string]string{
+				envCUDAVersion:      "9.0",
+				envNVVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, devices 'void', no capabilities, no requirements",
+			env: map[string]string{
+				envCUDAVersion:      "9.0",
+				envNVVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, devices 'none', no capabilities, no requirements",
+			env: map[string]string{
+				envCUDAVersion:      "9.0",
+				envNVVisibleDevices: "none",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "",
+				DriverCapabilities: allDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices set, no capabilities, no requirements",
+			env: map[string]string{
+				envCUDAVersion:      "9.0",
+				envNVVisibleDevices: "gpu0,gpu1",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: allDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities 'empty', no requirements",
+			env: map[string]string{
+				envCUDAVersion:          "9.0",
+				envNVVisibleDevices:     "gpu0,gpu1",
+				envNVDriverCapabilities: "",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities 'all', no requirements",
+			env: map[string]string{
+				envCUDAVersion:          "9.0",
+				envNVVisibleDevices:     "gpu0,gpu1",
+				envNVDriverCapabilities: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: allDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, no requirements",
+			env: map[string]string{
+				envCUDAVersion:          "9.0",
+				envNVVisibleDevices:     "gpu0,gpu1",
+				envNVDriverCapabilities: "video,display",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: "video,display",
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, requirements set",
+			env: map[string]string{
+				envCUDAVersion:              "9.0",
+				envNVVisibleDevices:         "gpu0,gpu1",
+				envNVDriverCapabilities:     "video,display",
+				envNVRequirePrefix + "REQ0": "req0=true",
+				envNVRequirePrefix + "REQ1": "req1=false",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: "video,display",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, requirements set, disable requirements",
+			env: map[string]string{
+				envCUDAVersion:              "9.0",
+				envNVVisibleDevices:         "gpu0,gpu1",
+				envNVDriverCapabilities:     "video,display",
+				envNVRequirePrefix + "REQ0": "req0=true",
+				envNVRequirePrefix + "REQ1": "req1=false",
+				envNVDisableRequire:         "true",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: "video,display",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+				DisableRequire:     true,
+			},
+		},
+		{
+			description: "Modern image, no devices, no capabilities, no requirements, no envCUDAVersion",
+			env: map[string]string{
+				envNVRequireCUDA: "cuda>=9.0",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, no devices, no capabilities, no requirement, envCUDAVersion set",
+			env: map[string]string{
+				envCUDAVersion:   "9.0",
+				envNVRequireCUDA: "cuda>=9.0",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'all', no capabilities, no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:    "cuda>=9.0",
+				envNVVisibleDevices: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices 'empty', no capabilities, no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:    "cuda>=9.0",
+				envNVVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'void', no capabilities, no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:    "cuda>=9.0",
+				envNVVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'none', no capabilities, no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:    "cuda>=9.0",
+				envNVVisibleDevices: "none",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices set, no capabilities, no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:    "cuda>=9.0",
+				envNVVisibleDevices: "gpu0,gpu1",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities 'empty', no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:        "cuda>=9.0",
+				envNVVisibleDevices:     "gpu0,gpu1",
+				envNVDriverCapabilities: "",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities 'all', no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:        "cuda>=9.0",
+				envNVVisibleDevices:     "gpu0,gpu1",
+				envNVDriverCapabilities: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: allDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, no requirements",
+			env: map[string]string{
+				envNVRequireCUDA:        "cuda>=9.0",
+				envNVVisibleDevices:     "gpu0,gpu1",
+				envNVDriverCapabilities: "video,display",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: "video,display",
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, requirements set",
+			env: map[string]string{
+				envNVRequireCUDA:            "cuda>=9.0",
+				envNVVisibleDevices:         "gpu0,gpu1",
+				envNVDriverCapabilities:     "video,display",
+				envNVRequirePrefix + "REQ0": "req0=true",
+				envNVRequirePrefix + "REQ1": "req1=false",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: "video,display",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, requirements set, disable requirements",
+			env: map[string]string{
+				envNVRequireCUDA:            "cuda>=9.0",
+				envNVVisibleDevices:         "gpu0,gpu1",
+				envNVDriverCapabilities:     "video,display",
+				envNVRequirePrefix + "REQ0": "req0=true",
+				envNVRequirePrefix + "REQ1": "req1=false",
+				envNVDisableRequire:         "true",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "gpu0,gpu1",
+				DriverCapabilities: "video,display",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+				DisableRequire:     true,
+			},
+		},
+		{
+			description: "No cuda envs, devices 'all'",
+			env: map[string]string{
+				envNVVisibleDevices: "all",
+			},
+			privileged: false,
+
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migConfig set, privileged",
+			env: map[string]string{
+				envNVRequireCUDA:      "cuda>=9.0",
+				envNVVisibleDevices:   "all",
+				envNVMigConfigDevices: "mig0,mig1",
+			},
+			privileged: true,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				MigConfigDevices:   "mig0,mig1",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migConfig set, unprivileged",
+			env: map[string]string{
+				envNVRequireCUDA:      "cuda>=9.0",
+				envNVVisibleDevices:   "all",
+				envNVMigConfigDevices: "mig0,mig1",
+			},
+			privileged:    false,
+			expectedPanic: true,
+		},
+		{
+			description: "Modern image, devices 'all', migMonitor set, privileged",
+			env: map[string]string{
+				envNVRequireCUDA:       "cuda>=9.0",
+				envNVVisibleDevices:    "all",
+				envNVMigMonitorDevices: "mig0,mig1",
+			},
+			privileged: true,
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				MigMonitorDevices:  "mig0,mig1",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+				DisableRequire:     false,
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migMonitor set, unprivileged",
+			env: map[string]string{
+				envNVRequireCUDA:       "cuda>=9.0",
+				envNVVisibleDevices:    "all",
+				envNVMigMonitorDevices: "mig0,mig1",
+			},
+			privileged:    false,
+			expectedPanic: true,
+		},
+		{
+			description: "Hook config set as driver-capabilities-all",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				envNVDriverCapabilities: "all",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: "video,display",
+			},
+		},
+		{
+			description: "Hook config set, envvar sets driver-capabilities",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				envNVDriverCapabilities: "video,display",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display,compute,utility",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: "video,display",
+			},
+		},
+		{
+			description: "Hook config set, envvar unset sets default driver-capabilities",
+			env: map[string]string{
+				envNVVisibleDevices: "all",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			// Wrap the call to getNvidiaConfig() in a closure.
+			var config *nvidiaConfig
+			getConfig := func() {
+				hookConfig := tc.hookConfig
+				if hookConfig == nil {
+					defaultConfig := getDefaultHookConfig()
+					hookConfig = &defaultConfig
+				}
+				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
+			}
+
+			// For any tests that are expected to panic, make sure they do.
+			if tc.expectedPanic {
+				require.Panics(t, getConfig)
+				return
+			}
+
+			// For all other tests, just grab the config
+			getConfig()
+
+			// And start comparing the test results to the expected results.
+			if tc.expectedConfig == nil {
+				require.Nil(t, config, tc.description)
+				return
+			}
+
+			require.NotNil(t, config, tc.description)
+
+			require.Equal(t, tc.expectedConfig.Devices, config.Devices)
+			require.Equal(t, tc.expectedConfig.MigConfigDevices, config.MigConfigDevices)
+			require.Equal(t, tc.expectedConfig.MigMonitorDevices, config.MigMonitorDevices)
+			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
+
+			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
+			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
+		})
+	}
+}
+
+func TestGetDevicesFromMounts(t *testing.T) {
+	var tests = []struct {
+		description     string
+		mounts          []Mount
+		expectedDevices *string
+	}{
+		{
+			description:     "No mounts",
+			mounts:          nil,
+			expectedDevices: nil,
+		},
+		{
+			description: "Host path is not /dev/null",
+			mounts: []Mount{
+				{
+					Source:      "/not/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
+				},
+			},
+			expectedDevices: nil,
+		},
+		{
+			description: "Container path is not prefixed by 'root'",
+			mounts: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join("/other/prefix", "GPU0"),
+				},
+			},
+			expectedDevices: nil,
+		},
+		{
+			description: "Container path is only 'root'",
+			mounts: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: deviceListAsVolumeMountsRoot,
+				},
+			},
+			expectedDevices: nil,
+		},
+		{
+			description: "Discover 2 devices",
+			mounts: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			expectedDevices: &[]string{"GPU0,GPU1"}[0],
+		},
+		{
+			description: "Discover 2 devices with slashes in the name",
+			mounts: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0-MIG0/0/1"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1-MIG0/0/1"),
+				},
+			},
+			expectedDevices: &[]string{"GPU0-MIG0/0/1,GPU1-MIG0/0/1"}[0],
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			devices := getDevicesFromMounts(tc.mounts)
+			require.Equal(t, tc.expectedDevices, devices)
+		})
+	}
+}
+
+func TestDeviceListSourcePriority(t *testing.T) {
+	var tests = []struct {
+		description        string
+		mountDevices       []Mount
+		envvarDevices      string
+		privileged         bool
+		acceptUnprivileged bool
+		acceptMounts       bool
+		expectedDevices    *string
+	}{
+		{
+			description: "Mount devices, unprivileged, no accept unprivileged",
+			mountDevices: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			envvarDevices:      "GPU2,GPU3",
+			privileged:         false,
+			acceptUnprivileged: false,
+			acceptMounts:       true,
+			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
+		},
+		{
+			description:        "No mount devices, unprivileged, no accept unprivileged",
+			mountDevices:       nil,
+			envvarDevices:      "GPU0,GPU1",
+			privileged:         false,
+			acceptUnprivileged: false,
+			acceptMounts:       true,
+			expectedDevices:    nil,
+		},
+		{
+			description:        "No mount devices, privileged, no accept unprivileged",
+			mountDevices:       nil,
+			envvarDevices:      "GPU0,GPU1",
+			privileged:         true,
+			acceptUnprivileged: false,
+			acceptMounts:       true,
+			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
+		},
+		{
+			description:        "No mount devices, unprivileged, accept unprivileged",
+			mountDevices:       nil,
+			envvarDevices:      "GPU0,GPU1",
+			privileged:         false,
+			acceptUnprivileged: true,
+			acceptMounts:       true,
+			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
+		},
+		{
+			description: "Mount devices, unprivileged, accept unprivileged, no accept mounts",
+			mountDevices: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			envvarDevices:      "GPU2,GPU3",
+			privileged:         false,
+			acceptUnprivileged: true,
+			acceptMounts:       false,
+			expectedDevices:    &[]string{"GPU2,GPU3"}[0],
+		},
+		{
+			description: "Mount devices, unprivileged, no accept unprivileged, no accept mounts",
+			mountDevices: []Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			envvarDevices:      "GPU2,GPU3",
+			privileged:         false,
+			acceptUnprivileged: false,
+			acceptMounts:       false,
+			expectedDevices:    nil,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			// Wrap the call to getDevices() in a closure.
+			var devices *string
+			getDevices := func() {
+				env := map[string]string{
+					envNVVisibleDevices: tc.envvarDevices,
+				}
+				hookConfig := getDefaultHookConfig()
+				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
+				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
+				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)
+			}
+
+			// For all other tests, just grab the devices and check the results
+			getDevices()
+
+			require.Equal(t, tc.expectedDevices, devices)
+		})
+	}
+}
+
+func TestGetDevicesFromEnvvar(t *testing.T) {
+	all := "all"
+	empty := ""
+	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
+	gpuID := "GPU-12345"
+	anotherGPUID := "GPU-67890"
+
+	var tests = []struct {
+		description     string
+		envSwarmGPU     *string
+		env             map[string]string
+		expectedDevices *string
+	}{
+		{
+			description: "empty env returns nil for non-legacy image",
+		},
+		{
+			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: "",
+			},
+		},
+		{
+			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: "void",
+			},
+		},
+		{
+			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: "none",
+			},
+			expectedDevices: &empty,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: gpuID,
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: gpuID,
+				envCUDAVersion:      "legacy",
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envCUDAVersion: "legacy",
+			},
+			expectedDevices: &all,
+		},
+		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
+		// not enabled
+		{
+			description: "missing NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   "",
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   "void",
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   "none",
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &empty,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   gpuID,
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   gpuID,
+				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
+			},
+			expectedDevices: &all,
+		},
+		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
+		// enabled
+		{
+			description: "empty env returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+		},
+		{
+			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: "",
+			},
+		},
+		{
+			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: "void",
+			},
+		},
+		{
+			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: "none",
+			},
+			expectedDevices: &empty,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: gpuID,
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: gpuID,
+				envCUDAVersion:        "legacy",
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS is selected if present",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envNVVisibleDevices:   gpuID,
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
+	}
+
+	for i, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			envSwarmGPU = tc.envSwarmGPU
+			devices := getDevicesFromEnvvar(image.CUDA(tc.env))
+			if tc.expectedDevices == nil {
+				require.Nil(t, devices, "%d: %v", i, tc)
+				return
+			}
+
+			require.NotNil(t, devices, "%d: %v", i, tc)
+			require.Equal(t, *tc.expectedDevices, *devices, "%d: %v", i, tc)
+		})
+	}
+}
+
+func TestGetDriverCapabilities(t *testing.T) {
+
+	supportedCapabilities := "compute,utility,display,video"
+
+	testCases := []struct {
+		description           string
+		env                   map[string]string
+		legacyImage           bool
+		supportedCapabilities string
+		expectedPanic         bool
+		expectedCapabilities  string
+	}{
+		{
+			description: "Env is set for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "display,video",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description: "Env is all for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "all",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description:           "Env unset for legacy image is 'all'",
+			env:                   map[string]string{},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is set for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "display,video",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description:           "Env unset for modern image is default",
+			env:                   map[string]string{},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description: "Env is all for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "all",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description: "Invalid capabilities panic",
+			env: map[string]string{
+				envNVDriverCapabilities: "compute,utility",
+			},
+			supportedCapabilities: "not-compute,not-utility",
+			expectedPanic:         true,
+		},
+		{
+			description:           "Default is restricted for modern image",
+			legacyImage:           false,
+			supportedCapabilities: "compute",
+			expectedCapabilities:  "compute",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			var capabilites DriverCapabilities
+
+			getDriverCapabilities := func() {
+				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
+				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getDriverCapabilities)
+				return
+			}
+
+			getDriverCapabilities()
+			require.EqualValues(t, tc.expectedCapabilities, capabilites)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"log"
+	"os"
+	"path"
+	"reflect"
+
+	"github.com/BurntSushi/toml"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+)
+
+const (
+	configPath = "/etc/nvidia-container-runtime/config.toml"
+	driverPath = "/run/nvidia/driver"
+)
+
+var defaultPaths = [...]string{
+	path.Join(driverPath, configPath),
+	configPath,
+}
+
+// CLIConfig : options for nvidia-container-cli.
+type CLIConfig struct {
+	Root        *string  `toml:"root"`
+	Path        *string  `toml:"path"`
+	Environment []string `toml:"environment"`
+	Debug       *string  `toml:"debug"`
+	Ldcache     *string  `toml:"ldcache"`
+	LoadKmods   bool     `toml:"load-kmods"`
+	NoPivot     bool     `toml:"no-pivot"`
+	NoCgroups   bool     `toml:"no-cgroups"`
+	User        *string  `toml:"user"`
+	Ldconfig    *string  `toml:"ldconfig"`
+}
+
+// HookConfig : options for the nvidia-container-runtime-hook.
+type HookConfig struct {
+	DisableRequire                 bool               `toml:"disable-require"`
+	SwarmResource                  *string            `toml:"swarm-resource"`
+	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
+
+	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
+}
+
+func getDefaultHookConfig() HookConfig {
+	return HookConfig{
+		DisableRequire:                 false,
+		SwarmResource:                  nil,
+		AcceptEnvvarUnprivileged:       true,
+		AcceptDeviceListAsVolumeMounts: false,
+		SupportedDriverCapabilities:    allDriverCapabilities,
+		NvidiaContainerCLI: CLIConfig{
+			Root:        nil,
+			Path:        nil,
+			Environment: []string{},
+			Debug:       nil,
+			Ldcache:     nil,
+			LoadKmods:   true,
+			NoPivot:     false,
+			NoCgroups:   false,
+			User:        nil,
+			Ldconfig:    nil,
+		},
+		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
+	}
+}
+
+func getHookConfig() (config HookConfig) {
+	var err error
+
+	if len(*configflag) > 0 {
+		config = getDefaultHookConfig()
+		_, err = toml.DecodeFile(*configflag, &config)
+		if err != nil {
+			log.Panicln("couldn't open configuration file:", err)
+		}
+	} else {
+		for _, p := range defaultPaths {
+			config = getDefaultHookConfig()
+			_, err = toml.DecodeFile(p, &config)
+			if err == nil {
+				break
+			} else if !os.IsNotExist(err) {
+				log.Panicln("couldn't open default configuration file:", err)
+			}
+		}
+	}
+
+	if config.SupportedDriverCapabilities == all {
+		config.SupportedDriverCapabilities = allDriverCapabilities
+	}
+	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
+	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
+	}
+
+	return config
+}
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c HookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHookConfig(t *testing.T) {
+	testCases := []struct {
+		lines                      []string
+		expectedPanic              bool
+		expectedDriverCapabilities DriverCapabilities
+	}{
+		{
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"all\"",
+			},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"compute,utility,not-compute\"",
+			},
+			expectedPanic: true,
+		},
+		{
+			lines:                      []string{},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"\"",
+			},
+			expectedDriverCapabilities: none,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"utility,compute\"",
+			},
+			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			var filename string
+			defer func() {
+				if len(filename) > 0 {
+					os.Remove(filename)
+				}
+				configflag = nil
+			}()
+
+			if tc.lines != nil {
+				configFile, err := os.CreateTemp("", "*.toml")
+				require.NoError(t, err)
+				defer configFile.Close()
+
+				filename = configFile.Name()
+				configflag = &filename
+
+				for _, line := range tc.lines {
+					_, err := configFile.WriteString(fmt.Sprintf("%s\n", line))
+					require.NoError(t, err)
+				}
+			}
+
+			var config HookConfig
+			getHookConfig := func() {
+				config = getHookConfig()
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getHookConfig)
+				return
+			}
+
+			getHookConfig()
+
+			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/hook_test.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsPrivileged(t *testing.T) {
+	var tests = []struct {
+		spec     string
+		expected bool
+	}{
+		{
+			`
+			{
+				"ociVersion": "1.0.0",
+				"process": {
+					"capabilities": {
+						"bounding": [ "CAP_SYS_ADMIN" ]
+					}
+				}
+			}
+			`,
+			true,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0",
+				"process": {
+					"capabilities": {
+						"bounding": [ "CAP_SYS_OTHER" ]
+					}
+				}
+			}
+			`,
+			false,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0",
+				"process": {}
+			}
+			`,
+			false,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0-rc2-dev",
+				"process": {
+					"capabilities": [ "CAP_SYS_ADMIN" ]
+				}
+			}
+			`,
+			true,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0-rc2-dev",
+				"process": {
+					"capabilities": [ "CAP_SYS_OTHER" ]
+				}
+			}
+			`,
+			false,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0-rc2-dev",
+				"process": {}
+			}
+			`,
+			false,
+		},
+	}
+	for i, tc := range tests {
+		var spec Spec
+		_ = json.Unmarshal([]byte(tc.spec), &spec)
+		privileged := isPrivileged(&spec)
+
+		require.Equal(t, tc.expected, privileged, "%d: %v", i, tc)
+	}
+}

cmd/nvidia-container-runtime-hook/main.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+var (
+	debugflag   = flag.Bool("debug", false, "enable debug output")
+	versionflag = flag.Bool("version", false, "enable version output")
+	configflag  = flag.String("config", "", "configuration file")
+)
+
+func exit() {
+	if err := recover(); err != nil {
+		if _, ok := err.(runtime.Error); ok {
+			log.Println(err)
+		}
+		if *debugflag {
+			log.Printf("%s", debug.Stack())
+		}
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func getCLIPath(config CLIConfig) string {
+	if config.Path != nil {
+		return *config.Path
+	}
+
+	var root string
+	if config.Root != nil {
+		root = *config.Root
+	}
+	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
+		log.Panicln("couldn't set PATH variable:", err)
+	}
+
+	path, err := exec.LookPath("nvidia-container-cli")
+	if err != nil {
+		log.Panicln("couldn't find binary nvidia-container-cli in", os.Getenv("PATH"), ":", err)
+	}
+	return path
+}
+
+// getRootfsPath returns an absolute path. We don't need to resolve symlinks for now.
+func getRootfsPath(config containerConfig) string {
+	rootfs, err := filepath.Abs(config.Rootfs)
+	if err != nil {
+		log.Panicln(err)
+	}
+	return rootfs
+}
+
+func doPrestart() {
+	var err error
+
+	defer exit()
+	log.SetFlags(0)
+
+	hook := getHookConfig()
+	cli := hook.NvidiaContainerCLI
+
+	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+	}
+
+	container := getContainerConfig(hook)
+	nvidia := container.Nvidia
+	if nvidia == nil {
+		// Not a GPU container, nothing to do.
+		return
+	}
+
+	rootfs := getRootfsPath(container)
+
+	args := []string{getCLIPath(cli)}
+	if cli.Root != nil {
+		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
+	}
+	if cli.LoadKmods {
+		args = append(args, "--load-kmods")
+	}
+	if cli.NoPivot {
+		args = append(args, "--no-pivot")
+	}
+	if *debugflag {
+		args = append(args, "--debug=/dev/stderr")
+	} else if cli.Debug != nil {
+		args = append(args, fmt.Sprintf("--debug=%s", *cli.Debug))
+	}
+	if cli.Ldcache != nil {
+		args = append(args, fmt.Sprintf("--ldcache=%s", *cli.Ldcache))
+	}
+	if cli.User != nil {
+		args = append(args, fmt.Sprintf("--user=%s", *cli.User))
+	}
+	args = append(args, "configure")
+
+	if cli.Ldconfig != nil {
+		args = append(args, fmt.Sprintf("--ldconfig=%s", *cli.Ldconfig))
+	}
+	if cli.NoCgroups {
+		args = append(args, "--no-cgroups")
+	}
+	if len(nvidia.Devices) > 0 {
+		args = append(args, fmt.Sprintf("--device=%s", nvidia.Devices))
+	}
+	if len(nvidia.MigConfigDevices) > 0 {
+		args = append(args, fmt.Sprintf("--mig-config=%s", nvidia.MigConfigDevices))
+	}
+	if len(nvidia.MigMonitorDevices) > 0 {
+		args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
+	}
+
+	for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
+		if len(cap) == 0 {
+			break
+		}
+		args = append(args, capabilityToCLI(cap))
+	}
+
+	if !hook.DisableRequire && !nvidia.DisableRequire {
+		for _, req := range nvidia.Requirements {
+			args = append(args, fmt.Sprintf("--require=%s", req))
+		}
+	}
+
+	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
+	args = append(args, rootfs)
+
+	env := append(os.Environ(), cli.Environment...)
+	err = syscall.Exec(args[0], args, env)
+	log.Panicln("exec failed:", err)
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, "\nCommands:\n")
+	fmt.Fprintf(os.Stderr, "  prestart\n        run the prestart hook\n")
+	fmt.Fprintf(os.Stderr, "  poststart\n        no-op\n")
+	fmt.Fprintf(os.Stderr, "  poststop\n        no-op\n")
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+
+	if *versionflag {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime Hook", info.GetVersionString())
+		return
+	}
+
+	args := flag.Args()
+	if len(args) == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+
+	switch args[0] {
+	case "prestart":
+		doPrestart()
+		os.Exit(0)
+	case "poststart":
+		fallthrough
+	case "poststop":
+		os.Exit(0)
+	default:
+		flag.Usage()
+		os.Exit(2)
+	}
+}
+
+// logInterceptor implements the info.Logger interface to allow for logging from this function.
+type logInterceptor struct{}
+
+func (l *logInterceptor) Infof(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
+
+func (l *logInterceptor) Debugf(format string, args ...interface{}) {}