Add cdi and legacy mode runtimes

This change adds .cdi and .legacy mode-specific runtimes the list of
runtimes supported by the operator. These are also installed as
part of the NVIDIA Container Toolkit.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

tools/container/containerd/config_v1_test.go

@@ -141,6 +141,8 @@ func TestUpdateV1Config(t *testing.T) {
 	expectedBinaries := []string{
 		"/test/runtime/dir/nvidia-container-runtime",
 		"/test/runtime/dir/nvidia-container-runtime.experimental",
+		"/test/runtime/dir/nvidia-container-runtime.cdi",
+		"/test/runtime/dir/nvidia-container-runtime.legacy",
 	}
 
 	testCases := []struct {
@@ -149,15 +151,15 @@ func TestUpdateV1Config(t *testing.T) {
 	}{
 		{
 			runtimeClass:     "nvidia",
-			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "NAME",
-			expectedRuntimes: []string{"NAME", "nvidia-experimental"},
+			expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "nvidia-experimental",
-			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 	}
 
@@ -216,6 +218,8 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
 		runcBinary,
 		"/test/runtime/dir/nvidia-container-runtime",
 		"/test/runtime/dir/nvidia-container-runtime.experimental",
+		"/test/runtime/dir/nvidia-container-runtime.cdi",
+		"/test/runtime/dir/nvidia-container-runtime.legacy",
 	}
 
 	testCases := []struct {
@@ -224,15 +228,15 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
 	}{
 		{
 			runtimeClass:     "nvidia",
-			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "NAME",
-			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
+			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "nvidia-experimental",
-			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 	}
 
@@ -303,6 +307,8 @@ func TestRevertV1Config(t *testing.T) {
 							"runtimes": map[string]interface{}{
 								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
 								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
+								"nvidia-cdi":          runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy":       runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
 							},
 						},
 					},
@@ -318,6 +324,8 @@ func TestRevertV1Config(t *testing.T) {
 							"runtimes": map[string]interface{}{
 								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
 								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
+								"nvidia-cdi":          runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy":       runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
 							},
 							"default_runtime":      defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
 							"default_runtime_name": "nvidia",

tools/container/containerd/config_v2_test.go

@@ -71,25 +71,27 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		o := &options{
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			setAsDefault: tc.setAsDefault,
+			o := &options{
-			runtimeClass: tc.runtimeClass,
+				setAsDefault: tc.setAsDefault,
-			runtimeDir:   runtimeDir,
+				runtimeClass: tc.runtimeClass,
-		}
+				runtimeDir:   runtimeDir,
+			}
 
-		config, err := toml.TreeFromMap(map[string]interface{}{})
+			config, err := toml.TreeFromMap(map[string]interface{}{})
-		require.NoError(t, err, "%d: %v", i, tc)
+			require.NoError(t, err, "%d: %v", i, tc)
 
-		v2 := &containerd.Config{
+			v2 := &containerd.Config{
-			Tree:        config,
+				Tree:        config,
-			RuntimeType: runtimeType,
+				RuntimeType: runtimeType,
-		}
+			}
 
-		err = UpdateConfig(v2, o)
+			err = UpdateConfig(v2, o)
-		require.NoError(t, err, "%d: %v", i, tc)
+			require.NoError(t, err, "%d: %v", i, tc)
 
-		defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+			defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
-		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+		})
 	}
 }
 
@@ -100,6 +102,8 @@ func TestUpdateV2Config(t *testing.T) {
 	expectedBinaries := []string{
 		"/test/runtime/dir/nvidia-container-runtime",
 		"/test/runtime/dir/nvidia-container-runtime.experimental",
+		"/test/runtime/dir/nvidia-container-runtime.cdi",
+		"/test/runtime/dir/nvidia-container-runtime.legacy",
 	}
 
 	testCases := []struct {
@@ -108,15 +112,15 @@ func TestUpdateV2Config(t *testing.T) {
 	}{
 		{
 			runtimeClass:     "nvidia",
-			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "NAME",
-			expectedRuntimes: []string{"NAME", "nvidia-experimental"},
+			expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "nvidia-experimental",
-			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 	}
 
@@ -173,6 +177,8 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 		runcBinary,
 		"/test/runtime/dir/nvidia-container-runtime",
 		"/test/runtime/dir/nvidia-container-runtime.experimental",
+		"/test/runtime/dir/nvidia-container-runtime.cdi",
+		"/test/runtime/dir/nvidia-container-runtime.legacy",
 	}
 
 	testCases := []struct {
@@ -181,15 +187,15 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 	}{
 		{
 			runtimeClass:     "nvidia",
-			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "NAME",
-			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
+			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 		{
 			runtimeClass:     "nvidia-experimental",
-			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
 		},
 	}
 
@@ -284,28 +290,30 @@ func TestRevertV2Config(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		o := &options{
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			runtimeClass: "nvidia",
+			o := &options{
-		}
+				runtimeClass: "nvidia",
+			}
 
-		config, err := toml.TreeFromMap(tc.config)
+			config, err := toml.TreeFromMap(tc.config)
-		require.NoError(t, err, "%d: %v", i, tc)
+			require.NoError(t, err, "%d: %v", i, tc)
 
-		expected, err := toml.TreeFromMap(tc.expected)
+			expected, err := toml.TreeFromMap(tc.expected)
-		require.NoError(t, err, "%d: %v", i, tc)
+			require.NoError(t, err, "%d: %v", i, tc)
 
-		v2 := &containerd.Config{
+			v2 := &containerd.Config{
-			Tree:        config,
+				Tree:        config,
-			RuntimeType: runtimeType,
+				RuntimeType: runtimeType,
-		}
+			}
 
-		err = RevertConfig(v2, o)
+			err = RevertConfig(v2, o)
-		require.NoError(t, err, "%d: %v", i, tc)
+			require.NoError(t, err, "%d: %v", i, tc)
 
-		configContents, _ := toml.Marshal(config)
+			configContents, _ := toml.Marshal(config)
-		expectedContents, _ := toml.Marshal(expected)
+			expectedContents, _ := toml.Marshal(expected)
 
-		require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
+		})
 	}
 }
 

tools/container/docker/docker_test.go

@@ -93,6 +93,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -110,6 +118,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -127,6 +143,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -150,6 +174,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -176,6 +208,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -196,6 +236,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -216,6 +264,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -244,6 +300,14 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
 						"args": []string{},
 					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
 				},
 			},
 		},
@@ -315,6 +379,29 @@ func TestRevertConfig(t *testing.T) {
 			},
 			expectedConfig: map[string]interface{}{},
 		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
+						"args": []string{},
+					},
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
 		{
 			config: map[string]interface{}{
 				"default-runtime": "nvidia",

tools/container/operator/operator.go

@@ -59,7 +59,7 @@ func GetRuntimes(opts ...Option) Runtimes {
 	runtimes := make(Runtimes)
 	runtimes.add(c.nvidiaRuntime())
 
-	modes := []string{"experimental"}
+	modes := []string{"experimental", "cdi", "legacy"}
 	for _, mode := range modes {
 		runtimes.add(c.modeRuntime(mode))
 	}

tools/container/operator/operator_test.go

@@ -41,6 +41,14 @@ func TestOptions(t *testing.T) {
 					name: "nvidia-experimental",
 					Path: "/usr/bin/nvidia-container-runtime.experimental",
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 		{
@@ -56,6 +64,14 @@ func TestOptions(t *testing.T) {
 					name: "nvidia-experimental",
 					Path: "/usr/bin/nvidia-container-runtime.experimental",
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 		{
@@ -72,6 +88,14 @@ func TestOptions(t *testing.T) {
 					name: "nvidia-experimental",
 					Path: "/usr/bin/nvidia-container-runtime.experimental",
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 		{
@@ -88,6 +112,14 @@ func TestOptions(t *testing.T) {
 					name: "nvidia-experimental",
 					Path: "/usr/bin/nvidia-container-runtime.experimental",
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 		{
@@ -102,6 +134,14 @@ func TestOptions(t *testing.T) {
 					name: "nvidia-experimental",
 					Path: "/usr/bin/nvidia-container-runtime.experimental",
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 		{
@@ -118,6 +158,14 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime.experimental",
 					SetAsDefault: true,
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 		{
@@ -132,6 +180,14 @@ func TestOptions(t *testing.T) {
 					name: "nvidia-experimental",
 					Path: "/usr/bin/nvidia-container-runtime.experimental",
 				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
 			},
 		},
 	}

tools/container/toolkit/runtime.go

@@ -21,13 +21,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
 	log "github.com/sirupsen/logrus"
 )
 
 const (
-	nvidiaContainerRuntimeSource  = "/usr/bin/nvidia-container-runtime"
+	nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
-	nvidiaContainerRuntimeTarget  = "nvidia-container-runtime.real"
-	nvidiaContainerRuntimeWrapper = "nvidia-container-runtime"
 
 	nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental"
 )
@@ -35,15 +34,21 @@ const (
 // installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
 // and implementing the required wrapper
 func installContainerRuntimes(toolkitDir string, driverRoot string) error {
-	r := newNvidiaContainerRuntimeInstaller()
+	runtimes := operator.GetRuntimes()
+	for _, runtime := range runtimes {
+		if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource {
+			continue
+		}
+		r := newNvidiaContainerRuntimeInstaller(runtime.Path)
 
-	_, err := r.install(toolkitDir)
+		_, err := r.install(toolkitDir)
-	if err != nil {
+		if err != nil {
-		return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
+			return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
+		}
 	}
 
 	// Install the experimental runtime and treat failures as non-fatal.
-	err = installExperimentalRuntime(toolkitDir, driverRoot)
+	err := installExperimentalRuntime(toolkitDir, driverRoot)
 	if err != nil {
 		log.Warnf("Could not install experimental runtime: %v", err)
 	}
@@ -68,12 +73,18 @@ func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
 	return nil
 }
 
-func newNvidiaContainerRuntimeInstaller() *executable {
+// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime.
+// This installer will copy the specified source exectuable to the toolkit directory.
+// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is
+// created to allow for the configuration of the runtime environment.
+func newNvidiaContainerRuntimeInstaller(source string) *executable {
+	wrapperName := filepath.Base(source)
+	dotfileName := wrapperName + ".real"
 	target := executableTarget{
-		dotfileName: nvidiaContainerRuntimeTarget,
+		dotfileName: dotfileName,
-		wrapperName: nvidiaContainerRuntimeWrapper,
+		wrapperName: wrapperName,
 	}
-	return newRuntimeInstaller(nvidiaContainerRuntimeSource, target, nil)
+	return newRuntimeInstaller(source, target, nil)
 }
 
 func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {

tools/container/toolkit/runtime_test.go

@@ -25,7 +25,7 @@ import (
 )
 
 func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
-	r := newNvidiaContainerRuntimeInstaller()
+	r := newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource)
 
 	const shebang = "#! /bin/sh"
 	const destFolder = "/dest/folder"