From 62d88e7c95b5740ba576cc32f9ee79279af849a2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 24 Feb 2023 11:06:59 +0200 Subject: [PATCH] Add cdi and legacy mode runtimes This change adds .cdi and .legacy mode-specific runtimes the list of runtimes supported by the operator. These are also installed as part of the NVIDIA Container Toolkit. Signed-off-by: Evan Lezar --- tools/container/containerd/config_v1_test.go | 20 +++-- tools/container/containerd/config_v2_test.go | 82 +++++++++--------- tools/container/docker/docker_test.go | 87 ++++++++++++++++++++ tools/container/operator/operator.go | 2 +- tools/container/operator/operator_test.go | 56 +++++++++++++ tools/container/toolkit/runtime.go | 35 +++++--- tools/container/toolkit/runtime_test.go | 2 +- 7 files changed, 227 insertions(+), 57 deletions(-) diff --git a/tools/container/containerd/config_v1_test.go b/tools/container/containerd/config_v1_test.go index dd8bf54c..9b94bba2 100644 --- a/tools/container/containerd/config_v1_test.go +++ b/tools/container/containerd/config_v1_test.go @@ -141,6 +141,8 @@ func TestUpdateV1Config(t *testing.T) { expectedBinaries := []string{ "/test/runtime/dir/nvidia-container-runtime", "/test/runtime/dir/nvidia-container-runtime.experimental", + "/test/runtime/dir/nvidia-container-runtime.cdi", + "/test/runtime/dir/nvidia-container-runtime.legacy", } testCases := []struct { @@ -149,15 +151,15 @@ func TestUpdateV1Config(t *testing.T) { }{ { runtimeClass: "nvidia", - expectedRuntimes: []string{"nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "NAME", - expectedRuntimes: []string{"NAME", "nvidia-experimental"}, + expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "nvidia-experimental", - expectedRuntimes: []string{"nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, } @@ -216,6 +218,8 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) { runcBinary, "/test/runtime/dir/nvidia-container-runtime", "/test/runtime/dir/nvidia-container-runtime.experimental", + "/test/runtime/dir/nvidia-container-runtime.cdi", + "/test/runtime/dir/nvidia-container-runtime.legacy", } testCases := []struct { @@ -224,15 +228,15 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) { }{ { runtimeClass: "nvidia", - expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "NAME", - expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"}, + expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "nvidia-experimental", - expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, } @@ -303,6 +307,8 @@ func TestRevertV1Config(t *testing.T) { "runtimes": map[string]interface{}{ "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), "nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"), + "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), + "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), }, }, }, @@ -318,6 +324,8 @@ func TestRevertV1Config(t *testing.T) { "runtimes": map[string]interface{}{ "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), "nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"), + "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), + "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), }, "default_runtime": defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"), "default_runtime_name": "nvidia", diff --git a/tools/container/containerd/config_v2_test.go b/tools/container/containerd/config_v2_test.go index 6df8e69c..d5e5597e 100644 --- a/tools/container/containerd/config_v2_test.go +++ b/tools/container/containerd/config_v2_test.go @@ -71,25 +71,27 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) { } for i, tc := range testCases { - o := &options{ - setAsDefault: tc.setAsDefault, - runtimeClass: tc.runtimeClass, - runtimeDir: runtimeDir, - } + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + o := &options{ + setAsDefault: tc.setAsDefault, + runtimeClass: tc.runtimeClass, + runtimeDir: runtimeDir, + } - config, err := toml.TreeFromMap(map[string]interface{}{}) - require.NoError(t, err, "%d: %v", i, tc) + config, err := toml.TreeFromMap(map[string]interface{}{}) + require.NoError(t, err, "%d: %v", i, tc) - v2 := &containerd.Config{ - Tree: config, - RuntimeType: runtimeType, - } + v2 := &containerd.Config{ + Tree: config, + RuntimeType: runtimeType, + } - err = UpdateConfig(v2, o) - require.NoError(t, err, "%d: %v", i, tc) + err = UpdateConfig(v2, o) + require.NoError(t, err, "%d: %v", i, tc) - defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}) - require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc) + defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}) + require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc) + }) } } @@ -100,6 +102,8 @@ func TestUpdateV2Config(t *testing.T) { expectedBinaries := []string{ "/test/runtime/dir/nvidia-container-runtime", "/test/runtime/dir/nvidia-container-runtime.experimental", + "/test/runtime/dir/nvidia-container-runtime.cdi", + "/test/runtime/dir/nvidia-container-runtime.legacy", } testCases := []struct { @@ -108,15 +112,15 @@ func TestUpdateV2Config(t *testing.T) { }{ { runtimeClass: "nvidia", - expectedRuntimes: []string{"nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "NAME", - expectedRuntimes: []string{"NAME", "nvidia-experimental"}, + expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "nvidia-experimental", - expectedRuntimes: []string{"nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, } @@ -173,6 +177,8 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) { runcBinary, "/test/runtime/dir/nvidia-container-runtime", "/test/runtime/dir/nvidia-container-runtime.experimental", + "/test/runtime/dir/nvidia-container-runtime.cdi", + "/test/runtime/dir/nvidia-container-runtime.legacy", } testCases := []struct { @@ -181,15 +187,15 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) { }{ { runtimeClass: "nvidia", - expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "NAME", - expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"}, + expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, { runtimeClass: "nvidia-experimental", - expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"}, + expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"}, }, } @@ -284,28 +290,30 @@ func TestRevertV2Config(t *testing.T) { } for i, tc := range testCases { - o := &options{ - runtimeClass: "nvidia", - } + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + o := &options{ + runtimeClass: "nvidia", + } - config, err := toml.TreeFromMap(tc.config) - require.NoError(t, err, "%d: %v", i, tc) + config, err := toml.TreeFromMap(tc.config) + require.NoError(t, err, "%d: %v", i, tc) - expected, err := toml.TreeFromMap(tc.expected) - require.NoError(t, err, "%d: %v", i, tc) + expected, err := toml.TreeFromMap(tc.expected) + require.NoError(t, err, "%d: %v", i, tc) - v2 := &containerd.Config{ - Tree: config, - RuntimeType: runtimeType, - } + v2 := &containerd.Config{ + Tree: config, + RuntimeType: runtimeType, + } - err = RevertConfig(v2, o) - require.NoError(t, err, "%d: %v", i, tc) + err = RevertConfig(v2, o) + require.NoError(t, err, "%d: %v", i, tc) - configContents, _ := toml.Marshal(config) - expectedContents, _ := toml.Marshal(expected) + configContents, _ := toml.Marshal(config) + expectedContents, _ := toml.Marshal(expected) - require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc) + require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc) + }) } } diff --git a/tools/container/docker/docker_test.go b/tools/container/docker/docker_test.go index 462192ed..2bba2116 100644 --- a/tools/container/docker/docker_test.go +++ b/tools/container/docker/docker_test.go @@ -93,6 +93,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -110,6 +118,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -127,6 +143,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -150,6 +174,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -176,6 +208,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -196,6 +236,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -216,6 +264,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -244,6 +300,14 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime.experimental", "args": []string{}, }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, }, }, }, @@ -315,6 +379,29 @@ func TestRevertConfig(t *testing.T) { }, expectedConfig: map[string]interface{}{}, }, + { + config: map[string]interface{}{ + "runtimes": map[string]interface{}{ + "nvidia": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime", + "args": []string{}, + }, + "nvidia-experimental": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.experimental", + "args": []string{}, + }, + "nvidia-cdi": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.cdi", + "args": []string{}, + }, + "nvidia-legacy": map[string]interface{}{ + "path": "/test/runtime/dir/nvidia-container-runtime.legacy", + "args": []string{}, + }, + }, + }, + expectedConfig: map[string]interface{}{}, + }, { config: map[string]interface{}{ "default-runtime": "nvidia", diff --git a/tools/container/operator/operator.go b/tools/container/operator/operator.go index b52e97ad..fe90f715 100644 --- a/tools/container/operator/operator.go +++ b/tools/container/operator/operator.go @@ -59,7 +59,7 @@ func GetRuntimes(opts ...Option) Runtimes { runtimes := make(Runtimes) runtimes.add(c.nvidiaRuntime()) - modes := []string{"experimental"} + modes := []string{"experimental", "cdi", "legacy"} for _, mode := range modes { runtimes.add(c.modeRuntime(mode)) } diff --git a/tools/container/operator/operator_test.go b/tools/container/operator/operator_test.go index 07417fb9..e2f4abf7 100644 --- a/tools/container/operator/operator_test.go +++ b/tools/container/operator/operator_test.go @@ -41,6 +41,14 @@ func TestOptions(t *testing.T) { name: "nvidia-experimental", Path: "/usr/bin/nvidia-container-runtime.experimental", }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, { @@ -56,6 +64,14 @@ func TestOptions(t *testing.T) { name: "nvidia-experimental", Path: "/usr/bin/nvidia-container-runtime.experimental", }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, { @@ -72,6 +88,14 @@ func TestOptions(t *testing.T) { name: "nvidia-experimental", Path: "/usr/bin/nvidia-container-runtime.experimental", }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, { @@ -88,6 +112,14 @@ func TestOptions(t *testing.T) { name: "nvidia-experimental", Path: "/usr/bin/nvidia-container-runtime.experimental", }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, { @@ -102,6 +134,14 @@ func TestOptions(t *testing.T) { name: "nvidia-experimental", Path: "/usr/bin/nvidia-container-runtime.experimental", }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, { @@ -118,6 +158,14 @@ func TestOptions(t *testing.T) { Path: "/usr/bin/nvidia-container-runtime.experimental", SetAsDefault: true, }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, { @@ -132,6 +180,14 @@ func TestOptions(t *testing.T) { name: "nvidia-experimental", Path: "/usr/bin/nvidia-container-runtime.experimental", }, + "nvidia-cdi": Runtime{ + name: "nvidia-cdi", + Path: "/usr/bin/nvidia-container-runtime.cdi", + }, + "nvidia-legacy": Runtime{ + name: "nvidia-legacy", + Path: "/usr/bin/nvidia-container-runtime.legacy", + }, }, }, } diff --git a/tools/container/toolkit/runtime.go b/tools/container/toolkit/runtime.go index bd39649a..d8d7abad 100644 --- a/tools/container/toolkit/runtime.go +++ b/tools/container/toolkit/runtime.go @@ -21,13 +21,12 @@ import ( "path/filepath" "strings" + "github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator" log "github.com/sirupsen/logrus" ) const ( - nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime" - nvidiaContainerRuntimeTarget = "nvidia-container-runtime.real" - nvidiaContainerRuntimeWrapper = "nvidia-container-runtime" + nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime" nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental" ) @@ -35,15 +34,21 @@ const ( // installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables // and implementing the required wrapper func installContainerRuntimes(toolkitDir string, driverRoot string) error { - r := newNvidiaContainerRuntimeInstaller() + runtimes := operator.GetRuntimes() + for _, runtime := range runtimes { + if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource { + continue + } + r := newNvidiaContainerRuntimeInstaller(runtime.Path) - _, err := r.install(toolkitDir) - if err != nil { - return fmt.Errorf("error installing NVIDIA container runtime: %v", err) + _, err := r.install(toolkitDir) + if err != nil { + return fmt.Errorf("error installing NVIDIA container runtime: %v", err) + } } // Install the experimental runtime and treat failures as non-fatal. - err = installExperimentalRuntime(toolkitDir, driverRoot) + err := installExperimentalRuntime(toolkitDir, driverRoot) if err != nil { log.Warnf("Could not install experimental runtime: %v", err) } @@ -68,12 +73,18 @@ func installExperimentalRuntime(toolkitDir string, driverRoot string) error { return nil } -func newNvidiaContainerRuntimeInstaller() *executable { +// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime. +// This installer will copy the specified source exectuable to the toolkit directory. +// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is +// created to allow for the configuration of the runtime environment. +func newNvidiaContainerRuntimeInstaller(source string) *executable { + wrapperName := filepath.Base(source) + dotfileName := wrapperName + ".real" target := executableTarget{ - dotfileName: nvidiaContainerRuntimeTarget, - wrapperName: nvidiaContainerRuntimeWrapper, + dotfileName: dotfileName, + wrapperName: wrapperName, } - return newRuntimeInstaller(nvidiaContainerRuntimeSource, target, nil) + return newRuntimeInstaller(source, target, nil) } func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable { diff --git a/tools/container/toolkit/runtime_test.go b/tools/container/toolkit/runtime_test.go index 06b1b3c2..14afb14b 100644 --- a/tools/container/toolkit/runtime_test.go +++ b/tools/container/toolkit/runtime_test.go @@ -25,7 +25,7 @@ import ( ) func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) { - r := newNvidiaContainerRuntimeInstaller() + r := newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource) const shebang = "#! /bin/sh" const destFolder = "/dest/folder"