UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2024-11-21 15:57:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add cdi and legacy mode runtimes

Browse Source 
This change adds .cdi and .legacy mode-specific runtimes the list of
runtimes supported by the operator. These are also installed as
part of the NVIDIA Container Toolkit.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar 2023-02-24 11:06:59 +02:00
parent dca8e3123f
commit 62d88e7c95
7 changed files with 227 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
tools/container/containerd/config_v1_test.go
View File

@ -141,6 +141,8 @@ func TestUpdateV1Config(t *testing.T) {
expectedBinaries := []string{
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -149,15 +151,15 @@ func TestUpdateV1Config(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"NAME", "nvidia-experimental"},
expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
@ -216,6 +218,8 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
runcBinary,
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -224,15 +228,15 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
@ -303,6 +307,8 @@ func TestRevertV1Config(t *testing.T) {
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
},
},
},
@ -318,6 +324,8 @@ func TestRevertV1Config(t *testing.T) {
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
},
"default_runtime": defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
"default_runtime_name": "nvidia",

82
tools/container/containerd/config_v2_test.go
View File

@ -71,25 +71,27 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
}
for i, tc := range testCases {
o := &options{
setAsDefault: tc.setAsDefault,
runtimeClass: tc.runtimeClass,
runtimeDir: runtimeDir,
}
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
setAsDefault: tc.setAsDefault,
runtimeClass: tc.runtimeClass,
runtimeDir: runtimeDir,
}
config, err := toml.TreeFromMap(map[string]interface{}{})
require.NoError(t, err, "%d: %v", i, tc)
config, err := toml.TreeFromMap(map[string]interface{}{})
require.NoError(t, err, "%d: %v", i, tc)
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
err = UpdateConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
})
}
}
@ -100,6 +102,8 @@ func TestUpdateV2Config(t *testing.T) {
expectedBinaries := []string{
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -108,15 +112,15 @@ func TestUpdateV2Config(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"NAME", "nvidia-experimental"},
expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
@ -173,6 +177,8 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
runcBinary,
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -181,15 +187,15 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
@ -284,28 +290,30 @@ func TestRevertV2Config(t *testing.T) {
}
for i, tc := range testCases {
o := &options{
runtimeClass: "nvidia",
}
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
runtimeClass: "nvidia",
}
config, err := toml.TreeFromMap(tc.config)
require.NoError(t, err, "%d: %v", i, tc)
config, err := toml.TreeFromMap(tc.config)
require.NoError(t, err, "%d: %v", i, tc)
expected, err := toml.TreeFromMap(tc.expected)
require.NoError(t, err, "%d: %v", i, tc)
expected, err := toml.TreeFromMap(tc.expected)
require.NoError(t, err, "%d: %v", i, tc)
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
err = RevertConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
err = RevertConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
configContents, _ := toml.Marshal(config)
expectedContents, _ := toml.Marshal(expected)
configContents, _ := toml.Marshal(config)
expectedContents, _ := toml.Marshal(expected)
require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
})
}
}

87
tools/container/docker/docker_test.go
View File

@ -93,6 +93,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -110,6 +118,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -127,6 +143,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -150,6 +174,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -176,6 +208,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -196,6 +236,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -216,6 +264,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -244,6 +300,14 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
@ -315,6 +379,29 @@ func TestRevertConfig(t *testing.T) {
},
expectedConfig: map[string]interface{}{},
},
{
config: map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
expectedConfig: map[string]interface{}{},
},
{
config: map[string]interface{}{
"default-runtime": "nvidia",

2
tools/container/operator/operator.go
View File

@ -59,7 +59,7 @@ func GetRuntimes(opts ...Option) Runtimes {
runtimes := make(Runtimes)
runtimes.add(c.nvidiaRuntime())
modes := []string{"experimental"}
modes := []string{"experimental", "cdi", "legacy"}
for _, mode := range modes {
runtimes.add(c.modeRuntime(mode))
}

56
tools/container/operator/operator_test.go
View File

@ -41,6 +41,14 @@ func TestOptions(t *testing.T) {
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
@ -56,6 +64,14 @@ func TestOptions(t *testing.T) {
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
@ -72,6 +88,14 @@ func TestOptions(t *testing.T) {
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
@ -88,6 +112,14 @@ func TestOptions(t *testing.T) {
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
@ -102,6 +134,14 @@ func TestOptions(t *testing.T) {
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
@ -118,6 +158,14 @@ func TestOptions(t *testing.T) {
Path: "/usr/bin/nvidia-container-runtime.experimental",
SetAsDefault: true,
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
@ -132,6 +180,14 @@ func TestOptions(t *testing.T) {
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
}

35
tools/container/toolkit/runtime.go
View File

@ -21,13 +21,12 @@ import (
"path/filepath"
"strings"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
log "github.com/sirupsen/logrus"
)
const (
nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
nvidiaContainerRuntimeTarget = "nvidia-container-runtime.real"
nvidiaContainerRuntimeWrapper = "nvidia-container-runtime"
nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental"
)
@ -35,15 +34,21 @@ const (
// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
// and implementing the required wrapper
func installContainerRuntimes(toolkitDir string, driverRoot string) error {
r := newNvidiaContainerRuntimeInstaller()
runtimes := operator.GetRuntimes()
for _, runtime := range runtimes {
if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource {
continue
}
r := newNvidiaContainerRuntimeInstaller(runtime.Path)
_, err := r.install(toolkitDir)
if err != nil {
return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
_, err := r.install(toolkitDir)
if err != nil {
return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
}
}
// Install the experimental runtime and treat failures as non-fatal.
err = installExperimentalRuntime(toolkitDir, driverRoot)
err := installExperimentalRuntime(toolkitDir, driverRoot)
if err != nil {
log.Warnf("Could not install experimental runtime: %v", err)
}
@ -68,12 +73,18 @@ func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
return nil
}
func newNvidiaContainerRuntimeInstaller() *executable {
// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime.
// This installer will copy the specified source exectuable to the toolkit directory.
// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is
// created to allow for the configuration of the runtime environment.
func newNvidiaContainerRuntimeInstaller(source string) *executable {
wrapperName := filepath.Base(source)
dotfileName := wrapperName + ".real"
target := executableTarget{
dotfileName: nvidiaContainerRuntimeTarget,
wrapperName: nvidiaContainerRuntimeWrapper,
dotfileName: dotfileName,
wrapperName: wrapperName,
}
return newRuntimeInstaller(nvidiaContainerRuntimeSource, target, nil)
return newRuntimeInstaller(source, target, nil)
}
func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {

2
tools/container/toolkit/runtime_test.go
View File

@ -25,7 +25,7 @@ import (
)
func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
r := newNvidiaContainerRuntimeInstaller()
r := newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource)
const shebang = "#! /bin/sh"
const destFolder = "/dest/folder"