UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2024-11-21 15:57:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fetch current container runtime config through the command line

Browse Source 
Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

add default runtime binary path to runtimes field of toolkit config toml

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

[no-relnote] Get low-level runtimes consistently

We ensure that we use the same low-level runtimes regardless
of the runtime engine being configured. This ensures consistent
behaviour.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Co-authored-by: Evan Lezar <elezar@nvidia.com>

address review comment

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>
This commit is contained in:
Tariq Ibrahim 2024-08-08 15:40:00 -07:00
parent 4604e3b6c8
commit f477dc0df1
No known key found for this signature in database
GPG Key ID: 8367AA3C6B8DF06D
21 changed files with 596 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
cmd/nvidia-ctk/runtime/configure/configure.go
View File

@ -44,6 +44,10 @@ const (
defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
defaultCrioConfigFilePath = "/etc/crio/crio.conf"
defaultDockerConfigFilePath = "/etc/docker/daemon.json"
defaultConfigSource = configSourceFile
configSourceCommand = "command"
configSourceFile = "file"
)
type command struct {
@ -64,6 +68,7 @@ type config struct {
dryRun bool
runtime string
configFilePath string
configSource string
mode string
hookFilePath string
@ -120,6 +125,12 @@ func (m command) build() *cli.Command {
Usage: "the config mode for runtimes that support multiple configuration mechanisms",
Destination: &config.mode,
},
&cli.StringFlag{
Name: "config-source",
Usage: "the source to retrieve the container runtime configuration; one of [command, file]\"",
Destination: &config.configSource,
Value: defaultConfigSource,
},
&cli.StringFlag{
Name: "oci-hook-path",
Usage: "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
@ -202,6 +213,18 @@ func (m command) validateFlags(c *cli.Context, config *config) error {
config.runtimeConfigOverrideJSON = ""
}
switch config.configSource {
case configSourceCommand:
if config.runtime == "docker" {
m.logger.Warningf("A %v Config Source is not supported for %v; using %v", config.configSource, config.runtime, configSourceFile)
config.configSource = configSourceFile
}
case configSourceFile:
break
default:
return fmt.Errorf("unrecognized Config Source: %v", config.configSource)
}
return nil
}
@ -220,20 +243,25 @@ func (m command) configureWrapper(c *cli.Context, config *config) error {
func (m command) configureConfigFile(c *cli.Context, config *config) error {
configFilePath := config.resolveConfigFilePath()
var cfg engine.Interface
var err error
configSource, err := config.resolveConfigSource()
if err != nil {
return err
}
var cfg engine.Interface
switch config.runtime {
case "containerd":
cfg, err = containerd.New(
containerd.WithLogger(m.logger),
containerd.WithPath(configFilePath),
containerd.WithConfigSource(toml.FromFile(configFilePath)),
containerd.WithConfigSource(configSource),
)
case "crio":
cfg, err = crio.New(
crio.WithLogger(m.logger),
crio.WithPath(configFilePath),
crio.WithConfigSource(toml.FromFile(configFilePath)),
crio.WithConfigSource(configSource),
)
case "docker":
cfg, err = docker.New(
@ -295,6 +323,29 @@ func (c *config) resolveConfigFilePath() string {
return ""
}
// resolveConfigSource returns the default config source or the user provided config source
func (c *config) resolveConfigSource() (toml.Loader, error) {
switch c.configSource {
case configSourceCommand:
return c.getCommandConfigSource(), nil
case configSourceFile:
return toml.FromFile(c.configFilePath), nil
default:
return nil, fmt.Errorf("unrecognized config source: %s", c.configSource)
}
}
// getConfigSourceCommand returns the default cli command to fetch the current runtime config
func (c *config) getCommandConfigSource() toml.Loader {
switch c.runtime {
case "containerd":
return containerd.CommandLineSource("")
case "crio":
return crio.CommandLineSource("")
}
return toml.Empty
}
// getOuputConfigPath returns the configured config path or "" if dry-run is enabled
func (c *config) getOuputConfigPath() string {
if c.dryRun {

11
internal/config/toml.go
View File

@ -170,11 +170,22 @@ func (t *Toml) Get(key string) interface{} {
return (*toml.Tree)(t).Get(key)
}
// GetDefault returns the value for the specified key and falls back to the default value if the Get call fails
func (t *Toml) GetDefault(key string, def interface{}) interface{} {
return (*toml.Tree)(t).GetDefault(key, def)
}
// Set sets the specified key to the specified value in the TOML config.
func (t *Toml) Set(key string, value interface{}) {
(*toml.Tree)(t).Set(key, value)
}
// WriteTo encode the Tree as Toml and writes it to the writer w.
// Returns the number of bytes written in case of success, or an error if anything happened.
func (t *Toml) WriteTo(w io.Writer) (int64, error) {
return (*toml.Tree)(t).WriteTo(w)
}
// commentDefaults applies the required comments for default values to the Toml.
func (t *Toml) commentDefaults() *Toml {
asToml := (*toml.Tree)(t)

6
pkg/config/engine/api.go
View File

@ -23,4 +23,10 @@ type Interface interface {
Set(string, interface{})
RemoveRuntime(string) error
Save(string) (int64, error)
GetRuntimeConfig(string) (RuntimeConfig, error)
}
// RuntimeConfig defines the interface to query container runtime handler configuration
type RuntimeConfig interface {
GetBinaryPath() string
}

20
pkg/config/engine/containerd/config_v1.go
View File

@ -19,9 +19,8 @@ package containerd
import (
"fmt"
"github.com/pelletier/go-toml"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
)
// ConfigV1 represents a version 1 containerd config
@ -39,11 +38,7 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error
config.Set("version", int64(1))
// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
runtimeNamesForConfig := []string{"runc"}
if name, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
runtimeNamesForConfig = append(runtimeNamesForConfig, name)
}
runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
for _, r := range runtimeNamesForConfig {
options := config.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", r})
if options == nil {
@ -157,3 +152,14 @@ func (c *ConfigV1) Set(key string, value interface{}) {
func (c ConfigV1) Save(path string) (int64, error) {
return (Config)(c).Save(path)
}
func (c *ConfigV1) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
if c == nil || c.Tree == nil {
return nil, fmt.Errorf("config is nil")
}
runtimeData := c.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", name})
return &containerdCfgRuntime{
tree: runtimeData,
}, nil
}

12
pkg/config/engine/containerd/config_v1_test.go
View File

@ -138,7 +138,7 @@ func TestAddRuntimeV1(t *testing.T) {
`,
},
{
description: "options from runc take precedence over default runtime",
description: "options from the default runtime take precedence over runc",
config: `
[plugins]
[plugins.cri]
@ -186,14 +186,14 @@ func TestAddRuntimeV1(t *testing.T) {
BinaryName = "/usr/bin/default"
SystemdCgroup = false
[plugins.cri.containerd.runtimes.test]
privileged_without_host_devices = true
runtime_engine = "engine"
runtime_root = "root"
runtime_type = "type"
privileged_without_host_devices = false
runtime_engine = "defaultengine"
runtime_root = "defaultroot"
runtime_type = "defaulttype"
[plugins.cri.containerd.runtimes.test.options]
BinaryName = "/usr/bin/test"
Runtime = "/usr/bin/test"
SystemdCgroup = true
SystemdCgroup = false
`,
},
}

7
pkg/config/engine/containerd/config_v2.go
View File

@ -19,6 +19,7 @@ package containerd
import (
"fmt"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
)
@ -31,11 +32,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
config.Set("version", int64(2))
// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
runtimeNamesForConfig := []string{"runc"}
if name, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
runtimeNamesForConfig = append(runtimeNamesForConfig, name)
}
runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
for _, r := range runtimeNamesForConfig {
options := config.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", r})
if options == nil {

108
pkg/config/engine/containerd/config_v2_test.go
View File

@ -137,7 +137,7 @@ func TestAddRuntime(t *testing.T) {
`,
},
{
description: "options from runc take precedence over default runtime",
description: "options from the default runtime take precedence over runc",
config: `
version = 2
[plugins]
@ -186,13 +186,13 @@ func TestAddRuntime(t *testing.T) {
BinaryName = "/usr/bin/default"
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
privileged_without_host_devices = true
runtime_engine = "engine"
runtime_root = "root"
runtime_type = "type"
privileged_without_host_devices = false
runtime_engine = "defaultengine"
runtime_root = "defaultroot"
runtime_type = "defaulttype"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
BinaryName = "/usr/bin/test"
SystemdCgroup = true
SystemdCgroup = false
`,
},
}
@ -216,3 +216,99 @@ func TestAddRuntime(t *testing.T) {
})
}
}
func TestGetRuntimeConfig(t *testing.T) {
logger, _ := testlog.NewNullLogger()
config := `
version = 2
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "nvidia"
disable_snapshot_annotations = true
discard_unpacked_layers = false
ignore_blockio_not_enabled_errors = false
ignore_rdt_not_enabled_errors = false
no_pivot = false
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
privileged_without_host_devices_all_devices_allowed = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""
sandbox_mode = ""
snapshotter = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
privileged_without_host_devices_all_devices_allowed = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"
sandbox_mode = "podsandbox"
snapshotter = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
CriuImagePath = ""
CriuPath = ""
CriuWorkPath = ""
IoGid = 0
IoUid = 0
NoNewKeyring = false
NoPivotRoot = false
Root = ""
ShimCgroup = ""
SystemdCgroup = false
`
testCases := []struct {
description string
runtime string
expected string
expectedError error
}{
{
description: "valid runtime config, existing runtime",
runtime: "runc",
expected: "/usr/bin/runc",
expectedError: nil,
},
{
description: "valid runtime config, non-existing runtime",
runtime: "some-other-runtime",
expected: "",
expectedError: nil,
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
cfg, err := toml.Load(config)
require.NoError(t, err)
c := &Config{
Logger: logger,
Tree: cfg,
}
rc, err := c.GetRuntimeConfig(tc.runtime)
require.Equal(t, tc.expectedError, err)
require.Equal(t, tc.expected, rc.GetBinaryPath())
})
}
}

40
pkg/config/engine/containerd/containerd.go
View File

@ -35,6 +35,22 @@ type Config struct {
var _ engine.Interface = (*Config)(nil)
type containerdCfgRuntime struct {
tree *toml.Tree
}
var _ engine.RuntimeConfig = (*containerdCfgRuntime)(nil)
// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
func (c *containerdCfgRuntime) GetBinaryPath() string {
if c == nil || c.tree == nil {
return ""
}
binPath, _ := c.tree.GetPath([]string{"options", "BinaryName"}).(string)
return binPath
}
// New creates a containerd config with the specified options
func New(opts ...Option) (engine.Interface, error) {
b := &builder{
@ -98,3 +114,27 @@ func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
return -1, fmt.Errorf("unsupported type for version field: %v", v)
}
}
func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
if c == nil || c.Tree == nil {
return nil, fmt.Errorf("config is nil")
}
runtimeData := c.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
return &containerdCfgRuntime{
tree: runtimeData,
}, nil
}
// CommandLineSource returns the CLI-based containerd config loader
func CommandLineSource(hostRoot string) toml.Loader {
commandLine := chrootIfRequired(hostRoot, "containerd", "config", "dump")
return toml.FromCommandLine(commandLine[0], commandLine[1:]...)
}
func chrootIfRequired(hostRoot string, commandLine ...string) []string {
if hostRoot == "" || hostRoot == "/" {
return commandLine
}
return append([]string{"chroot", hostRoot}, commandLine...)
}

46
pkg/config/engine/crio/crio.go
View File

@ -30,6 +30,22 @@ type Config struct {
Logger logger.Interface
}
type crioRuntime struct {
tree *toml.Tree
}
var _ engine.RuntimeConfig = (*crioRuntime)(nil)
// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
func (c *crioRuntime) GetBinaryPath() string {
if c.tree != nil {
if binaryPath, ok := c.tree.GetPath([]string{"runtime_path"}).(string); ok {
return binaryPath
}
}
return ""
}
var _ engine.Interface = (*Config)(nil)
// New creates a cri-o config with the specified options
@ -65,11 +81,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
config := *c.Tree
// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
runtimeNamesForConfig := []string{"runc"}
if name, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok && name != "" {
runtimeNamesForConfig = append(runtimeNamesForConfig, name)
}
runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
for _, r := range runtimeNamesForConfig {
if options, ok := config.GetPath([]string{"crio", "runtime", "runtimes", r}).(*toml.Tree); ok {
c.Logger.Debugf("using options from runtime %v: %v", r, options.String())
@ -129,3 +141,27 @@ func (c *Config) RemoveRuntime(name string) error {
*c.Tree = config
return nil
}
func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
if c == nil || c.Tree == nil {
return nil, fmt.Errorf("config is nil")
}
runtimeData := c.GetSubtreeByPath([]string{"crio", "runtime", "runtimes", name})
return &crioRuntime{
tree: runtimeData,
}, nil
}
// CommandLineSource returns the CLI-based crio config loader
func CommandLineSource(hostRoot string) toml.Loader {
commandLine := chrootIfRequired(hostRoot, "crio", "status", "config")
return toml.FromCommandLine(commandLine[0], commandLine[1:]...)
}
func chrootIfRequired(hostRoot string, commandLine ...string) []string {
if hostRoot == "" || hostRoot == "/" {
return commandLine
}
return append([]string{"chroot", hostRoot}, commandLine...)
}

62
pkg/config/engine/crio/crio_test.go
View File

@ -91,7 +91,7 @@ func TestAddRuntime(t *testing.T) {
`,
},
{
description: "options from runc take precedence over default runtime",
description: "options from the default runtime take precedence over runc",
config: `
[crio]
[crio.runtime]
@ -120,7 +120,7 @@ func TestAddRuntime(t *testing.T) {
[crio.runtime.runtimes.test]
runtime_path = "/usr/bin/test"
runtime_type = "oci"
runc_option = "option"
default_option = "option"
`,
},
}
@ -144,3 +144,61 @@ func TestAddRuntime(t *testing.T) {
})
}
}
func TestGetRuntimeConfig(t *testing.T) {
logger, _ := testlog.NewNullLogger()
config := `
[crio.image]
signature_policy = "/etc/crio/policy.json"
[crio.runtime]
default_runtime = "crun"
[crio.runtime.runtimes.crun]
runtime_path = "/usr/libexec/crio/crun"
runtime_root = "/run/crun"
monitor_path = "/usr/libexec/crio/conmon"
allowed_annotations = [
"io.containers.trace-syscall",
]
[crio.runtime.runtimes.runc]
runtime_path = "/usr/libexec/crio/runc"
runtime_root = "/run/runc"
monitor_path = "/usr/libexec/crio/conmon"
`
testCases := []struct {
description string
runtime string
expected string
expectedError error
}{
{
description: "valid runtime config, existing runtime",
runtime: "crun",
expected: "/usr/libexec/crio/crun",
expectedError: nil,
},
{
description: "valid runtime config, non-existing runtime",
runtime: "some-other-runtime",
expected: "",
expectedError: nil,
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
cfg, err := toml.Load(config)
require.NoError(t, err)
c := &Config{
Logger: logger,
Tree: cfg,
}
rc, err := c.GetRuntimeConfig(tc.runtime)
require.Equal(t, tc.expectedError, err)
require.Equal(t, tc.expected, rc.GetBinaryPath())
})
}
}

33
pkg/config/engine/docker/docker.go
View File

@ -35,6 +35,20 @@ type Config map[string]interface{}
var _ engine.Interface = (*Config)(nil)
type dockerRuntime map[string]interface{}
var _ engine.RuntimeConfig = (*dockerRuntime)(nil)
// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
func (d dockerRuntime) GetBinaryPath() string {
if d == nil {
return ""
}
path, _ := d["path"].(string)
return path
}
// New creates a docker config with the specified options
func New(opts ...Option) (engine.Interface, error) {
b := &builder{}
@ -132,3 +146,22 @@ func (c Config) Save(path string) (int64, error) {
n, err := config.Raw(path).Write(output)
return int64(n), err
}
// GetRuntimeConfig returns the runtime info of the runtime passed as input
func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
if c == nil {
return nil, fmt.Errorf("config is nil")
}
cfg := *c
var runtimes map[string]interface{}
if _, ok := cfg["runtimes"]; ok {
runtimes = cfg["runtimes"].(map[string]interface{})
if r, ok := runtimes[name]; ok {
dr := dockerRuntime(r.(map[string]interface{}))
return &dr, nil
}
}
return &dockerRuntime{}, nil
}

34
pkg/config/engine/docker/docker_test.go
View File

@ -213,3 +213,37 @@ func TestUpdateConfigRuntimes(t *testing.T) {
}
}
func TestGetRuntimeConfig(t *testing.T) {
c := map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "nvidia-container-runtime",
"args": []string{},
},
},
}
cfg := Config(c)
testCases := []struct {
description string
runtime string
expected string
}{
{
description: "existing runtime",
runtime: "nvidia",
expected: "nvidia-container-runtime",
},
{
description: "non-existent runtime",
runtime: "some-other-runtime",
expected: "",
},
}
for _, tc := range testCases {
rc, err := cfg.GetRuntimeConfig(tc.runtime)
require.NoError(t, err)
require.Equal(t, tc.expected, rc.GetBinaryPath())
}
}

63
pkg/config/engine/engine.go Normal file
View File

@ -0,0 +1,63 @@
/**
# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package engine
import "strings"
// GetBinaryPathsForRuntimes returns the list of binary paths for common runtimes.
// The following list of runtimes is considered:
//
// the default runtime, "runc", and "crun"
//
// If an nvidia* runtime is set as the default runtime, this is ignored.
func GetBinaryPathsForRuntimes(cfg Interface) []string {
var binaryPaths []string
seen := make(map[string]bool)
for _, runtime := range GetLowLevelRuntimes(cfg) {
runtimeConfig, err := cfg.GetRuntimeConfig(runtime)
if err != nil {
// TODO: It will be useful to log the error when GetRuntimeConfig fails for a runtime
continue
}
binaryPath := runtimeConfig.GetBinaryPath()
if binaryPath == "" || seen[binaryPath] {
continue
}
seen[binaryPath] = true
binaryPaths = append(binaryPaths, binaryPath)
}
return binaryPaths
}
// GetLowLevelRuntimes returns a predefined list low-level runtimes from the specified config.
// nvidia* runtimes are ignored.
func GetLowLevelRuntimes(cfg Interface) []string {
var runtimes []string
isValidDefault := func(s string) bool {
if s == "" {
return false
}
// ignore nvidia* runtimes.
return !strings.HasPrefix(s, "nvidia")
}
if defaultRuntime := cfg.DefaultRuntime(); isValidDefault(defaultRuntime) {
runtimes = append(runtimes, defaultRuntime)
}
return append(runtimes, "runc", "crun")
}

45
pkg/config/toml/source-cli.go Normal file
View File

@ -0,0 +1,45 @@
/**
# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package toml
import (
"bytes"
"fmt"
"os/exec"
)
type tomlCliSource struct {
command string
args []string
}
func (c tomlCliSource) Load() (*Tree, error) {
//nolint:gosec // Subprocess launched with a potential tainted input or cmd arguments
cmd := exec.Command(c.command, c.args...)
var outb bytes.Buffer
var errb bytes.Buffer
cmd.Stdout = &outb
cmd.Stderr = &errb
if err := cmd.Run(); err != nil {
// TODO: Log to stderr in case of failure
return nil, fmt.Errorf("failed to run command %v %v: %w", c.command, c.args, err)
}
return LoadBytes(outb.Bytes())
}

12
pkg/config/toml/source.go
View File

@ -33,3 +33,15 @@ func FromFile(path string) Loader {
}
return tomlFile(path)
}
// FromCommandLine creates a TOML source from the output of a shell command and its corresponding args.
// If the command is empty, an empty config is returned.
func FromCommandLine(cmd string, args ...string) Loader {
if len(cmd) == 0 {
return Empty
}
return &tomlCliSource{
command: cmd,
args: args,
}
}

15
tools/container/nvidia-toolkit/run.go
View File

@ -22,11 +22,13 @@ const (
toolkitCommand = "toolkit"
toolkitSubDir = "toolkit"
defaultRuntime = "docker"
defaultRuntimeArgs = ""
defaultRuntime = "docker"
defaultRuntimeArgs = ""
defaultHostRootMount = "/host"
)
var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}}
var defaultLowLevelRuntimes = []string{"docker-runc", "runc", "crun"}
var waitingForSignal = make(chan bool, 1)
var signalReceived = make(chan bool, 1)
@ -155,6 +157,15 @@ func Run(c *cli.Context, o *options) error {
}
defer shutdown(o.pidFile)
if len(o.toolkitOptions.ContainerRuntimeRuntimes.Value()) == 0 {
lowlevelRuntimePaths, err := runtime.GetLowlevelRuntimePaths(&o.runtimeOptions, o.runtime)
if err != nil {
return fmt.Errorf("unable to determine runtime options: %w", err)
}
lowlevelRuntimePaths = append(lowlevelRuntimePaths, defaultLowLevelRuntimes...)
o.toolkitOptions.ContainerRuntimeRuntimes = *cli.NewStringSlice(lowlevelRuntimePaths...)
}
err = toolkit.Install(c, &o.toolkitOptions, o.toolkitRoot())
if err != nil {
return fmt.Errorf("unable to install toolkit: %v", err)

15
tools/container/runtime/containerd/containerd.go
View File

@ -23,6 +23,7 @@ import (
log "github.com/sirupsen/logrus"
cli "github.com/urfave/cli/v2"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
)
@ -85,6 +86,7 @@ func Setup(c *cli.Context, o *container.Options, co *Options) error {
cfg, err := containerd.New(
containerd.WithPath(o.Config),
containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)),
containerd.WithRuntimeType(co.runtimeType),
containerd.WithUseLegacyConfig(co.useLegacyConfig),
containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
@ -114,6 +116,7 @@ func Cleanup(c *cli.Context, o *container.Options, co *Options) error {
cfg, err := containerd.New(
containerd.WithPath(o.Config),
containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)),
containerd.WithRuntimeType(co.runtimeType),
containerd.WithUseLegacyConfig(co.useLegacyConfig),
containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
@ -164,3 +167,15 @@ func (o *Options) runtimeConfigOverride() (map[string]interface{}, error) {
return runtimeOptions, nil
}
func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error) {
cfg, err := containerd.New(
containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)),
containerd.WithRuntimeType(co.runtimeType),
containerd.WithUseLegacyConfig(co.useLegacyConfig),
)
if err != nil {
return nil, fmt.Errorf("unable to load containerd config: %w", err)
}
return engine.GetBinaryPathsForRuntimes(cfg), nil
}

13
tools/container/runtime/crio/crio.go
View File

@ -25,6 +25,7 @@ import (
cli "github.com/urfave/cli/v2"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
@ -117,6 +118,7 @@ func setupConfig(o *container.Options) error {
cfg, err := crio.New(
crio.WithPath(o.Config),
crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
@ -168,6 +170,7 @@ func cleanupConfig(o *container.Options) error {
cfg, err := crio.New(
crio.WithPath(o.Config),
crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
@ -190,3 +193,13 @@ func cleanupConfig(o *container.Options) error {
func RestartCrio(o *container.Options) error {
return o.Restart("crio", func(string) error { return fmt.Errorf("supporting crio via signal is unsupported") })
}
func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
cfg, err := crio.New(
crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)),
)
if err != nil {
return nil, fmt.Errorf("unable to load crio config: %w", err)
}
return engine.GetBinaryPathsForRuntimes(cfg), nil
}

11
tools/container/runtime/docker/docker.go
View File

@ -22,6 +22,7 @@ import (
log "github.com/sirupsen/logrus"
cli "github.com/urfave/cli/v2"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
)
@ -96,3 +97,13 @@ func Cleanup(c *cli.Context, o *container.Options) error {
func RestartDocker(o *container.Options) error {
return o.Restart("docker", SignalDocker)
}
func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
cfg, err := docker.New(
docker.WithPath(o.Config),
)
if err != nil {
return nil, fmt.Errorf("unable to load docker config: %w", err)
}
return engine.GetBinaryPathsForRuntimes(cfg), nil
}

13
tools/container/runtime/runtime.go
View File

@ -166,3 +166,16 @@ func Cleanup(c *cli.Context, opts *Options, runtime string) error {
return fmt.Errorf("undefined runtime %v", runtime)
}
}
func GetLowlevelRuntimePaths(opts *Options, runtime string) ([]string, error) {
switch runtime {
case containerd.Name:
return containerd.GetLowlevelRuntimePaths(&opts.Options, &opts.containerdOptions)
case crio.Name:
return crio.GetLowlevelRuntimePaths(&opts.Options)
case docker.Name:
return docker.GetLowlevelRuntimePaths(&opts.Options)
default:
return nil, fmt.Errorf("undefined runtime %v", runtime)
}
}

21
tools/container/toolkit/toolkit.go
View File

@ -24,7 +24,6 @@ import (
"path/filepath"
"strings"
toml "github.com/pelletier/go-toml"
log "github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
"tags.cncf.io/container-device-interface/pkg/cdi"
@ -419,7 +418,9 @@ func installLibrary(libName string, toolkitRoot string) error {
func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContainerCliExecutablePath string, nvidiaCTKPath string, nvidaContainerRuntimeHookPath string, opts *Options) error {
log.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
cfg, err := loadConfig(nvidiaContainerToolkitConfigSource)
cfg, err := config.New(
config.WithConfigFile(nvidiaContainerToolkitConfigSource),
)
if err != nil {
return fmt.Errorf("could not open source config file: %v", err)
}
@ -450,6 +451,12 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai
"nvidia-container-runtime-hook.path": nvidaContainerRuntimeHookPath,
"nvidia-container-runtime-hook.skip-mode-detection": opts.ContainerRuntimeHookSkipModeDetection,
}
toolkitRuntimeList := opts.ContainerRuntimeRuntimes.Value()
if len(toolkitRuntimeList) > 0 {
configValues["nvidia-container-runtime.runtimes"] = toolkitRuntimeList
}
for key, value := range configValues {
cfg.Set(key, value)
}
@ -503,16 +510,6 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai
return nil
}
func loadConfig(path string) (*toml.Tree, error) {
_, err := os.Stat(path)
if err == nil {
return toml.LoadFile(path)
} else if os.IsNotExist(err) {
return toml.TreeFromMap(nil)
}
return nil, err
}
// installContainerToolkitCLI installs the nvidia-ctk CLI executable and wrapper.
func installContainerToolkitCLI(toolkitDir string) (string, error) {
e := executable{