From f477dc0df1007c07f997cd575b0c690897458ac1 Mon Sep 17 00:00:00 2001 From: Tariq Ibrahim Date: Thu, 8 Aug 2024 15:40:00 -0700 Subject: [PATCH] fetch current container runtime config through the command line Signed-off-by: Tariq Ibrahim add default runtime binary path to runtimes field of toolkit config toml Signed-off-by: Tariq Ibrahim [no-relnote] Get low-level runtimes consistently We ensure that we use the same low-level runtimes regardless of the runtime engine being configured. This ensures consistent behaviour. Signed-off-by: Evan Lezar Co-authored-by: Evan Lezar address review comment Signed-off-by: Tariq Ibrahim --- cmd/nvidia-ctk/runtime/configure/configure.go | 57 ++++++++- internal/config/toml.go | 11 ++ pkg/config/engine/api.go | 6 + pkg/config/engine/containerd/config_v1.go | 20 ++-- .../engine/containerd/config_v1_test.go | 12 +- pkg/config/engine/containerd/config_v2.go | 7 +- .../engine/containerd/config_v2_test.go | 108 +++++++++++++++++- pkg/config/engine/containerd/containerd.go | 40 +++++++ pkg/config/engine/crio/crio.go | 46 +++++++- pkg/config/engine/crio/crio_test.go | 62 +++++++++- pkg/config/engine/docker/docker.go | 33 ++++++ pkg/config/engine/docker/docker_test.go | 34 ++++++ pkg/config/engine/engine.go | 63 ++++++++++ pkg/config/toml/source-cli.go | 45 ++++++++ pkg/config/toml/source.go | 12 ++ tools/container/nvidia-toolkit/run.go | 15 ++- .../runtime/containerd/containerd.go | 15 +++ tools/container/runtime/crio/crio.go | 13 +++ tools/container/runtime/docker/docker.go | 11 ++ tools/container/runtime/runtime.go | 13 +++ tools/container/toolkit/toolkit.go | 21 ++-- 21 files changed, 596 insertions(+), 48 deletions(-) create mode 100644 pkg/config/engine/engine.go create mode 100644 pkg/config/toml/source-cli.go diff --git a/cmd/nvidia-ctk/runtime/configure/configure.go b/cmd/nvidia-ctk/runtime/configure/configure.go index 0b321d60..2f82ae6d 100644 --- a/cmd/nvidia-ctk/runtime/configure/configure.go +++ b/cmd/nvidia-ctk/runtime/configure/configure.go @@ -44,6 +44,10 @@ const ( defaultContainerdConfigFilePath = "/etc/containerd/config.toml" defaultCrioConfigFilePath = "/etc/crio/crio.conf" defaultDockerConfigFilePath = "/etc/docker/daemon.json" + + defaultConfigSource = configSourceFile + configSourceCommand = "command" + configSourceFile = "file" ) type command struct { @@ -64,6 +68,7 @@ type config struct { dryRun bool runtime string configFilePath string + configSource string mode string hookFilePath string @@ -120,6 +125,12 @@ func (m command) build() *cli.Command { Usage: "the config mode for runtimes that support multiple configuration mechanisms", Destination: &config.mode, }, + &cli.StringFlag{ + Name: "config-source", + Usage: "the source to retrieve the container runtime configuration; one of [command, file]\"", + Destination: &config.configSource, + Value: defaultConfigSource, + }, &cli.StringFlag{ Name: "oci-hook-path", Usage: "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.", @@ -202,6 +213,18 @@ func (m command) validateFlags(c *cli.Context, config *config) error { config.runtimeConfigOverrideJSON = "" } + switch config.configSource { + case configSourceCommand: + if config.runtime == "docker" { + m.logger.Warningf("A %v Config Source is not supported for %v; using %v", config.configSource, config.runtime, configSourceFile) + config.configSource = configSourceFile + } + case configSourceFile: + break + default: + return fmt.Errorf("unrecognized Config Source: %v", config.configSource) + } + return nil } @@ -220,20 +243,25 @@ func (m command) configureWrapper(c *cli.Context, config *config) error { func (m command) configureConfigFile(c *cli.Context, config *config) error { configFilePath := config.resolveConfigFilePath() - var cfg engine.Interface var err error + configSource, err := config.resolveConfigSource() + if err != nil { + return err + } + + var cfg engine.Interface switch config.runtime { case "containerd": cfg, err = containerd.New( containerd.WithLogger(m.logger), containerd.WithPath(configFilePath), - containerd.WithConfigSource(toml.FromFile(configFilePath)), + containerd.WithConfigSource(configSource), ) case "crio": cfg, err = crio.New( crio.WithLogger(m.logger), crio.WithPath(configFilePath), - crio.WithConfigSource(toml.FromFile(configFilePath)), + crio.WithConfigSource(configSource), ) case "docker": cfg, err = docker.New( @@ -295,6 +323,29 @@ func (c *config) resolveConfigFilePath() string { return "" } +// resolveConfigSource returns the default config source or the user provided config source +func (c *config) resolveConfigSource() (toml.Loader, error) { + switch c.configSource { + case configSourceCommand: + return c.getCommandConfigSource(), nil + case configSourceFile: + return toml.FromFile(c.configFilePath), nil + default: + return nil, fmt.Errorf("unrecognized config source: %s", c.configSource) + } +} + +// getConfigSourceCommand returns the default cli command to fetch the current runtime config +func (c *config) getCommandConfigSource() toml.Loader { + switch c.runtime { + case "containerd": + return containerd.CommandLineSource("") + case "crio": + return crio.CommandLineSource("") + } + return toml.Empty +} + // getOuputConfigPath returns the configured config path or "" if dry-run is enabled func (c *config) getOuputConfigPath() string { if c.dryRun { diff --git a/internal/config/toml.go b/internal/config/toml.go index a1d37428..4c39e6b4 100644 --- a/internal/config/toml.go +++ b/internal/config/toml.go @@ -170,11 +170,22 @@ func (t *Toml) Get(key string) interface{} { return (*toml.Tree)(t).Get(key) } +// GetDefault returns the value for the specified key and falls back to the default value if the Get call fails +func (t *Toml) GetDefault(key string, def interface{}) interface{} { + return (*toml.Tree)(t).GetDefault(key, def) +} + // Set sets the specified key to the specified value in the TOML config. func (t *Toml) Set(key string, value interface{}) { (*toml.Tree)(t).Set(key, value) } +// WriteTo encode the Tree as Toml and writes it to the writer w. +// Returns the number of bytes written in case of success, or an error if anything happened. +func (t *Toml) WriteTo(w io.Writer) (int64, error) { + return (*toml.Tree)(t).WriteTo(w) +} + // commentDefaults applies the required comments for default values to the Toml. func (t *Toml) commentDefaults() *Toml { asToml := (*toml.Tree)(t) diff --git a/pkg/config/engine/api.go b/pkg/config/engine/api.go index b074dadf..bc6c4a68 100644 --- a/pkg/config/engine/api.go +++ b/pkg/config/engine/api.go @@ -23,4 +23,10 @@ type Interface interface { Set(string, interface{}) RemoveRuntime(string) error Save(string) (int64, error) + GetRuntimeConfig(string) (RuntimeConfig, error) +} + +// RuntimeConfig defines the interface to query container runtime handler configuration +type RuntimeConfig interface { + GetBinaryPath() string } diff --git a/pkg/config/engine/containerd/config_v1.go b/pkg/config/engine/containerd/config_v1.go index e94a22f5..bfe1290b 100644 --- a/pkg/config/engine/containerd/config_v1.go +++ b/pkg/config/engine/containerd/config_v1.go @@ -19,9 +19,8 @@ package containerd import ( "fmt" - "github.com/pelletier/go-toml" - "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml" ) // ConfigV1 represents a version 1 containerd config @@ -39,11 +38,7 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error config.Set("version", int64(1)) - // By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config. - runtimeNamesForConfig := []string{"runc"} - if name, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok && name != "" { - runtimeNamesForConfig = append(runtimeNamesForConfig, name) - } + runtimeNamesForConfig := engine.GetLowLevelRuntimes(c) for _, r := range runtimeNamesForConfig { options := config.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", r}) if options == nil { @@ -157,3 +152,14 @@ func (c *ConfigV1) Set(key string, value interface{}) { func (c ConfigV1) Save(path string) (int64, error) { return (Config)(c).Save(path) } + +func (c *ConfigV1) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) { + if c == nil || c.Tree == nil { + return nil, fmt.Errorf("config is nil") + } + runtimeData := c.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", name}) + + return &containerdCfgRuntime{ + tree: runtimeData, + }, nil +} diff --git a/pkg/config/engine/containerd/config_v1_test.go b/pkg/config/engine/containerd/config_v1_test.go index 04e8ee9e..787ab30b 100644 --- a/pkg/config/engine/containerd/config_v1_test.go +++ b/pkg/config/engine/containerd/config_v1_test.go @@ -138,7 +138,7 @@ func TestAddRuntimeV1(t *testing.T) { `, }, { - description: "options from runc take precedence over default runtime", + description: "options from the default runtime take precedence over runc", config: ` [plugins] [plugins.cri] @@ -186,14 +186,14 @@ func TestAddRuntimeV1(t *testing.T) { BinaryName = "/usr/bin/default" SystemdCgroup = false [plugins.cri.containerd.runtimes.test] - privileged_without_host_devices = true - runtime_engine = "engine" - runtime_root = "root" - runtime_type = "type" + privileged_without_host_devices = false + runtime_engine = "defaultengine" + runtime_root = "defaultroot" + runtime_type = "defaulttype" [plugins.cri.containerd.runtimes.test.options] BinaryName = "/usr/bin/test" Runtime = "/usr/bin/test" - SystemdCgroup = true + SystemdCgroup = false `, }, } diff --git a/pkg/config/engine/containerd/config_v2.go b/pkg/config/engine/containerd/config_v2.go index 8f3e601f..562ccdf8 100644 --- a/pkg/config/engine/containerd/config_v2.go +++ b/pkg/config/engine/containerd/config_v2.go @@ -19,6 +19,7 @@ package containerd import ( "fmt" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine" "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml" ) @@ -31,11 +32,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error { config.Set("version", int64(2)) - // By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config. - runtimeNamesForConfig := []string{"runc"} - if name, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok && name != "" { - runtimeNamesForConfig = append(runtimeNamesForConfig, name) - } + runtimeNamesForConfig := engine.GetLowLevelRuntimes(c) for _, r := range runtimeNamesForConfig { options := config.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", r}) if options == nil { diff --git a/pkg/config/engine/containerd/config_v2_test.go b/pkg/config/engine/containerd/config_v2_test.go index 2c2cd7d6..a9e0c3be 100644 --- a/pkg/config/engine/containerd/config_v2_test.go +++ b/pkg/config/engine/containerd/config_v2_test.go @@ -137,7 +137,7 @@ func TestAddRuntime(t *testing.T) { `, }, { - description: "options from runc take precedence over default runtime", + description: "options from the default runtime take precedence over runc", config: ` version = 2 [plugins] @@ -186,13 +186,13 @@ func TestAddRuntime(t *testing.T) { BinaryName = "/usr/bin/default" SystemdCgroup = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test] - privileged_without_host_devices = true - runtime_engine = "engine" - runtime_root = "root" - runtime_type = "type" + privileged_without_host_devices = false + runtime_engine = "defaultengine" + runtime_root = "defaultroot" + runtime_type = "defaulttype" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options] BinaryName = "/usr/bin/test" - SystemdCgroup = true + SystemdCgroup = false `, }, } @@ -216,3 +216,99 @@ func TestAddRuntime(t *testing.T) { }) } } + +func TestGetRuntimeConfig(t *testing.T) { + logger, _ := testlog.NewNullLogger() + config := ` + version = 2 + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "nvidia" + disable_snapshot_annotations = true + discard_unpacked_layers = false + ignore_blockio_not_enabled_errors = false + ignore_rdt_not_enabled_errors = false + no_pivot = false + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + sandbox_mode = "" + snapshotter = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + sandbox_mode = "podsandbox" + snapshotter = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 + IoUid = 0 + NoNewKeyring = false + NoPivotRoot = false + Root = "" + ShimCgroup = "" + SystemdCgroup = false +` + testCases := []struct { + description string + runtime string + expected string + expectedError error + }{ + { + description: "valid runtime config, existing runtime", + runtime: "runc", + expected: "/usr/bin/runc", + expectedError: nil, + }, + { + description: "valid runtime config, non-existing runtime", + runtime: "some-other-runtime", + expected: "", + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + cfg, err := toml.Load(config) + require.NoError(t, err) + + c := &Config{ + Logger: logger, + Tree: cfg, + } + rc, err := c.GetRuntimeConfig(tc.runtime) + require.Equal(t, tc.expectedError, err) + require.Equal(t, tc.expected, rc.GetBinaryPath()) + }) + } +} diff --git a/pkg/config/engine/containerd/containerd.go b/pkg/config/engine/containerd/containerd.go index 92bf9fff..01791c09 100644 --- a/pkg/config/engine/containerd/containerd.go +++ b/pkg/config/engine/containerd/containerd.go @@ -35,6 +35,22 @@ type Config struct { var _ engine.Interface = (*Config)(nil) +type containerdCfgRuntime struct { + tree *toml.Tree +} + +var _ engine.RuntimeConfig = (*containerdCfgRuntime)(nil) + +// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler +func (c *containerdCfgRuntime) GetBinaryPath() string { + if c == nil || c.tree == nil { + return "" + } + + binPath, _ := c.tree.GetPath([]string{"options", "BinaryName"}).(string) + return binPath +} + // New creates a containerd config with the specified options func New(opts ...Option) (engine.Interface, error) { b := &builder{ @@ -98,3 +114,27 @@ func (c *Config) parseVersion(useLegacyConfig bool) (int, error) { return -1, fmt.Errorf("unsupported type for version field: %v", v) } } + +func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) { + if c == nil || c.Tree == nil { + return nil, fmt.Errorf("config is nil") + } + runtimeData := c.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) + return &containerdCfgRuntime{ + tree: runtimeData, + }, nil +} + +// CommandLineSource returns the CLI-based containerd config loader +func CommandLineSource(hostRoot string) toml.Loader { + commandLine := chrootIfRequired(hostRoot, "containerd", "config", "dump") + return toml.FromCommandLine(commandLine[0], commandLine[1:]...) +} + +func chrootIfRequired(hostRoot string, commandLine ...string) []string { + if hostRoot == "" || hostRoot == "/" { + return commandLine + } + + return append([]string{"chroot", hostRoot}, commandLine...) +} diff --git a/pkg/config/engine/crio/crio.go b/pkg/config/engine/crio/crio.go index d243372d..acaa1185 100644 --- a/pkg/config/engine/crio/crio.go +++ b/pkg/config/engine/crio/crio.go @@ -30,6 +30,22 @@ type Config struct { Logger logger.Interface } +type crioRuntime struct { + tree *toml.Tree +} + +var _ engine.RuntimeConfig = (*crioRuntime)(nil) + +// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler +func (c *crioRuntime) GetBinaryPath() string { + if c.tree != nil { + if binaryPath, ok := c.tree.GetPath([]string{"runtime_path"}).(string); ok { + return binaryPath + } + } + return "" +} + var _ engine.Interface = (*Config)(nil) // New creates a cri-o config with the specified options @@ -65,11 +81,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error { config := *c.Tree - // By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config. - runtimeNamesForConfig := []string{"runc"} - if name, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok && name != "" { - runtimeNamesForConfig = append(runtimeNamesForConfig, name) - } + runtimeNamesForConfig := engine.GetLowLevelRuntimes(c) for _, r := range runtimeNamesForConfig { if options, ok := config.GetPath([]string{"crio", "runtime", "runtimes", r}).(*toml.Tree); ok { c.Logger.Debugf("using options from runtime %v: %v", r, options.String()) @@ -129,3 +141,27 @@ func (c *Config) RemoveRuntime(name string) error { *c.Tree = config return nil } + +func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) { + if c == nil || c.Tree == nil { + return nil, fmt.Errorf("config is nil") + } + runtimeData := c.GetSubtreeByPath([]string{"crio", "runtime", "runtimes", name}) + return &crioRuntime{ + tree: runtimeData, + }, nil +} + +// CommandLineSource returns the CLI-based crio config loader +func CommandLineSource(hostRoot string) toml.Loader { + commandLine := chrootIfRequired(hostRoot, "crio", "status", "config") + return toml.FromCommandLine(commandLine[0], commandLine[1:]...) +} + +func chrootIfRequired(hostRoot string, commandLine ...string) []string { + if hostRoot == "" || hostRoot == "/" { + return commandLine + } + + return append([]string{"chroot", hostRoot}, commandLine...) +} diff --git a/pkg/config/engine/crio/crio_test.go b/pkg/config/engine/crio/crio_test.go index d2b81b9e..1468d467 100644 --- a/pkg/config/engine/crio/crio_test.go +++ b/pkg/config/engine/crio/crio_test.go @@ -91,7 +91,7 @@ func TestAddRuntime(t *testing.T) { `, }, { - description: "options from runc take precedence over default runtime", + description: "options from the default runtime take precedence over runc", config: ` [crio] [crio.runtime] @@ -120,7 +120,7 @@ func TestAddRuntime(t *testing.T) { [crio.runtime.runtimes.test] runtime_path = "/usr/bin/test" runtime_type = "oci" - runc_option = "option" + default_option = "option" `, }, } @@ -144,3 +144,61 @@ func TestAddRuntime(t *testing.T) { }) } } + +func TestGetRuntimeConfig(t *testing.T) { + logger, _ := testlog.NewNullLogger() + config := ` +[crio.image] +signature_policy = "/etc/crio/policy.json" + +[crio.runtime] +default_runtime = "crun" + +[crio.runtime.runtimes.crun] +runtime_path = "/usr/libexec/crio/crun" +runtime_root = "/run/crun" +monitor_path = "/usr/libexec/crio/conmon" +allowed_annotations = [ + "io.containers.trace-syscall", +] + +[crio.runtime.runtimes.runc] +runtime_path = "/usr/libexec/crio/runc" +runtime_root = "/run/runc" +monitor_path = "/usr/libexec/crio/conmon" +` + testCases := []struct { + description string + runtime string + expected string + expectedError error + }{ + { + description: "valid runtime config, existing runtime", + runtime: "crun", + expected: "/usr/libexec/crio/crun", + expectedError: nil, + }, + { + description: "valid runtime config, non-existing runtime", + runtime: "some-other-runtime", + expected: "", + expectedError: nil, + }, + } + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + cfg, err := toml.Load(config) + require.NoError(t, err) + + c := &Config{ + Logger: logger, + Tree: cfg, + } + + rc, err := c.GetRuntimeConfig(tc.runtime) + require.Equal(t, tc.expectedError, err) + require.Equal(t, tc.expected, rc.GetBinaryPath()) + }) + } +} diff --git a/pkg/config/engine/docker/docker.go b/pkg/config/engine/docker/docker.go index 45a96255..3f0f3e47 100644 --- a/pkg/config/engine/docker/docker.go +++ b/pkg/config/engine/docker/docker.go @@ -35,6 +35,20 @@ type Config map[string]interface{} var _ engine.Interface = (*Config)(nil) +type dockerRuntime map[string]interface{} + +var _ engine.RuntimeConfig = (*dockerRuntime)(nil) + +// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler +func (d dockerRuntime) GetBinaryPath() string { + if d == nil { + return "" + } + + path, _ := d["path"].(string) + return path +} + // New creates a docker config with the specified options func New(opts ...Option) (engine.Interface, error) { b := &builder{} @@ -132,3 +146,22 @@ func (c Config) Save(path string) (int64, error) { n, err := config.Raw(path).Write(output) return int64(n), err } + +// GetRuntimeConfig returns the runtime info of the runtime passed as input +func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) { + if c == nil { + return nil, fmt.Errorf("config is nil") + } + + cfg := *c + + var runtimes map[string]interface{} + if _, ok := cfg["runtimes"]; ok { + runtimes = cfg["runtimes"].(map[string]interface{}) + if r, ok := runtimes[name]; ok { + dr := dockerRuntime(r.(map[string]interface{})) + return &dr, nil + } + } + return &dockerRuntime{}, nil +} diff --git a/pkg/config/engine/docker/docker_test.go b/pkg/config/engine/docker/docker_test.go index 115e189e..a279c7be 100644 --- a/pkg/config/engine/docker/docker_test.go +++ b/pkg/config/engine/docker/docker_test.go @@ -213,3 +213,37 @@ func TestUpdateConfigRuntimes(t *testing.T) { } } + +func TestGetRuntimeConfig(t *testing.T) { + c := map[string]interface{}{ + "runtimes": map[string]interface{}{ + "nvidia": map[string]interface{}{ + "path": "nvidia-container-runtime", + "args": []string{}, + }, + }, + } + cfg := Config(c) + + testCases := []struct { + description string + runtime string + expected string + }{ + { + description: "existing runtime", + runtime: "nvidia", + expected: "nvidia-container-runtime", + }, + { + description: "non-existent runtime", + runtime: "some-other-runtime", + expected: "", + }, + } + for _, tc := range testCases { + rc, err := cfg.GetRuntimeConfig(tc.runtime) + require.NoError(t, err) + require.Equal(t, tc.expected, rc.GetBinaryPath()) + } +} diff --git a/pkg/config/engine/engine.go b/pkg/config/engine/engine.go new file mode 100644 index 00000000..b20d0558 --- /dev/null +++ b/pkg/config/engine/engine.go @@ -0,0 +1,63 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package engine + +import "strings" + +// GetBinaryPathsForRuntimes returns the list of binary paths for common runtimes. +// The following list of runtimes is considered: +// +// the default runtime, "runc", and "crun" +// +// If an nvidia* runtime is set as the default runtime, this is ignored. +func GetBinaryPathsForRuntimes(cfg Interface) []string { + + var binaryPaths []string + seen := make(map[string]bool) + for _, runtime := range GetLowLevelRuntimes(cfg) { + runtimeConfig, err := cfg.GetRuntimeConfig(runtime) + if err != nil { + // TODO: It will be useful to log the error when GetRuntimeConfig fails for a runtime + continue + } + binaryPath := runtimeConfig.GetBinaryPath() + if binaryPath == "" || seen[binaryPath] { + continue + } + seen[binaryPath] = true + binaryPaths = append(binaryPaths, binaryPath) + } + + return binaryPaths +} + +// GetLowLevelRuntimes returns a predefined list low-level runtimes from the specified config. +// nvidia* runtimes are ignored. +func GetLowLevelRuntimes(cfg Interface) []string { + var runtimes []string + isValidDefault := func(s string) bool { + if s == "" { + return false + } + // ignore nvidia* runtimes. + return !strings.HasPrefix(s, "nvidia") + } + if defaultRuntime := cfg.DefaultRuntime(); isValidDefault(defaultRuntime) { + runtimes = append(runtimes, defaultRuntime) + } + return append(runtimes, "runc", "crun") +} diff --git a/pkg/config/toml/source-cli.go b/pkg/config/toml/source-cli.go new file mode 100644 index 00000000..a3469583 --- /dev/null +++ b/pkg/config/toml/source-cli.go @@ -0,0 +1,45 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package toml + +import ( + "bytes" + "fmt" + "os/exec" +) + +type tomlCliSource struct { + command string + args []string +} + +func (c tomlCliSource) Load() (*Tree, error) { + //nolint:gosec // Subprocess launched with a potential tainted input or cmd arguments + cmd := exec.Command(c.command, c.args...) + + var outb bytes.Buffer + var errb bytes.Buffer + + cmd.Stdout = &outb + cmd.Stderr = &errb + if err := cmd.Run(); err != nil { + // TODO: Log to stderr in case of failure + return nil, fmt.Errorf("failed to run command %v %v: %w", c.command, c.args, err) + } + + return LoadBytes(outb.Bytes()) +} diff --git a/pkg/config/toml/source.go b/pkg/config/toml/source.go index 2bd9191d..5cb0b7fe 100644 --- a/pkg/config/toml/source.go +++ b/pkg/config/toml/source.go @@ -33,3 +33,15 @@ func FromFile(path string) Loader { } return tomlFile(path) } + +// FromCommandLine creates a TOML source from the output of a shell command and its corresponding args. +// If the command is empty, an empty config is returned. +func FromCommandLine(cmd string, args ...string) Loader { + if len(cmd) == 0 { + return Empty + } + return &tomlCliSource{ + command: cmd, + args: args, + } +} diff --git a/tools/container/nvidia-toolkit/run.go b/tools/container/nvidia-toolkit/run.go index 65a807ec..8e389e94 100644 --- a/tools/container/nvidia-toolkit/run.go +++ b/tools/container/nvidia-toolkit/run.go @@ -22,11 +22,13 @@ const ( toolkitCommand = "toolkit" toolkitSubDir = "toolkit" - defaultRuntime = "docker" - defaultRuntimeArgs = "" + defaultRuntime = "docker" + defaultRuntimeArgs = "" + defaultHostRootMount = "/host" ) var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}} +var defaultLowLevelRuntimes = []string{"docker-runc", "runc", "crun"} var waitingForSignal = make(chan bool, 1) var signalReceived = make(chan bool, 1) @@ -155,6 +157,15 @@ func Run(c *cli.Context, o *options) error { } defer shutdown(o.pidFile) + if len(o.toolkitOptions.ContainerRuntimeRuntimes.Value()) == 0 { + lowlevelRuntimePaths, err := runtime.GetLowlevelRuntimePaths(&o.runtimeOptions, o.runtime) + if err != nil { + return fmt.Errorf("unable to determine runtime options: %w", err) + } + lowlevelRuntimePaths = append(lowlevelRuntimePaths, defaultLowLevelRuntimes...) + + o.toolkitOptions.ContainerRuntimeRuntimes = *cli.NewStringSlice(lowlevelRuntimePaths...) + } err = toolkit.Install(c, &o.toolkitOptions, o.toolkitRoot()) if err != nil { return fmt.Errorf("unable to install toolkit: %v", err) diff --git a/tools/container/runtime/containerd/containerd.go b/tools/container/runtime/containerd/containerd.go index df5db6d6..6a5df5ad 100644 --- a/tools/container/runtime/containerd/containerd.go +++ b/tools/container/runtime/containerd/containerd.go @@ -23,6 +23,7 @@ import ( log "github.com/sirupsen/logrus" cli "github.com/urfave/cli/v2" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine" "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd" "github.com/NVIDIA/nvidia-container-toolkit/tools/container" ) @@ -85,6 +86,7 @@ func Setup(c *cli.Context, o *container.Options, co *Options) error { cfg, err := containerd.New( containerd.WithPath(o.Config), + containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)), containerd.WithRuntimeType(co.runtimeType), containerd.WithUseLegacyConfig(co.useLegacyConfig), containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...), @@ -114,6 +116,7 @@ func Cleanup(c *cli.Context, o *container.Options, co *Options) error { cfg, err := containerd.New( containerd.WithPath(o.Config), + containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)), containerd.WithRuntimeType(co.runtimeType), containerd.WithUseLegacyConfig(co.useLegacyConfig), containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...), @@ -164,3 +167,15 @@ func (o *Options) runtimeConfigOverride() (map[string]interface{}, error) { return runtimeOptions, nil } + +func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error) { + cfg, err := containerd.New( + containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)), + containerd.WithRuntimeType(co.runtimeType), + containerd.WithUseLegacyConfig(co.useLegacyConfig), + ) + if err != nil { + return nil, fmt.Errorf("unable to load containerd config: %w", err) + } + return engine.GetBinaryPathsForRuntimes(cfg), nil +} diff --git a/tools/container/runtime/crio/crio.go b/tools/container/runtime/crio/crio.go index 69482191..e3fa77f6 100644 --- a/tools/container/runtime/crio/crio.go +++ b/tools/container/runtime/crio/crio.go @@ -25,6 +25,7 @@ import ( cli "github.com/urfave/cli/v2" "github.com/NVIDIA/nvidia-container-toolkit/internal/config" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine" "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio" "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook" "github.com/NVIDIA/nvidia-container-toolkit/tools/container" @@ -117,6 +118,7 @@ func setupConfig(o *container.Options) error { cfg, err := crio.New( crio.WithPath(o.Config), + crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)), ) if err != nil { return fmt.Errorf("unable to load config: %v", err) @@ -168,6 +170,7 @@ func cleanupConfig(o *container.Options) error { cfg, err := crio.New( crio.WithPath(o.Config), + crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)), ) if err != nil { return fmt.Errorf("unable to load config: %v", err) @@ -190,3 +193,13 @@ func cleanupConfig(o *container.Options) error { func RestartCrio(o *container.Options) error { return o.Restart("crio", func(string) error { return fmt.Errorf("supporting crio via signal is unsupported") }) } + +func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) { + cfg, err := crio.New( + crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)), + ) + if err != nil { + return nil, fmt.Errorf("unable to load crio config: %w", err) + } + return engine.GetBinaryPathsForRuntimes(cfg), nil +} diff --git a/tools/container/runtime/docker/docker.go b/tools/container/runtime/docker/docker.go index f3524825..0d6a4ff8 100644 --- a/tools/container/runtime/docker/docker.go +++ b/tools/container/runtime/docker/docker.go @@ -22,6 +22,7 @@ import ( log "github.com/sirupsen/logrus" cli "github.com/urfave/cli/v2" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine" "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker" "github.com/NVIDIA/nvidia-container-toolkit/tools/container" ) @@ -96,3 +97,13 @@ func Cleanup(c *cli.Context, o *container.Options) error { func RestartDocker(o *container.Options) error { return o.Restart("docker", SignalDocker) } + +func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) { + cfg, err := docker.New( + docker.WithPath(o.Config), + ) + if err != nil { + return nil, fmt.Errorf("unable to load docker config: %w", err) + } + return engine.GetBinaryPathsForRuntimes(cfg), nil +} diff --git a/tools/container/runtime/runtime.go b/tools/container/runtime/runtime.go index 6fa5442a..d5438e88 100644 --- a/tools/container/runtime/runtime.go +++ b/tools/container/runtime/runtime.go @@ -166,3 +166,16 @@ func Cleanup(c *cli.Context, opts *Options, runtime string) error { return fmt.Errorf("undefined runtime %v", runtime) } } + +func GetLowlevelRuntimePaths(opts *Options, runtime string) ([]string, error) { + switch runtime { + case containerd.Name: + return containerd.GetLowlevelRuntimePaths(&opts.Options, &opts.containerdOptions) + case crio.Name: + return crio.GetLowlevelRuntimePaths(&opts.Options) + case docker.Name: + return docker.GetLowlevelRuntimePaths(&opts.Options) + default: + return nil, fmt.Errorf("undefined runtime %v", runtime) + } +} diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index 484d7891..937798e6 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -24,7 +24,6 @@ import ( "path/filepath" "strings" - toml "github.com/pelletier/go-toml" log "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "tags.cncf.io/container-device-interface/pkg/cdi" @@ -419,7 +418,9 @@ func installLibrary(libName string, toolkitRoot string) error { func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContainerCliExecutablePath string, nvidiaCTKPath string, nvidaContainerRuntimeHookPath string, opts *Options) error { log.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath) - cfg, err := loadConfig(nvidiaContainerToolkitConfigSource) + cfg, err := config.New( + config.WithConfigFile(nvidiaContainerToolkitConfigSource), + ) if err != nil { return fmt.Errorf("could not open source config file: %v", err) } @@ -450,6 +451,12 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai "nvidia-container-runtime-hook.path": nvidaContainerRuntimeHookPath, "nvidia-container-runtime-hook.skip-mode-detection": opts.ContainerRuntimeHookSkipModeDetection, } + + toolkitRuntimeList := opts.ContainerRuntimeRuntimes.Value() + if len(toolkitRuntimeList) > 0 { + configValues["nvidia-container-runtime.runtimes"] = toolkitRuntimeList + } + for key, value := range configValues { cfg.Set(key, value) } @@ -503,16 +510,6 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai return nil } -func loadConfig(path string) (*toml.Tree, error) { - _, err := os.Stat(path) - if err == nil { - return toml.LoadFile(path) - } else if os.IsNotExist(err) { - return toml.TreeFromMap(nil) - } - return nil, err -} - // installContainerToolkitCLI installs the nvidia-ctk CLI executable and wrapper. func installContainerToolkitCLI(toolkitDir string) (string, error) { e := executable{