fetch current container runtime config through the command line

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

add default runtime binary path to runtimes field of toolkit config toml

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

[no-relnote] Get low-level runtimes consistently

We ensure that we use the same low-level runtimes regardless
of the runtime engine being configured. This ensures consistent
behaviour.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Co-authored-by: Evan Lezar <elezar@nvidia.com>

address review comment

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

pkg/config/engine/api.go

@@ -23,4 +23,10 @@ type Interface interface {
 	Set(string, interface{})
 	RemoveRuntime(string) error
 	Save(string) (int64, error)
+	GetRuntimeConfig(string) (RuntimeConfig, error)
+}
+
+// RuntimeConfig defines the interface to query container runtime handler configuration
+type RuntimeConfig interface {
+	GetBinaryPath() string
 }

pkg/config/engine/containerd/config_v1.go

@@ -19,9 +19,8 @@ package containerd
 import (
 	"fmt"
 
-	"github.com/pelletier/go-toml"
-
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 // ConfigV1 represents a version 1 containerd config
@@ -39,11 +38,7 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error
 
 	config.Set("version", int64(1))
 
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
 	for _, r := range runtimeNamesForConfig {
 		options := config.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", r})
 		if options == nil {
@@ -157,3 +152,14 @@ func (c *ConfigV1) Set(key string, value interface{}) {
 func (c ConfigV1) Save(path string) (int64, error) {
 	return (Config)(c).Save(path)
 }
+
+func (c *ConfigV1) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", name})
+
+	return &containerdCfgRuntime{
+		tree: runtimeData,
+	}, nil
+}

pkg/config/engine/containerd/config_v1_test.go

@@ -138,7 +138,7 @@ func TestAddRuntimeV1(t *testing.T) {
 				`,
 		},
 		{
-			description: "options from runc take precedence over default runtime",
+			description: "options from the default runtime take precedence over runc",
 			config: `
 			[plugins]
 			[plugins.cri]
@@ -186,14 +186,14 @@ func TestAddRuntimeV1(t *testing.T) {
 						BinaryName = "/usr/bin/default"
 						SystemdCgroup = false
 					[plugins.cri.containerd.runtimes.test]
-					privileged_without_host_devices = true
-					runtime_engine = "engine"
-					runtime_root = "root"
-					runtime_type = "type"
+					privileged_without_host_devices = false
+					runtime_engine = "defaultengine"
+					runtime_root = "defaultroot"
+					runtime_type = "defaulttype"
 					[plugins.cri.containerd.runtimes.test.options]
 						BinaryName = "/usr/bin/test"
 						Runtime = "/usr/bin/test"
-						SystemdCgroup = true
+						SystemdCgroup = false
 				`,
 		},
 	}

pkg/config/engine/containerd/config_v2.go

@@ -19,6 +19,7 @@ package containerd
 import (
 	"fmt"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
@@ -31,11 +32,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
 
 	config.Set("version", int64(2))
 
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
 	for _, r := range runtimeNamesForConfig {
 		options := config.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", r})
 		if options == nil {

pkg/config/engine/containerd/config_v2_test.go

@@ -137,7 +137,7 @@ func TestAddRuntime(t *testing.T) {
 				`,
 		},
 		{
-			description: "options from runc take precedence over default runtime",
+			description: "options from the default runtime take precedence over runc",
 			config: `
 			version = 2
 			[plugins]
@@ -186,13 +186,13 @@ func TestAddRuntime(t *testing.T) {
 						BinaryName = "/usr/bin/default"
 						SystemdCgroup = false
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
-					privileged_without_host_devices = true
-					runtime_engine = "engine"
-					runtime_root = "root"
-					runtime_type = "type"
+					privileged_without_host_devices = false
+					runtime_engine = "defaultengine"
+					runtime_root = "defaultroot"
+					runtime_type = "defaulttype"
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
 						BinaryName = "/usr/bin/test"
-						SystemdCgroup = true
+						SystemdCgroup = false
 				`,
 		},
 	}
@@ -216,3 +216,99 @@ func TestAddRuntime(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRuntimeConfig(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	config := `
+	version = 2
+	[plugins]
+	[plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      ignore_blockio_not_enabled_errors = false
+      ignore_rdt_not_enabled_errors = false
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        base_runtime_spec = ""
+        cni_conf_dir = ""
+        cni_max_conf_num = 0
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        privileged_without_host_devices_all_devices_allowed = false
+        runtime_engine = ""
+        runtime_path = ""
+        runtime_root = ""
+        runtime_type = ""
+        sandbox_mode = ""
+        snapshotter = ""
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          base_runtime_spec = ""
+          cni_conf_dir = ""
+          cni_max_conf_num = 0
+          container_annotations = []
+          pod_annotations = []
+          privileged_without_host_devices = false
+          privileged_without_host_devices_all_devices_allowed = false
+          runtime_engine = ""
+          runtime_path = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+          sandbox_mode = "podsandbox"
+          snapshotter = ""
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
+            IoUid = 0
+            NoNewKeyring = false
+            NoPivotRoot = false
+            Root = ""
+            ShimCgroup = ""
+            SystemdCgroup = false
+`
+	testCases := []struct {
+		description   string
+		runtime       string
+		expected      string
+		expectedError error
+	}{
+		{
+			description:   "valid runtime config, existing runtime",
+			runtime:       "runc",
+			expected:      "/usr/bin/runc",
+			expectedError: nil,
+		},
+		{
+			description:   "valid runtime config, non-existing runtime",
+			runtime:       "some-other-runtime",
+			expected:      "",
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			cfg, err := toml.Load(config)
+			require.NoError(t, err)
+
+			c := &Config{
+				Logger: logger,
+				Tree:   cfg,
+			}
+			rc, err := c.GetRuntimeConfig(tc.runtime)
+			require.Equal(t, tc.expectedError, err)
+			require.Equal(t, tc.expected, rc.GetBinaryPath())
+		})
+	}
+}

pkg/config/engine/containerd/containerd.go

@@ -35,6 +35,22 @@ type Config struct {
 
 var _ engine.Interface = (*Config)(nil)
 
+type containerdCfgRuntime struct {
+	tree *toml.Tree
+}
+
+var _ engine.RuntimeConfig = (*containerdCfgRuntime)(nil)
+
+// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
+func (c *containerdCfgRuntime) GetBinaryPath() string {
+	if c == nil || c.tree == nil {
+		return ""
+	}
+
+	binPath, _ := c.tree.GetPath([]string{"options", "BinaryName"}).(string)
+	return binPath
+}
+
 // New creates a containerd config with the specified options
 func New(opts ...Option) (engine.Interface, error) {
 	b := &builder{
@@ -98,3 +114,27 @@ func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
 		return -1, fmt.Errorf("unsupported type for version field: %v", v)
 	}
 }
+
+func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
+	return &containerdCfgRuntime{
+		tree: runtimeData,
+	}, nil
+}
+
+// CommandLineSource returns the CLI-based containerd config loader
+func CommandLineSource(hostRoot string) toml.Loader {
+	commandLine := chrootIfRequired(hostRoot, "containerd", "config", "dump")
+	return toml.FromCommandLine(commandLine[0], commandLine[1:]...)
+}
+
+func chrootIfRequired(hostRoot string, commandLine ...string) []string {
+	if hostRoot == "" || hostRoot == "/" {
+		return commandLine
+	}
+
+	return append([]string{"chroot", hostRoot}, commandLine...)
+}

pkg/config/engine/crio/crio.go

@@ -30,6 +30,22 @@ type Config struct {
 	Logger logger.Interface
 }
 
+type crioRuntime struct {
+	tree *toml.Tree
+}
+
+var _ engine.RuntimeConfig = (*crioRuntime)(nil)
+
+// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
+func (c *crioRuntime) GetBinaryPath() string {
+	if c.tree != nil {
+		if binaryPath, ok := c.tree.GetPath([]string{"runtime_path"}).(string); ok {
+			return binaryPath
+		}
+	}
+	return ""
+}
+
 var _ engine.Interface = (*Config)(nil)
 
 // New creates a cri-o config with the specified options
@@ -65,11 +81,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
 
 	config := *c.Tree
 
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
 	for _, r := range runtimeNamesForConfig {
 		if options, ok := config.GetPath([]string{"crio", "runtime", "runtimes", r}).(*toml.Tree); ok {
 			c.Logger.Debugf("using options from runtime %v: %v", r, options.String())
@@ -129,3 +141,27 @@ func (c *Config) RemoveRuntime(name string) error {
 	*c.Tree = config
 	return nil
 }
+
+func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"crio", "runtime", "runtimes", name})
+	return &crioRuntime{
+		tree: runtimeData,
+	}, nil
+}
+
+// CommandLineSource returns the CLI-based crio config loader
+func CommandLineSource(hostRoot string) toml.Loader {
+	commandLine := chrootIfRequired(hostRoot, "crio", "status", "config")
+	return toml.FromCommandLine(commandLine[0], commandLine[1:]...)
+}
+
+func chrootIfRequired(hostRoot string, commandLine ...string) []string {
+	if hostRoot == "" || hostRoot == "/" {
+		return commandLine
+	}
+
+	return append([]string{"chroot", hostRoot}, commandLine...)
+}

pkg/config/engine/crio/crio_test.go

@@ -91,7 +91,7 @@ func TestAddRuntime(t *testing.T) {
 			`,
 		},
 		{
-			description: "options from runc take precedence over default runtime",
+			description: "options from the default runtime take precedence over runc",
 			config: `
 			[crio]
 			[crio.runtime]
@@ -120,7 +120,7 @@ func TestAddRuntime(t *testing.T) {
 			[crio.runtime.runtimes.test]
 			runtime_path = "/usr/bin/test"
 			runtime_type = "oci"
-			runc_option = "option"
+			default_option = "option"
 			`,
 		},
 	}
@@ -144,3 +144,61 @@ func TestAddRuntime(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRuntimeConfig(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	config := `
+[crio.image]
+signature_policy = "/etc/crio/policy.json"
+
+[crio.runtime]
+default_runtime = "crun"
+
+[crio.runtime.runtimes.crun]
+runtime_path = "/usr/libexec/crio/crun"
+runtime_root = "/run/crun"
+monitor_path = "/usr/libexec/crio/conmon"
+allowed_annotations = [
+    "io.containers.trace-syscall",
+]
+
+[crio.runtime.runtimes.runc]
+runtime_path = "/usr/libexec/crio/runc"
+runtime_root = "/run/runc"
+monitor_path = "/usr/libexec/crio/conmon"
+`
+	testCases := []struct {
+		description   string
+		runtime       string
+		expected      string
+		expectedError error
+	}{
+		{
+			description:   "valid runtime config, existing runtime",
+			runtime:       "crun",
+			expected:      "/usr/libexec/crio/crun",
+			expectedError: nil,
+		},
+		{
+			description:   "valid runtime config, non-existing runtime",
+			runtime:       "some-other-runtime",
+			expected:      "",
+			expectedError: nil,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			cfg, err := toml.Load(config)
+			require.NoError(t, err)
+
+			c := &Config{
+				Logger: logger,
+				Tree:   cfg,
+			}
+
+			rc, err := c.GetRuntimeConfig(tc.runtime)
+			require.Equal(t, tc.expectedError, err)
+			require.Equal(t, tc.expected, rc.GetBinaryPath())
+		})
+	}
+}

pkg/config/engine/docker/docker.go

@@ -35,6 +35,20 @@ type Config map[string]interface{}
 
 var _ engine.Interface = (*Config)(nil)
 
+type dockerRuntime map[string]interface{}
+
+var _ engine.RuntimeConfig = (*dockerRuntime)(nil)
+
+// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
+func (d dockerRuntime) GetBinaryPath() string {
+	if d == nil {
+		return ""
+	}
+
+	path, _ := d["path"].(string)
+	return path
+}
+
 // New creates a docker config with the specified options
 func New(opts ...Option) (engine.Interface, error) {
 	b := &builder{}
@@ -132,3 +146,22 @@ func (c Config) Save(path string) (int64, error) {
 	n, err := config.Raw(path).Write(output)
 	return int64(n), err
 }
+
+// GetRuntimeConfig returns the runtime info of the runtime passed as input
+func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+
+	cfg := *c
+
+	var runtimes map[string]interface{}
+	if _, ok := cfg["runtimes"]; ok {
+		runtimes = cfg["runtimes"].(map[string]interface{})
+		if r, ok := runtimes[name]; ok {
+			dr := dockerRuntime(r.(map[string]interface{}))
+			return &dr, nil
+		}
+	}
+	return &dockerRuntime{}, nil
+}

pkg/config/engine/docker/docker_test.go

@@ -213,3 +213,37 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 
 	}
 }
+
+func TestGetRuntimeConfig(t *testing.T) {
+	c := map[string]interface{}{
+		"runtimes": map[string]interface{}{
+			"nvidia": map[string]interface{}{
+				"path": "nvidia-container-runtime",
+				"args": []string{},
+			},
+		},
+	}
+	cfg := Config(c)
+
+	testCases := []struct {
+		description string
+		runtime     string
+		expected    string
+	}{
+		{
+			description: "existing runtime",
+			runtime:     "nvidia",
+			expected:    "nvidia-container-runtime",
+		},
+		{
+			description: "non-existent runtime",
+			runtime:     "some-other-runtime",
+			expected:    "",
+		},
+	}
+	for _, tc := range testCases {
+		rc, err := cfg.GetRuntimeConfig(tc.runtime)
+		require.NoError(t, err)
+		require.Equal(t, tc.expected, rc.GetBinaryPath())
+	}
+}

pkg/config/engine/engine.go

@@ -0,0 +1,63 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package engine
+
+import "strings"
+
+// GetBinaryPathsForRuntimes returns the list of binary paths for common runtimes.
+// The following list of runtimes is considered:
+//
+//	the default runtime, "runc", and "crun"
+//
+// If an nvidia* runtime is set as the default runtime, this is ignored.
+func GetBinaryPathsForRuntimes(cfg Interface) []string {
+
+	var binaryPaths []string
+	seen := make(map[string]bool)
+	for _, runtime := range GetLowLevelRuntimes(cfg) {
+		runtimeConfig, err := cfg.GetRuntimeConfig(runtime)
+		if err != nil {
+			// TODO: It will be useful to log the error when GetRuntimeConfig fails for a runtime
+			continue
+		}
+		binaryPath := runtimeConfig.GetBinaryPath()
+		if binaryPath == "" || seen[binaryPath] {
+			continue
+		}
+		seen[binaryPath] = true
+		binaryPaths = append(binaryPaths, binaryPath)
+	}
+
+	return binaryPaths
+}
+
+// GetLowLevelRuntimes returns a predefined list low-level runtimes from the specified config.
+// nvidia* runtimes are ignored.
+func GetLowLevelRuntimes(cfg Interface) []string {
+	var runtimes []string
+	isValidDefault := func(s string) bool {
+		if s == "" {
+			return false
+		}
+		// ignore nvidia* runtimes.
+		return !strings.HasPrefix(s, "nvidia")
+	}
+	if defaultRuntime := cfg.DefaultRuntime(); isValidDefault(defaultRuntime) {
+		runtimes = append(runtimes, defaultRuntime)
+	}
+	return append(runtimes, "runc", "crun")
+}

pkg/config/toml/source-cli.go

@@ -0,0 +1,45 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+)
+
+type tomlCliSource struct {
+	command string
+	args    []string
+}
+
+func (c tomlCliSource) Load() (*Tree, error) {
+	//nolint:gosec  // Subprocess launched with a potential tainted input or cmd arguments
+	cmd := exec.Command(c.command, c.args...)
+
+	var outb bytes.Buffer
+	var errb bytes.Buffer
+
+	cmd.Stdout = &outb
+	cmd.Stderr = &errb
+	if err := cmd.Run(); err != nil {
+		// TODO: Log to stderr in case of failure
+		return nil, fmt.Errorf("failed to run command %v %v: %w", c.command, c.args, err)
+	}
+
+	return LoadBytes(outb.Bytes())
+}

pkg/config/toml/source.go

@@ -33,3 +33,15 @@ func FromFile(path string) Loader {
 	}
 	return tomlFile(path)
 }
+
+// FromCommandLine creates a TOML source from the output of a shell command and its corresponding args.
+// If the command is empty, an empty config is returned.
+func FromCommandLine(cmd string, args ...string) Loader {
+	if len(cmd) == 0 {
+		return Empty
+	}
+	return &tomlCliSource{
+		command: cmd,
+		args:    args,
+	}
+}