fetch current container runtime config through the command line

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

add default runtime binary path to runtimes field of toolkit config toml

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

[no-relnote] Get low-level runtimes consistently

We ensure that we use the same low-level runtimes regardless
of the runtime engine being configured. This ensures consistent
behaviour.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Co-authored-by: Evan Lezar <elezar@nvidia.com>

address review comment

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

tools/container/nvidia-toolkit/run.go

@@ -22,11 +22,13 @@ const (
 	toolkitCommand     = "toolkit"
 	toolkitSubDir      = "toolkit"
 
-	defaultRuntime     = "docker"
-	defaultRuntimeArgs = ""
+	defaultRuntime       = "docker"
+	defaultRuntimeArgs   = ""
+	defaultHostRootMount = "/host"
 )
 
 var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}}
+var defaultLowLevelRuntimes = []string{"docker-runc", "runc", "crun"}
 
 var waitingForSignal = make(chan bool, 1)
 var signalReceived = make(chan bool, 1)
@@ -155,6 +157,15 @@ func Run(c *cli.Context, o *options) error {
 	}
 	defer shutdown(o.pidFile)
 
+	if len(o.toolkitOptions.ContainerRuntimeRuntimes.Value()) == 0 {
+		lowlevelRuntimePaths, err := runtime.GetLowlevelRuntimePaths(&o.runtimeOptions, o.runtime)
+		if err != nil {
+			return fmt.Errorf("unable to determine runtime options: %w", err)
+		}
+		lowlevelRuntimePaths = append(lowlevelRuntimePaths, defaultLowLevelRuntimes...)
+
+		o.toolkitOptions.ContainerRuntimeRuntimes = *cli.NewStringSlice(lowlevelRuntimePaths...)
+	}
 	err = toolkit.Install(c, &o.toolkitOptions, o.toolkitRoot())
 	if err != nil {
 		return fmt.Errorf("unable to install toolkit: %v", err)

tools/container/runtime/containerd/containerd.go

@@ -23,6 +23,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
 	"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
 )
@@ -85,6 +86,7 @@ func Setup(c *cli.Context, o *container.Options, co *Options) error {
 
 	cfg, err := containerd.New(
 		containerd.WithPath(o.Config),
+		containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)),
 		containerd.WithRuntimeType(co.runtimeType),
 		containerd.WithUseLegacyConfig(co.useLegacyConfig),
 		containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
@@ -114,6 +116,7 @@ func Cleanup(c *cli.Context, o *container.Options, co *Options) error {
 
 	cfg, err := containerd.New(
 		containerd.WithPath(o.Config),
+		containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)),
 		containerd.WithRuntimeType(co.runtimeType),
 		containerd.WithUseLegacyConfig(co.useLegacyConfig),
 		containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
@@ -164,3 +167,15 @@ func (o *Options) runtimeConfigOverride() (map[string]interface{}, error) {
 
 	return runtimeOptions, nil
 }
+
+func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error) {
+	cfg, err := containerd.New(
+		containerd.WithConfigSource(containerd.CommandLineSource(o.HostRootMount)),
+		containerd.WithRuntimeType(co.runtimeType),
+		containerd.WithUseLegacyConfig(co.useLegacyConfig),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load containerd config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}

tools/container/runtime/crio/crio.go

@@ -25,6 +25,7 @@ import (
 	cli "github.com/urfave/cli/v2"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
 	"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
@@ -117,6 +118,7 @@ func setupConfig(o *container.Options) error {
 
 	cfg, err := crio.New(
 		crio.WithPath(o.Config),
+		crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)),
 	)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
@@ -168,6 +170,7 @@ func cleanupConfig(o *container.Options) error {
 
 	cfg, err := crio.New(
 		crio.WithPath(o.Config),
+		crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)),
 	)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
@@ -190,3 +193,13 @@ func cleanupConfig(o *container.Options) error {
 func RestartCrio(o *container.Options) error {
 	return o.Restart("crio", func(string) error { return fmt.Errorf("supporting crio via signal is unsupported") })
 }
+
+func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
+	cfg, err := crio.New(
+		crio.WithConfigSource(crio.CommandLineSource(o.HostRootMount)),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load crio config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}

tools/container/runtime/docker/docker.go

@@ -22,6 +22,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
 	"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
 )
@@ -96,3 +97,13 @@ func Cleanup(c *cli.Context, o *container.Options) error {
 func RestartDocker(o *container.Options) error {
 	return o.Restart("docker", SignalDocker)
 }
+
+func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
+	cfg, err := docker.New(
+		docker.WithPath(o.Config),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load docker config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}

tools/container/runtime/runtime.go

@@ -166,3 +166,16 @@ func Cleanup(c *cli.Context, opts *Options, runtime string) error {
 		return fmt.Errorf("undefined runtime %v", runtime)
 	}
 }
+
+func GetLowlevelRuntimePaths(opts *Options, runtime string) ([]string, error) {
+	switch runtime {
+	case containerd.Name:
+		return containerd.GetLowlevelRuntimePaths(&opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.GetLowlevelRuntimePaths(&opts.Options)
+	case docker.Name:
+		return docker.GetLowlevelRuntimePaths(&opts.Options)
+	default:
+		return nil, fmt.Errorf("undefined runtime %v", runtime)
+	}
+}

tools/container/toolkit/toolkit.go

@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	toml "github.com/pelletier/go-toml"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	"tags.cncf.io/container-device-interface/pkg/cdi"
@@ -419,7 +418,9 @@ func installLibrary(libName string, toolkitRoot string) error {
 func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContainerCliExecutablePath string, nvidiaCTKPath string, nvidaContainerRuntimeHookPath string, opts *Options) error {
 	log.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
 
-	cfg, err := loadConfig(nvidiaContainerToolkitConfigSource)
+	cfg, err := config.New(
+		config.WithConfigFile(nvidiaContainerToolkitConfigSource),
+	)
 	if err != nil {
 		return fmt.Errorf("could not open source config file: %v", err)
 	}
@@ -450,6 +451,12 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai
 		"nvidia-container-runtime-hook.path":                nvidaContainerRuntimeHookPath,
 		"nvidia-container-runtime-hook.skip-mode-detection": opts.ContainerRuntimeHookSkipModeDetection,
 	}
+
+	toolkitRuntimeList := opts.ContainerRuntimeRuntimes.Value()
+	if len(toolkitRuntimeList) > 0 {
+		configValues["nvidia-container-runtime.runtimes"] = toolkitRuntimeList
+	}
+
 	for key, value := range configValues {
 		cfg.Set(key, value)
 	}
@@ -503,16 +510,6 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai
 	return nil
 }
 
-func loadConfig(path string) (*toml.Tree, error) {
-	_, err := os.Stat(path)
-	if err == nil {
-		return toml.LoadFile(path)
-	} else if os.IsNotExist(err) {
-		return toml.TreeFromMap(nil)
-	}
-	return nil, err
-}
-
 // installContainerToolkitCLI installs the nvidia-ctk CLI executable and wrapper.
 func installContainerToolkitCLI(toolkitDir string) (string, error) {
 	e := executable{