Merge pull request #763 from elezar/allow-config-override-by-envvar

Add support for specifying the config file path in an environment variable

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"path"
 	"reflect"
 	"strings"
 	"sync"
@@ -14,11 +13,6 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 )
 
-const (
-	configPath = "/etc/nvidia-container-runtime/config.toml"
-	driverPath = "/run/nvidia/driver"
-)
-
 // hookConfig wraps the toolkit config.
 // This allows for functions to be defined on the local type.
 type hookConfig struct {
@@ -29,29 +23,27 @@ type hookConfig struct {
 
 // loadConfig loads the required paths for the hook config.
 func loadConfig() (*config.Config, error) {
-	var configPaths []string
+	configFilePath, required := getConfigFilePath()
-	var required bool
+	cfg, err := config.New(
-	if len(*configflag) != 0 {
+		config.WithConfigFile(configFilePath),
-		configPaths = append(configPaths, *configflag)
+		config.WithRequired(true),
-		required = true
+	)
-	} else {
+	if err == nil {
-		configPaths = append(configPaths, path.Join(driverPath, configPath), configPath)
+		return cfg.Config()
+	} else if os.IsNotExist(err) && !required {
+		return config.GetDefault()
 	}
+	return nil, fmt.Errorf("couldn't open required configuration file: %v", err)
+}
 
-	for _, p := range configPaths {
+func getConfigFilePath() (string, bool) {
-		cfg, err := config.New(
+	if configFromFlag := *configflag; configFromFlag != "" {
-			config.WithConfigFile(p),
+		return configFromFlag, true
-			config.WithRequired(true),
-		)
-		if err == nil {
-			return cfg.Config()
-		} else if os.IsNotExist(err) && !required {
-			continue
-		}
-		return nil, fmt.Errorf("couldn't open required configuration file: %v", err)
 	}
-
+	if configFromEnvvar := os.Getenv(config.FilePathOverrideEnvVar); configFromEnvvar != "" {
-	return config.GetDefault()
+		return configFromEnvvar, true
+	}
+	return config.GetConfigFilePath(), false
 }
 
 func getHookConfig() (*hookConfig, error) {

cmd/nvidia-ctk-installer/toolkit/installer/directory.go

@@ -28,7 +28,7 @@ type createDirectory struct {
 	logger logger.Interface
 }
 
-func (t *toolkitInstaller) createDirectory() Installer {
+func (t *ToolkitInstaller) createDirectory() Installer {
 	return &createDirectory{
 		logger: t.logger,
 	}

cmd/nvidia-ctk-installer/toolkit/installer/executables.go

@@ -28,20 +28,18 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 )
 
 type executable struct {
 	requiresKernelModule bool
 	path                 string
 	symlink              string
-	args                 []string
 	env                  map[string]string
 }
 
-func (t *toolkitInstaller) collectExecutables(destDir string) ([]Installer, error) {
+func (t *ToolkitInstaller) collectExecutables(destDir string) ([]Installer, error) {
-	configHome := filepath.Join(destDir, ".config")
+	configFilePath := t.ConfigFilePath(destDir)
-	configDir := filepath.Join(configHome, "nvidia-container-runtime")
-	configPath := filepath.Join(configDir, "config.toml")
 
 	executables := []executable{
 		{
@@ -56,7 +54,7 @@ func (t *toolkitInstaller) collectExecutables(destDir string) ([]Installer, erro
 			path:                 runtime.Path,
 			requiresKernelModule: true,
 			env: map[string]string{
-				"XDG_CONFIG_HOME": configHome,
+				config.FilePathOverrideEnvVar: configFilePath,
 			},
 		}
 		executables = append(executables, e)
@@ -72,7 +70,9 @@ func (t *toolkitInstaller) collectExecutables(destDir string) ([]Installer, erro
 		executable{
 			path:    "nvidia-container-runtime-hook",
 			symlink: "nvidia-container-toolkit",
-			args:    []string{fmt.Sprintf("-config %s", configPath)},
+			env: map[string]string{
+				config.FilePathOverrideEnvVar: configFilePath,
+			},
 		},
 	)
 
@@ -94,7 +94,6 @@ func (t *toolkitInstaller) collectExecutables(destDir string) ([]Installer, erro
 			Source:            executablePath,
 			WrappedExecutable: dotRealFilename,
 			CheckModules:      executable.requiresKernelModule,
-			Args:              executable.args,
 			Envvars: map[string]string{
 				"PATH": strings.Join([]string{destDir, "$PATH"}, ":"),
 			},
@@ -124,7 +123,6 @@ type wrapper struct {
 	Envvars           map[string]string
 	WrappedExecutable string
 	CheckModules      bool
-	Args              []string
 }
 
 type render struct {
@@ -165,9 +163,6 @@ fi
 {{$key}}={{$value}} \
 {{- end }}
 	{{ .DestDir }}/{{ .WrappedExecutable }} \
-{{- range $arg := .Args }}
-		{{$arg}} \
-{{- end }}
 		"$@"
 `
 

cmd/nvidia-ctk-installer/toolkit/installer/executables_test.go

@@ -68,19 +68,6 @@ fi
 PATH=/foo/bar/baz \
 	/dest-dir/some-runtime \
 		"$@"
-`,
-		},
-		{
-			description: "args are added",
-			w: &wrapper{
-				WrappedExecutable: "some-runtime",
-				Args:              []string{"--config foo", "bar"},
-			},
-			expected: `#! /bin/sh
-	/dest-dir/some-runtime \
-		--config foo \
-		bar \
-		"$@"
 `,
 		},
 	}

cmd/nvidia-ctk-installer/toolkit/installer/installer.go

@@ -33,7 +33,7 @@ type Installer interface {
 	Install(string) error
 }
 
-type toolkitInstaller struct {
+type ToolkitInstaller struct {
 	logger       logger.Interface
 	ignoreErrors bool
 	sourceRoot   string
@@ -43,11 +43,11 @@ type toolkitInstaller struct {
 	ensureTargetDirectory Installer
 }
 
-var _ Installer = (*toolkitInstaller)(nil)
+var _ Installer = (*ToolkitInstaller)(nil)
 
 // New creates a toolkit installer with the specified options.
-func New(opts ...Option) (Installer, error) {
+func New(opts ...Option) (*ToolkitInstaller, error) {
-	t := &toolkitInstaller{
+	t := &ToolkitInstaller{
 		sourceRoot: "/",
 	}
 	for _, opt := range opts {
@@ -73,7 +73,7 @@ func New(opts ...Option) (Installer, error) {
 }
 
 // Install ensures that the required toolkit files are installed in the specified directory.
-func (t *toolkitInstaller) Install(destDir string) error {
+func (t *ToolkitInstaller) Install(destDir string) error {
 	var installers []Installer
 
 	installers = append(installers, t.ensureTargetDirectory)
@@ -98,6 +98,11 @@ func (t *toolkitInstaller) Install(destDir string) error {
 	return errs
 }
 
+func (t *ToolkitInstaller) ConfigFilePath(destDir string) string {
+	toolkitConfigDir := filepath.Join(destDir, ".config", "nvidia-container-runtime")
+	return filepath.Join(toolkitConfigDir, "config.toml")
+}
+
 type symlink struct {
 	linkname string
 	target   string

cmd/nvidia-ctk-installer/toolkit/installer/installer_test.go

@@ -112,7 +112,7 @@ func TestToolkitInstaller(t *testing.T) {
 			return nil
 		},
 	}
-	i := toolkitInstaller{
+	i := ToolkitInstaller{
 		logger:                logger,
 		artifactRoot:          r,
 		ensureTargetDirectory: createDirectory,
@@ -172,8 +172,8 @@ if [ "${?}" != "0" ]; then
 	echo "nvidia driver modules are not yet loaded, invoking runc directly"
 	exec runc "$@"
 fi
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
 PATH=/foo/bar/baz:$PATH \
-XDG_CONFIG_HOME=/foo/bar/baz/.config \
 	/foo/bar/baz/nvidia-container-runtime.real \
 		"$@"
 `,
@@ -187,8 +187,8 @@ if [ "${?}" != "0" ]; then
 	echo "nvidia driver modules are not yet loaded, invoking runc directly"
 	exec runc "$@"
 fi
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
 PATH=/foo/bar/baz:$PATH \
-XDG_CONFIG_HOME=/foo/bar/baz/.config \
 	/foo/bar/baz/nvidia-container-runtime.cdi.real \
 		"$@"
 `,
@@ -202,8 +202,8 @@ if [ "${?}" != "0" ]; then
 	echo "nvidia driver modules are not yet loaded, invoking runc directly"
 	exec runc "$@"
 fi
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
 PATH=/foo/bar/baz:$PATH \
-XDG_CONFIG_HOME=/foo/bar/baz/.config \
 	/foo/bar/baz/nvidia-container-runtime.legacy.real \
 		"$@"
 `,
@@ -240,9 +240,9 @@ PATH=/foo/bar/baz:$PATH \
 				path: "/foo/bar/baz/nvidia-container-runtime-hook",
 				mode: 0777,
 				wrapper: `#! /bin/sh
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
 PATH=/foo/bar/baz:$PATH \
 	/foo/bar/baz/nvidia-container-runtime-hook.real \
-		-config /foo/bar/baz/.config/nvidia-container-runtime/config.toml \
 		"$@"
 `,
 			},

cmd/nvidia-ctk-installer/toolkit/installer/libraries.go

@@ -28,7 +28,7 @@ import (
 // A predefined set of library candidates are considered, with the first one
 // resulting in success being installed to the toolkit folder. The install process
 // resolves the symlink for the library and copies the versioned library itself.
-func (t *toolkitInstaller) collectLibraries() ([]Installer, error) {
+func (t *ToolkitInstaller) collectLibraries() ([]Installer, error) {
 	requiredLibraries := []string{
 		"libnvidia-container.so.1",
 		"libnvidia-container-go.so.1",

cmd/nvidia-ctk-installer/toolkit/installer/options.go

@@ -19,29 +19,29 @@ package installer
 
 import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 
-type Option func(*toolkitInstaller)
+type Option func(*ToolkitInstaller)
 
 func WithLogger(logger logger.Interface) Option {
-	return func(ti *toolkitInstaller) {
+	return func(ti *ToolkitInstaller) {
 		ti.logger = logger
 	}
 }
 
 func WithArtifactRoot(artifactRoot *artifactRoot) Option {
-	return func(ti *toolkitInstaller) {
+	return func(ti *ToolkitInstaller) {
 		ti.artifactRoot = artifactRoot
 	}
 }
 
 func WithIgnoreErrors(ignoreErrors bool) Option {
-	return func(ti *toolkitInstaller) {
+	return func(ti *ToolkitInstaller) {
 		ti.ignoreErrors = ignoreErrors
 	}
 }
 
 // WithSourceRoot sets the root directory for locating artifacts to be installed.
 func WithSourceRoot(sourceRoot string) Option {
-	return func(ti *toolkitInstaller) {
+	return func(ti *ToolkitInstaller) {
 		ti.sourceRoot = sourceRoot
 	}
 }

cmd/nvidia-ctk-installer/toolkit/toolkit.go

@@ -37,8 +37,6 @@ import (
 const (
 	// DefaultNvidiaDriverRoot specifies the default NVIDIA driver run directory
 	DefaultNvidiaDriverRoot = "/run/nvidia/driver"
-
-	configFilename = "config.toml"
 )
 
 type cdiOptions struct {
@@ -316,7 +314,7 @@ func (t *Installer) Install(cli *cli.Context, opts *Options) error {
 		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not install toolkit components: %w", err))
 	}
 
-	err = t.installToolkitConfig(cli, opts)
+	err = t.installToolkitConfig(cli, opts, toolkit.ConfigFilePath(t.toolkitRoot))
 	if err != nil && !opts.ignoreErrors {
 		return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err)
 	} else if err != nil {
@@ -343,13 +341,11 @@ func (t *Installer) Install(cli *cli.Context, opts *Options) error {
 
 // installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring
 // that the settings are updated to match the desired install and nvidia driver directories.
-func (t *Installer) installToolkitConfig(c *cli.Context, opts *Options) error {
+func (t *Installer) installToolkitConfig(c *cli.Context, opts *Options, toolkitConfigPath string) error {
-	toolkitConfigDir := filepath.Join(t.toolkitRoot, ".config", "nvidia-container-runtime")
-	toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename)
 
 	t.logger.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
 
-	err := t.createDirectories(toolkitConfigDir)
+	err := t.createDirectories(filepath.Dir(toolkitConfigPath))
 	if err != nil && !opts.ignoreErrors {
 		return fmt.Errorf("could not create required directories: %v", err)
 	} else if err != nil {

internal/config/config.go

@@ -31,8 +31,10 @@ import (
 )
 
 const (
-	configOverride = "XDG_CONFIG_HOME"
+	FilePathOverrideEnvVar = "NVIDIA_CTK_CONFIG_FILE_PATH"
-	configFilePath = "nvidia-container-runtime/config.toml"
+	RelativeFilePath       = "nvidia-container-runtime/config.toml"
+
+	configRootOverride = "XDG_CONFIG_HOME"
 
 	nvidiaCTKExecutable          = "nvidia-ctk"
 	nvidiaCTKDefaultFilePath     = "/usr/bin/nvidia-ctk"
@@ -74,11 +76,15 @@ type Config struct {
 
 // GetConfigFilePath returns the path to the config file for the configured system
 func GetConfigFilePath() string {
-	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+	if configFilePathOverride := os.Getenv(FilePathOverrideEnvVar); configFilePathOverride != "" {
-		return filepath.Join(XDGConfigDir, configFilePath)
+		return configFilePathOverride
+	}
+	configRoot := "/etc"
+	if XDGConfigDir := os.Getenv(configRootOverride); len(XDGConfigDir) != 0 {
+		configRoot = XDGConfigDir
 	}
 
-	return filepath.Join("/etc", configFilePath)
+	return filepath.Join(configRoot, RelativeFilePath)
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file

internal/config/config_test.go

@@ -27,9 +27,26 @@ import (
 
 func TestGetConfigWithCustomConfig(t *testing.T) {
 	testDir := t.TempDir()
-	t.Setenv(configOverride, testDir)
+	t.Setenv(configRootOverride, testDir)
 
-	filename := filepath.Join(testDir, configFilePath)
+	filename := filepath.Join(testDir, RelativeFilePath)
+
+	// By default debug is disabled
+	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, os.WriteFile(filename, contents, 0600))
+
+	cfg, err := GetConfig()
+	require.NoError(t, err)
+	require.Equal(t, "/nvidia-container-toolkit.log", cfg.NVIDIAContainerRuntimeConfig.DebugFilePath)
+}
+
+func TestGetConfigWithConfigFilePathOverride(t *testing.T) {
+	testDir := t.TempDir()
+	filename := filepath.Join(testDir, RelativeFilePath)
+
+	t.Setenv(FilePathOverrideEnvVar, filename)
 
 	// By default debug is disabled
 	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")