Merge branch 'CNT-4016/add-nvidia-ctk-config-default' into 'main'

Add nvidia-ctk config generate-default command to generate default config file contents

See merge request nvidia/container-toolkit/container-toolkit!338
This commit is contained in:
Evan Lezar 2023-05-08 10:40:42 +00:00
commit 6b367445a3
12 changed files with 343 additions and 191 deletions

View File

@ -496,7 +496,7 @@ func TestGetNvidiaConfig(t *testing.T) {
getConfig := func() {
hookConfig := tc.hookConfig
if hookConfig == nil {
defaultConfig := getDefaultHookConfig()
defaultConfig, _ := getDefaultHookConfig()
hookConfig = &defaultConfig
}
config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
@ -708,7 +708,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
env := map[string]string{
envNVVisibleDevices: tc.envvarDevices,
}
hookConfig := getDefaultHookConfig()
hookConfig, _ := getDefaultHookConfig()
hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)

View File

@ -1,6 +1,7 @@
package main
import (
"fmt"
"log"
"os"
"path"
@ -48,8 +49,18 @@ type HookConfig struct {
NVIDIAContainerRuntimeHook config.RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
}
func getDefaultHookConfig() HookConfig {
return HookConfig{
func getDefaultHookConfig() (HookConfig, error) {
rtConfig, err := config.GetDefaultRuntimeConfig()
if err != nil {
return HookConfig{}, err
}
rtHookConfig, err := config.GetDefaultRuntimeHookConfig()
if err != nil {
return HookConfig{}, err
}
c := HookConfig{
DisableRequire: false,
SwarmResource: nil,
AcceptEnvvarUnprivileged: true,
@ -67,28 +78,37 @@ func getDefaultHookConfig() HookConfig {
User: nil,
Ldconfig: nil,
},
NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
NVIDIAContainerRuntimeHook: *config.GetDefaultRuntimeHookConfig(),
NVIDIAContainerRuntime: *rtConfig,
NVIDIAContainerRuntimeHook: *rtHookConfig,
}
return c, nil
}
func getHookConfig() (config HookConfig) {
func getHookConfig() (*HookConfig, error) {
var err error
var config HookConfig
if len(*configflag) > 0 {
config = getDefaultHookConfig()
config, err = getDefaultHookConfig()
if err != nil {
return nil, fmt.Errorf("couldn't get default configuration: %v", err)
}
_, err = toml.DecodeFile(*configflag, &config)
if err != nil {
log.Panicln("couldn't open configuration file:", err)
return nil, fmt.Errorf("couldn't open configuration file: %v", err)
}
} else {
for _, p := range defaultPaths {
config = getDefaultHookConfig()
config, err = getDefaultHookConfig()
if err != nil {
return nil, fmt.Errorf("couldn't get default configuration: %v", err)
}
_, err = toml.DecodeFile(p, &config)
if err == nil {
break
} else if !os.IsNotExist(err) {
log.Panicln("couldn't open default configuration file:", err)
return nil, fmt.Errorf("couldn't open default configuration file: %v", err)
}
}
}
@ -102,7 +122,7 @@ func getHookConfig() (config HookConfig) {
log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
}
return config
return &config, nil
}
// getConfigOption returns the toml config option associated with the

View File

@ -89,7 +89,8 @@ func TestGetHookConfig(t *testing.T) {
var config HookConfig
getHookConfig := func() {
config = getHookConfig()
c, _ := getHookConfig()
config = *c
}
if tc.expectedPanic {

View File

@ -71,14 +71,17 @@ func doPrestart() {
defer exit()
log.SetFlags(0)
hook := getHookConfig()
hook, err := getHookConfig()
if err != nil || hook == nil {
log.Panicln("error getting hook config:", err)
}
cli := hook.NvidiaContainerCLI
if !hook.NVIDIAContainerRuntimeHook.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
}
container := getContainerConfig(hook)
container := getContainerConfig(*hook)
nvidia := container.Nvidia
if nvidia == nil {
// Not a GPU container, nothing to do.

View File

@ -0,0 +1,50 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package config
import (
createdefault "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/create-default"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
type command struct {
logger *logrus.Logger
}
// NewCommand constructs an config command with the specified logger
func NewCommand(logger *logrus.Logger) *cli.Command {
c := command{
logger: logger,
}
return c.build()
}
// build
func (m command) build() *cli.Command {
// Create the 'config' command
c := cli.Command{
Name: "config",
Usage: "Interact with the NVIDIA Container Toolkit configuration",
}
c.Subcommands = []*cli.Command{
createdefault.NewCommand(m.logger),
}
return &c
}

View File

@ -0,0 +1,102 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package defaultsubcommand
import (
"fmt"
"io"
"os"
nvctkConfig "github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
type command struct {
logger *logrus.Logger
}
// options stores the subcommand options
type options struct {
output string
}
// NewCommand constructs a default command with the specified logger
func NewCommand(logger *logrus.Logger) *cli.Command {
c := command{
logger: logger,
}
return c.build()
}
// build creates the CLI command
func (m command) build() *cli.Command {
opts := options{}
// Create the 'default' command
c := cli.Command{
Name: "generate-default",
Aliases: []string{"default"},
Usage: "Generate the default NVIDIA Container Toolkit configuration file",
Before: func(c *cli.Context) error {
return m.validateFlags(c, &opts)
},
Action: func(c *cli.Context) error {
return m.run(c, &opts)
},
}
c.Flags = []cli.Flag{
&cli.StringFlag{
Name: "output",
Usage: "Specify the file to output the generated configuration for to. If this is '' the configuration is ouput to STDOUT.",
Destination: &opts.output,
},
}
return &c
}
func (m command) validateFlags(c *cli.Context, opts *options) error {
return nil
}
func (m command) run(c *cli.Context, opts *options) error {
defaultConfig, err := nvctkConfig.GetDefaultConfigToml()
if err != nil {
return fmt.Errorf("unable to get default config: %v", err)
}
var output io.Writer
if opts.output == "" {
output = os.Stdout
} else {
outputFile, err := os.Create(opts.output)
if err != nil {
return fmt.Errorf("unable to create output file: %v", err)
}
defer outputFile.Close()
output = outputFile
}
_, err = defaultConfig.WriteTo(output)
if err != nil {
return fmt.Errorf("unable to write to output: %v", err)
}
return nil
}

View File

@ -20,6 +20,7 @@ import (
"os"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
@ -32,16 +33,16 @@ import (
var logger = log.New()
// config defines the options that can be set for the CLI through config files,
// options defines the options that can be set for the CLI through config files,
// environment variables, or command line flags
type config struct {
type options struct {
// Debug indicates whether the CLI is started in "debug" mode
Debug bool
}
func main() {
// Create a config struct to hold the parsed environment variables or command line flags
config := config{}
// Create a options struct to hold the parsed environment variables or command line flags
opts := options{}
// Create the top-level CLI
c := cli.NewApp()
@ -57,7 +58,7 @@ func main() {
Name: "debug",
Aliases: []string{"d"},
Usage: "Enable debug-level logging",
Destination: &config.Debug,
Destination: &opts.Debug,
EnvVars: []string{"NVIDIA_CTK_DEBUG"},
},
}
@ -65,7 +66,7 @@ func main() {
// Set log-level for all subcommands
c.Before = func(c *cli.Context) error {
logLevel := log.InfoLevel
if config.Debug {
if opts.Debug {
logLevel = log.DebugLevel
}
logger.SetLevel(logLevel)
@ -79,6 +80,7 @@ func main() {
infoCLI.NewCommand(logger),
cdi.NewCommand(logger),
system.NewCommand(logger),
config.NewCommand(logger),
}
// Run the CLI

View File

@ -16,33 +16,7 @@
package config
import (
"github.com/pelletier/go-toml"
)
// ContainerCLIConfig stores the options for the nvidia-container-cli
type ContainerCLIConfig struct {
Root string
}
// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
cfg := getDefaultContainerCLIConfig()
if toml == nil {
return cfg
}
cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
return cfg
}
// getDefaultContainerCLIConfig defines the default values for the config
func getDefaultContainerCLIConfig() *ContainerCLIConfig {
c := ContainerCLIConfig{
Root: "",
}
return &c
Root string `toml:"root"`
}

View File

@ -17,11 +17,14 @@
package config
import (
"bufio"
"fmt"
"io"
"os"
"path"
"strings"
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
"github.com/pelletier/go-toml"
)
@ -64,7 +67,7 @@ func GetConfig() (*Config, error) {
tomlFile, err := os.Open(configFilePath)
if err != nil {
return getDefaultConfig(), nil
return getDefaultConfig()
}
defer tomlFile.Close()
@ -88,39 +91,144 @@ func loadConfigFrom(reader io.Reader) (*Config, error) {
// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
func getConfigFrom(toml *toml.Tree) (*Config, error) {
cfg := getDefaultConfig()
if toml == nil {
return cfg, nil
}
cfg.AcceptEnvvarUnprivileged = toml.GetDefault("accept-nvidia-visible-devices-envvar-when-unprivileged", cfg.AcceptEnvvarUnprivileged).(bool)
cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
runtimeConfig, err := getRuntimeConfigFrom(toml)
cfg, err := getDefaultConfig()
if err != nil {
return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
return nil, err
}
cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
runtimeHookConfig, err := getRuntimeHookConfigFrom(toml)
if err != nil {
return nil, fmt.Errorf("failed to load nvidia-container-runtime-hook config: %v", err)
if err := toml.Unmarshal(cfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal config: %v", err)
}
cfg.NVIDIAContainerRuntimeHookConfig = *runtimeHookConfig
return cfg, nil
}
// getDefaultConfig defines the default values for the config
func getDefaultConfig() *Config {
c := Config{
AcceptEnvvarUnprivileged: true,
NVIDIAContainerCLIConfig: *getDefaultContainerCLIConfig(),
NVIDIACTKConfig: *getDefaultCTKConfig(),
NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
func getDefaultConfig() (*Config, error) {
tomlConfig, err := GetDefaultConfigToml()
if err != nil {
return nil, err
}
return &c
// tomlConfig above includes information about the default values and comments.
// we need to marshal it back to a string and then unmarshal it to strip the comments.
contents, err := tomlConfig.ToTomlString()
if err != nil {
return nil, err
}
reloaded, err := toml.Load(contents)
if err != nil {
return nil, err
}
d := Config{}
if err := reloaded.Unmarshal(&d); err != nil {
return nil, fmt.Errorf("failed to unmarshal config: %v", err)
}
// The default value for the accept-nvidia-visible-devices-envvar-when-unprivileged is non-standard.
// As such we explicitly handle it being set here.
if reloaded.Get("accept-nvidia-visible-devices-envvar-when-unprivileged") == nil {
d.AcceptEnvvarUnprivileged = true
}
// The default value for the nvidia-container-runtime.debug is non-standard.
// As such we explicitly handle it being set here.
if reloaded.Get("nvidia-container-runtime.debug") == nil {
d.NVIDIAContainerRuntimeConfig.DebugFilePath = "/dev/null"
}
return &d, nil
}
// GetDefaultConfigToml returns the default config as a toml Tree.
func GetDefaultConfigToml() (*toml.Tree, error) {
tree, err := toml.TreeFromMap(nil)
if err != nil {
return nil, err
}
tree.Set("disable-require", false)
tree.SetWithComment("swarm-resource", "", true, "DOCKER_RESOURCE_GPU")
tree.SetWithComment("accept-nvidia-visible-devices-envvar-when-unprivileged", "", true, true)
tree.SetWithComment("accept-nvidia-visible-devices-as-volume-mounts", "", true, false)
// nvidia-container-cli
tree.SetWithComment("nvidia-container-cli.root", "", true, "/run/nvidia/driver")
tree.SetWithComment("nvidia-container-cli.path", "", true, "/usr/bin/nvidia-container-cli")
tree.Set("nvidia-container-cli.environment", []string{})
tree.SetWithComment("nvidia-container-cli.debug", "", true, "/var/log/nvidia-container-toolkit.log")
tree.SetWithComment("nvidia-container-cli.ldcache", "", true, "/etc/ld.so.cache")
tree.Set("nvidia-container-cli.load-kmods", true)
tree.SetWithComment("nvidia-container-cli.no-cgroups", "", true, false)
tree.SetWithComment("nvidia-container-cli.user", "", getCommentedUserGroup(), getUserGroup())
tree.Set("nvidia-container-cli.ldconfig", getLdConfigPath())
// nvidia-container-runtime
tree.SetWithComment("nvidia-container-runtime.debug", "", true, "/var/log/nvidia-container-runtime.log")
tree.Set("nvidia-container-runtime.log-level", "info")
commentLines := []string{
"Specify the runtimes to consider. This list is processed in order and the PATH",
"searched for matching executables unless the entry is an absolute path.",
}
tree.SetWithComment("nvidia-container-runtime.runtimes", strings.Join(commentLines, "\n "), false, []string{"docker-runc", "runc"})
tree.Set("nvidia-container-runtime.mode", "auto")
tree.Set("nvidia-container-runtime.modes.csv.mount-spec-path", "/etc/nvidia-container-runtime/host-files-for-container.d")
tree.Set("nvidia-container-runtime.modes.cdi.default-kind", "nvidia.com/gpu")
tree.Set("nvidia-container-runtime.modes.cdi.annotation-prefixes", []string{cdi.AnnotationPrefix})
// nvidia-ctk
tree.Set("nvidia-ctk.path", "nvidia-ctk")
return tree, nil
}
func getLdConfigPath() string {
if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
return "@/sbin/ldconfig.real"
}
return "@/sbin/ldconfig"
}
// getUserGroup returns the user and group to use for the nvidia-container-cli and whether the config option should be commented.
func getUserGroup() string {
return "root:video"
}
// getCommentedUserGroup returns whether the nvidia-container-cli user and group config option should be commented.
func getCommentedUserGroup() bool {
uncommentIf := map[string]bool{
"suse": true,
"opensuse": true,
}
idsLike := getDistIDLike()
for _, id := range idsLike {
if uncommentIf[id] {
return false
}
}
return true
}
// getDistIDLike returns the ID_LIKE field from /etc/os-release.
func getDistIDLike() []string {
releaseFile, err := os.Open("/etc/os-release")
if err != nil {
return nil
}
defer releaseFile.Close()
scanner := bufio.NewScanner(releaseFile)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "ID_LIKE=") {
value := strings.Trim(strings.TrimPrefix(line, "ID_LIKE="), "\"")
return strings.Split(value, " ")
}
}
return nil
}

View File

@ -16,47 +16,18 @@
package config
import (
"fmt"
"github.com/pelletier/go-toml"
)
// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
type RuntimeHookConfig struct {
// SkipModeDetection disables the mode check for the runtime hook.
SkipModeDetection bool `toml:"skip-mode-detection"`
}
// dummyHookConfig allows us to unmarshal only a RuntimeHookConfig from a *toml.Tree
type dummyHookConfig struct {
RuntimeHook RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
}
// getRuntimeHookConfigFrom reads the nvidia container runtime config from the specified toml Tree.
func getRuntimeHookConfigFrom(toml *toml.Tree) (*RuntimeHookConfig, error) {
cfg := GetDefaultRuntimeHookConfig()
if toml == nil {
return cfg, nil
}
d := dummyHookConfig{
RuntimeHook: *cfg,
}
if err := toml.Unmarshal(&d); err != nil {
return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
}
return &d.RuntimeHook, nil
}
// GetDefaultRuntimeHookConfig defines the default values for the config
func GetDefaultRuntimeHookConfig() *RuntimeHookConfig {
c := RuntimeHookConfig{
SkipModeDetection: false,
func GetDefaultRuntimeHookConfig() (*RuntimeHookConfig, error) {
cfg, err := getDefaultConfig()
if err != nil {
return nil, err
}
return &c
return &cfg.NVIDIAContainerRuntimeHookConfig, nil
}

View File

@ -16,21 +16,6 @@
package config
import (
"fmt"
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
"github.com/pelletier/go-toml"
"github.com/sirupsen/logrus"
)
const (
dockerRuncExecutableName = "docker-runc"
runcExecutableName = "runc"
auto = "auto"
)
// RuntimeConfig stores the config options for the NVIDIA Container Runtime
type RuntimeConfig struct {
DebugFilePath string `toml:"debug"`
@ -61,52 +46,12 @@ type csvModeConfig struct {
MountSpecPath string `toml:"mount-spec-path"`
}
// dummy allows us to unmarshal only a RuntimeConfig from a *toml.Tree
type dummy struct {
Runtime RuntimeConfig `toml:"nvidia-container-runtime"`
}
// getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
func getRuntimeConfigFrom(toml *toml.Tree) (*RuntimeConfig, error) {
cfg := GetDefaultRuntimeConfig()
if toml == nil {
return cfg, nil
}
d := dummy{
Runtime: *cfg,
}
if err := toml.Unmarshal(&d); err != nil {
return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
}
return &d.Runtime, nil
}
// GetDefaultRuntimeConfig defines the default values for the config
func GetDefaultRuntimeConfig() *RuntimeConfig {
c := RuntimeConfig{
DebugFilePath: "/dev/null",
LogLevel: logrus.InfoLevel.String(),
Runtimes: []string{
dockerRuncExecutableName,
runcExecutableName,
},
Mode: auto,
Modes: modesConfig{
CSV: csvModeConfig{
MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
},
CDI: cdiModeConfig{
DefaultKind: "nvidia.com/gpu",
AnnotationPrefixes: []string{
cdi.AnnotationPrefix,
},
},
},
func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
cfg, err := getDefaultConfig()
if err != nil {
return nil, err
}
return &c
return &cfg.NVIDIAContainerRuntimeConfig, nil
}

View File

@ -16,31 +16,7 @@
package config
import "github.com/pelletier/go-toml"
// CTKConfig stores the config options for the NVIDIA Container Toolkit CLI (nvidia-ctk)
type CTKConfig struct {
Path string `toml:"path"`
}
// getCTKConfigFrom reads the nvidia container runtime config from the specified toml Tree.
func getCTKConfigFrom(toml *toml.Tree) *CTKConfig {
cfg := getDefaultCTKConfig()
if toml == nil {
return cfg
}
cfg.Path = toml.GetDefault("nvidia-ctk.path", cfg.Path).(string)
return cfg
}
// getDefaultCTKConfig defines the default values for the config
func getDefaultCTKConfig() *CTKConfig {
c := CTKConfig{
Path: "nvidia-ctk",
}
return &c
}