UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-05-29 09:43:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'CNT-3894/configure-mode-specific-runtimes' into 'main'

Browse Source 
Configure .cdi and .legacy executables in Toolkit Container

See merge request nvidia/container-toolkit/container-toolkit!308
This commit is contained in:
Evan Lezar 2023-03-08 05:12:50 +00:00
commit 1c696b1e39
30 changed files with 1768 additions and 1195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
cmd/nvidia-ctk/runtime/configure/configure.go
View File

@ -22,8 +22,8 @@ import (
"os"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/crio"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
"github.com/pelletier/go-toml"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
@ -127,13 +127,14 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
configFilePath = defaultDockerConfigFilePath
}
cfg, err := docker.LoadConfig(configFilePath)
cfg, err := docker.New(
docker.WithPath(configFilePath),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
err = docker.UpdateConfig(
cfg,
err = cfg.AddRuntime(
config.nvidiaOptions.RuntimeName,
config.nvidiaOptions.RuntimePath,
config.nvidiaOptions.SetAsDefault,
@ -150,12 +151,16 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
return nil
}
err = docker.FlushConfig(cfg, configFilePath)
n, err := cfg.Save(configFilePath)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
m.logger.Infof("Removed empty config from %v", configFilePath)
} else {
m.logger.Infof("Wrote updated config to %v", configFilePath)
}
m.logger.Infof("It is recommended that the docker daemon be restarted.")
return nil
@ -168,13 +173,14 @@ func (m command) configureCrio(c *cli.Context, config *config) error {
configFilePath = defaultCrioConfigFilePath
}
cfg, err := crio.LoadConfig(configFilePath)
cfg, err := crio.New(
crio.WithPath(configFilePath),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
err = crio.UpdateConfig(
cfg,
err = cfg.AddRuntime(
config.nvidiaOptions.RuntimeName,
config.nvidiaOptions.RuntimePath,
config.nvidiaOptions.SetAsDefault,
@ -191,12 +197,16 @@ func (m command) configureCrio(c *cli.Context, config *config) error {
os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
return nil
}
err = crio.FlushConfig(configFilePath, cfg)
n, err := cfg.Save(configFilePath)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
m.logger.Infof("Removed empty config from %v", configFilePath)
} else {
m.logger.Infof("Wrote updated config to %v", configFilePath)
}
m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
return nil

125
internal/config/crio/crio.go
View File

@ -1,125 +0,0 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package crio
import (
"fmt"
"os"
"github.com/pelletier/go-toml"
log "github.com/sirupsen/logrus"
)
// LoadConfig loads the cri-o config from disk
func LoadConfig(config string) (*toml.Tree, error) {
log.Infof("Loading config: %v", config)
info, err := os.Stat(config)
if os.IsExist(err) && info.IsDir() {
return nil, fmt.Errorf("config file is a directory")
}
configFile := config
if os.IsNotExist(err) {
configFile = "/dev/null"
log.Infof("Config file does not exist, creating new one")
}
cfg, err := toml.LoadFile(configFile)
if err != nil {
return nil, err
}
log.Infof("Successfully loaded config")
return cfg, nil
}
// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
func UpdateConfig(config *toml.Tree, runtimeClass string, runtimePath string, setAsDefault bool) error {
switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
case *toml.Tree:
runc, _ = toml.Load(runc.String())
config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass}, runc)
}
config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_path"}, runtimePath)
config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_type"}, "oci")
if setAsDefault {
config.SetPath([]string{"crio", "runtime", "default_runtime"}, runtimeClass)
}
return nil
}
// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
func RevertConfig(config *toml.Tree, runtimeClass string) error {
if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
if runtimeClass == runtime {
config.DeletePath([]string{"crio", "runtime", "default_runtime"})
}
}
runtimeClassPath := []string{"crio", "runtime", "runtimes", runtimeClass}
config.DeletePath(runtimeClassPath)
for i := 0; i < len(runtimeClassPath); i++ {
remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
if len(entry.Keys()) != 0 {
break
}
config.DeletePath(remainingPath)
}
}
return nil
}
// FlushConfig flushes the updated/reverted config out to disk
func FlushConfig(config string, cfg *toml.Tree) error {
log.Infof("Flushing config")
output, err := cfg.ToTomlString()
if err != nil {
return fmt.Errorf("unable to convert to TOML: %v", err)
}
switch len(output) {
case 0:
err := os.Remove(config)
if err != nil {
return fmt.Errorf("unable to remove empty file: %v", err)
}
log.Infof("Config empty, removing file")
default:
f, err := os.Create(config)
if err != nil {
return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
}
defer f.Close()
_, err = f.WriteString(output)
if err != nil {
return fmt.Errorf("unable to write output: %v", err)
}
}
log.Infof("Successfully flushed config")
return nil
}

117
internal/config/docker/docker.go
View File

@ -1,117 +0,0 @@
/**
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package docker
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
log "github.com/sirupsen/logrus"
)
// LoadConfig loads the docker config from disk
func LoadConfig(configFilePath string) (map[string]interface{}, error) {
log.Infof("Loading docker config from %v", configFilePath)
info, err := os.Stat(configFilePath)
if os.IsExist(err) && info.IsDir() {
return nil, fmt.Errorf("config file is a directory")
}
cfg := make(map[string]interface{})
if os.IsNotExist(err) {
log.Infof("Config file does not exist, creating new one")
return cfg, nil
}
readBytes, err := ioutil.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("unable to read config: %v", err)
}
reader := bytes.NewReader(readBytes)
if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
return nil, err
}
log.Infof("Successfully loaded config")
return cfg, nil
}
// UpdateConfig updates the docker config to include the nvidia runtimes
func UpdateConfig(config map[string]interface{}, runtimeName string, runtimePath string, setAsDefault bool) error {
// Read the existing runtimes
runtimes := make(map[string]interface{})
if _, exists := config["runtimes"]; exists {
runtimes = config["runtimes"].(map[string]interface{})
}
// Add / update the runtime definitions
runtimes[runtimeName] = map[string]interface{}{
"path": runtimePath,
"args": []string{},
}
// Update the runtimes definition
if len(runtimes) > 0 {
config["runtimes"] = runtimes
}
if setAsDefault {
config["default-runtime"] = runtimeName
}
return nil
}
// FlushConfig flushes the updated/reverted config out to disk
func FlushConfig(cfg map[string]interface{}, configFilePath string) error {
log.Infof("Flushing docker config to %v", configFilePath)
output, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return fmt.Errorf("unable to convert to JSON: %v", err)
}
switch len(output) {
case 0:
err := os.Remove(configFilePath)
if err != nil {
return fmt.Errorf("unable to remove empty file: %v", err)
}
log.Infof("Config empty, removing file")
default:
f, err := os.Create(configFilePath)
if err != nil {
return fmt.Errorf("unable to open %v for writing: %v", configFilePath, err)
}
defer f.Close()
_, err = f.WriteString(string(output))
if err != nil {
return fmt.Errorf("unable to write output: %v", err)
}
}
log.Infof("Successfully flushed config")
return nil
}

25
internal/config/engine/api.go Normal file
View File

@ -0,0 +1,25 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package engine
// Interface defines the API for a runtime config updater.
type Interface interface {
DefaultRuntime() string
AddRuntime(string, string, bool) error
RemoveRuntime(string) error
Save(string) (int64, error)
}

130
internal/config/engine/containerd/config_v1.go Normal file
View File

@ -0,0 +1,130 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package containerd
import (
"fmt"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/pelletier/go-toml"
)
// ConfigV1 represents a version 1 containerd config
type ConfigV1 Config
var _ engine.Interface = (*ConfigV1)(nil)
// AddRuntime adds a runtime to the containerd config
func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
if c == nil || c.Tree == nil {
return fmt.Errorf("config is nil")
}
config := *c.Tree
config.Set("version", int64(1))
switch runc := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", "runc"}).(type) {
case *toml.Tree:
runc, _ = toml.Load(runc.String())
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, runc)
}
if config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name}) == nil {
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_root"}, "")
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_engine"}, "")
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
}
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}, path)
if setAsDefault && c.UseDefaultRuntimeName {
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
} else if setAsDefault {
// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
}
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
}
*c.Tree = config
return nil
}
// DefaultRuntime returns the default runtime for the cri-o config
func (c ConfigV1) DefaultRuntime() string {
if runtime, ok := c.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
return runtime
}
return ""
}
// RemoveRuntime removes a runtime from the docker config
func (c *ConfigV1) RemoveRuntime(name string) error {
if c == nil || c.Tree == nil {
return nil
}
config := *c.Tree
// If the specified runtime was set as the default runtime we need to remove the default runtime too.
runtimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}).(string)
if !ok || runtimePath == "" {
runtimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}).(string)
}
defaultRuntimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}).(string)
if !ok || defaultRuntimePath == "" {
defaultRuntimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}).(string)
}
if runtimePath != "" && defaultRuntimePath != "" && runtimePath == defaultRuntimePath {
config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime"})
}
config.DeletePath([]string{"plugins", "cri", "containerd", "runtimes", name})
if runtime, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
if runtime == name {
config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
}
}
runtimeConfigPath := []string{"plugins", "cri", "containerd", "runtimes", name}
for i := 0; i < len(runtimeConfigPath); i++ {
if runtimes, ok := config.GetPath(runtimeConfigPath[:len(runtimeConfigPath)-i]).(*toml.Tree); ok {
if len(runtimes.Keys()) == 0 {
config.DeletePath(runtimeConfigPath[:len(runtimeConfigPath)-i])
}
}
}
if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
config.Delete("version")
}
*c.Tree = config
return nil
}
// Save wrotes the config to a file
func (c ConfigV1) Save(path string) (int64, error) {
return (Config)(c).Save(path)
}

125
internal/config/engine/containerd/config_v2.go Normal file
View File

@ -0,0 +1,125 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package containerd
import (
"fmt"
"os"
"github.com/pelletier/go-toml"
)
// AddRuntime adds a runtime to the containerd config
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
if c == nil || c.Tree == nil {
return fmt.Errorf("config is nil")
}
config := *c.Tree
config.Set("version", int64(2))
switch runc := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc"}).(type) {
case *toml.Tree:
runc, _ = toml.Load(runc.String())
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, runc)
}
if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
}
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
if setAsDefault {
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
}
*c.Tree = config
return nil
}
// DefaultRuntime returns the default runtime for the cri-o config
func (c Config) DefaultRuntime() string {
if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
return runtime
}
return ""
}
// RemoveRuntime removes a runtime from the docker config
func (c *Config) RemoveRuntime(name string) error {
if c == nil || c.Tree == nil {
return nil
}
config := *c.Tree
config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
if runtime == name {
config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
}
}
runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
for i := 0; i < len(runtimePath); i++ {
if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
if len(runtimes.Keys()) == 0 {
config.DeletePath(runtimePath[:len(runtimePath)-i])
}
}
}
if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
config.Delete("version")
}
*c.Tree = config
return nil
}
// Save writes the config to the specified path
func (c Config) Save(path string) (int64, error) {
config := c.Tree
output, err := config.ToTomlString()
if err != nil {
return 0, fmt.Errorf("unable to convert to TOML: %v", err)
}
if len(output) == 0 {
err := os.Remove(path)
if err != nil {
return 0, fmt.Errorf("unable to remove empty file: %v", err)
}
return 0, nil
}
f, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
}
defer f.Close()
n, err := f.WriteString(output)
if err != nil {
return 0, fmt.Errorf("unable to write output: %v", err)
}
return int64(n), err
}

39
internal/config/engine/containerd/containerd.go Normal file
View File

@ -0,0 +1,39 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package containerd
import (
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/pelletier/go-toml"
)
// Config represents the containerd config
type Config struct {
*toml.Tree
RuntimeType string
UseDefaultRuntimeName bool
}
// New creates a containerd config with the specified options
func New(opts ...Option) (engine.Interface, error) {
b := &builder{}
for _, opt := range opts {
opt(b)
}
return b.build()
}

140
internal/config/engine/containerd/option.go Normal file
View File

@ -0,0 +1,140 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package containerd
import (
"fmt"
"os"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/pelletier/go-toml"
log "github.com/sirupsen/logrus"
)
const (
defaultRuntimeType = "io.containerd.runc.v2"
)
type builder struct {
path string
runtimeType string
useLegacyConfig bool
}
// Option defines a function that can be used to configure the config builder
type Option func(*builder)
// WithPath sets the path for the config builder
func WithPath(path string) Option {
return func(b *builder) {
b.path = path
}
}
// WithRuntimeType sets the runtime type for the config builder
func WithRuntimeType(runtimeType string) Option {
return func(b *builder) {
b.runtimeType = runtimeType
}
}
// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
func WithUseLegacyConfig(useLegacyConfig bool) Option {
return func(b *builder) {
b.useLegacyConfig = useLegacyConfig
}
}
func (b *builder) build() (engine.Interface, error) {
if b.path == "" {
return nil, fmt.Errorf("config path is empty")
}
if b.runtimeType == "" {
b.runtimeType = defaultRuntimeType
}
config, err := loadConfig(b.path)
if err != nil {
return nil, fmt.Errorf("failed to load config: %v", err)
}
config.RuntimeType = b.runtimeType
config.UseDefaultRuntimeName = !b.useLegacyConfig
version, err := config.parseVersion(b.useLegacyConfig)
if err != nil {
return nil, fmt.Errorf("failed to parse config version: %v", err)
}
switch version {
case 1:
return (*ConfigV1)(config), nil
case 2:
return config, nil
}
return nil, fmt.Errorf("unsupported config version: %v", version)
}
// loadConfig loads the containerd config from disk
func loadConfig(config string) (*Config, error) {
log.Infof("Loading config: %v", config)
info, err := os.Stat(config)
if os.IsExist(err) && info.IsDir() {
return nil, fmt.Errorf("config file is a directory")
}
configFile := config
if os.IsNotExist(err) {
configFile = "/dev/null"
log.Infof("Config file does not exist, creating new one")
}
tomlConfig, err := toml.LoadFile(configFile)
if err != nil {
return nil, err
}
log.Infof("Successfully loaded config")
cfg := Config{
Tree: tomlConfig,
}
return &cfg, nil
}
// parseVersion returns the version of the config
func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
defaultVersion := 2
if useLegacyConfig {
defaultVersion = 1
}
switch v := c.Get("version").(type) {
case nil:
switch len(c.Keys()) {
case 0: // No config exists, or the config file is empty, use version inferred from containerd
return defaultVersion, nil
default: // A config file exists, has content, and no version is set
return 1, nil
}
case int64:
return int(v), nil
default:
return -1, fmt.Errorf("unsupported type for version field: %v", v)
}
}

131
internal/config/engine/crio/crio.go Normal file
View File

@ -0,0 +1,131 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package crio
import (
"fmt"
"os"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/pelletier/go-toml"
)
// Config represents the cri-o config
type Config toml.Tree
// New creates a cri-o config with the specified options
func New(opts ...Option) (engine.Interface, error) {
b := &builder{}
for _, opt := range opts {
opt(b)
}
return b.build()
}
// AddRuntime adds a new runtime to the crio config
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
if c == nil {
return fmt.Errorf("config is nil")
}
config := (toml.Tree)(*c)
switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
case *toml.Tree:
runc, _ = toml.Load(runc.String())
config.SetPath([]string{"crio", "runtime", "runtimes", name}, runc)
}
config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_path"}, path)
config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_type"}, "oci")
if setAsDefault {
config.SetPath([]string{"crio", "runtime", "default_runtime"}, name)
}
*c = (Config)(config)
return nil
}
// DefaultRuntime returns the default runtime for the cri-o config
func (c Config) DefaultRuntime() string {
config := (toml.Tree)(c)
if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
return runtime
}
return ""
}
// RemoveRuntime removes a runtime from the cri-o config
func (c *Config) RemoveRuntime(name string) error {
if c == nil {
return nil
}
config := (toml.Tree)(*c)
if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
if runtime == name {
config.DeletePath([]string{"crio", "runtime", "default_runtime"})
}
}
runtimeClassPath := []string{"crio", "runtime", "runtimes", name}
config.DeletePath(runtimeClassPath)
for i := 0; i < len(runtimeClassPath); i++ {
remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
if len(entry.Keys()) != 0 {
break
}
config.DeletePath(remainingPath)
}
}
*c = (Config)(config)
return nil
}
// Save writes the config to the specified path
func (c Config) Save(path string) (int64, error) {
config := (toml.Tree)(c)
output, err := config.ToTomlString()
if err != nil {
return 0, fmt.Errorf("unable to convert to TOML: %v", err)
}
if len(output) == 0 {
err := os.Remove(path)
if err != nil {
return 0, fmt.Errorf("unable to remove empty file: %v", err)
}
return 0, nil
}
f, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
}
defer f.Close()
n, err := f.WriteString(output)
if err != nil {
return 0, fmt.Errorf("unable to write output: %v", err)
}
return int64(n), err
}

73
internal/config/engine/crio/option.go Normal file
View File

@ -0,0 +1,73 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package crio
import (
"fmt"
"os"
"github.com/pelletier/go-toml"
log "github.com/sirupsen/logrus"
)
type builder struct {
path string
}
// Option defines a function that can be used to configure the config builder
type Option func(*builder)
// WithPath sets the path for the config builder
func WithPath(path string) Option {
return func(b *builder) {
b.path = path
}
}
func (b *builder) build() (*Config, error) {
if b.path == "" {
empty := toml.Tree{}
return (*Config)(&empty), nil
}
return loadConfig(b.path)
}
// loadConfig loads the cri-o config from disk
func loadConfig(config string) (*Config, error) {
log.Infof("Loading config: %v", config)
info, err := os.Stat(config)
if os.IsExist(err) && info.IsDir() {
return nil, fmt.Errorf("config file is a directory")
}
configFile := config
if os.IsNotExist(err) {
configFile = "/dev/null"
log.Infof("Config file does not exist, creating new one")
}
cfg, err := toml.LoadFile(configFile)
if err != nil {
return nil, err
}
log.Infof("Successfully loaded config")
return (*Config)(cfg), nil
}

140
internal/config/engine/docker/docker.go Normal file
View File

@ -0,0 +1,140 @@
/**
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package docker
import (
"encoding/json"
"fmt"
"os"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
)
const (
defaultDockerRuntime = "runc"
)
// Config defines a docker config file.
// TODO: This should not be public, but we need to access it from the tests in tools/container/docker
type Config map[string]interface{}
// New creates a docker config with the specified options
func New(opts ...Option) (engine.Interface, error) {
b := &builder{}
for _, opt := range opts {
opt(b)
}
return b.build()
}
// AddRuntime adds a new runtime to the docker config
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
if c == nil {
return fmt.Errorf("config is nil")
}
config := *c
// Read the existing runtimes
runtimes := make(map[string]interface{})
if _, exists := config["runtimes"]; exists {
runtimes = config["runtimes"].(map[string]interface{})
}
// Add / update the runtime definitions
runtimes[name] = map[string]interface{}{
"path": path,
"args": []string{},
}
config["runtimes"] = runtimes
if setAsDefault {
config["default-runtime"] = name
}
*c = config
return nil
}
// DefaultRuntime returns the default runtime for the docker config
func (c Config) DefaultRuntime() string {
r, ok := c["default-runtime"].(string)
if !ok {
return ""
}
return r
}
// RemoveRuntime removes a runtime from the docker config
func (c *Config) RemoveRuntime(name string) error {
if c == nil {
return nil
}
config := *c
if _, exists := config["default-runtime"]; exists {
defaultRuntime := config["default-runtime"].(string)
if defaultRuntime == name {
config["default-runtime"] = defaultDockerRuntime
}
}
if _, exists := config["runtimes"]; exists {
runtimes := config["runtimes"].(map[string]interface{})
delete(runtimes, name)
if len(runtimes) == 0 {
delete(config, "runtimes")
}
}
*c = config
return nil
}
// Save writes the config to the specified path
func (c Config) Save(path string) (int64, error) {
output, err := json.MarshalIndent(c, "", " ")
if err != nil {
return 0, fmt.Errorf("unable to convert to JSON: %v", err)
}
if len(output) == 0 {
err := os.Remove(path)
if err != nil {
return 0, fmt.Errorf("unable to remove empty file: %v", err)
}
return 0, nil
}
f, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
}
defer f.Close()
n, err := f.WriteString(string(output))
if err != nil {
return 0, fmt.Errorf("unable to write output: %v", err)
}
return int64(n), nil
}

8
internal/config/docker/docker_test.go → internal/config/engine/docker/docker_test.go
View File

@ -26,7 +26,7 @@ import (
func TestUpdateConfigDefaultRuntime(t *testing.T) {
testCases := []struct {
config map[string]interface{}
config Config
runtimeName string
setAsDefault bool
expectedDefaultRuntimeName interface{}
@ -63,7 +63,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
if tc.config == nil {
tc.config = make(map[string]interface{})
}
err := UpdateConfig(tc.config, tc.runtimeName, "", tc.setAsDefault)
err := tc.config.AddRuntime(tc.runtimeName, "", tc.setAsDefault)
require.NoError(t, err)
defaultRuntimeName := tc.config["default-runtime"]
@ -74,7 +74,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
func TestUpdateConfigRuntimes(t *testing.T) {
testCases := []struct {
config map[string]interface{}
config Config
runtimes map[string]string
expectedConfig map[string]interface{}
}{
@ -198,7 +198,7 @@ func TestUpdateConfigRuntimes(t *testing.T) {
for i, tc := range testCases {
t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
for runtimeName, runtimePath := range tc.runtimes {
err := UpdateConfig(tc.config, runtimeName, runtimePath, false)
err := tc.config.AddRuntime(runtimeName, runtimePath, false)
require.NoError(t, err)
}

80
internal/config/engine/docker/option.go Normal file
View File

@ -0,0 +1,80 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package docker
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
log "github.com/sirupsen/logrus"
)
type builder struct {
path string
}
// Option defines a function that can be used to configure the config builder
type Option func(*builder)
// WithPath sets the path for the config builder
func WithPath(path string) Option {
return func(b *builder) {
b.path = path
}
}
func (b *builder) build() (*Config, error) {
if b.path == "" {
empty := make(Config)
return &empty, nil
}
return loadConfig(b.path)
}
// loadConfig loads the docker config from disk
func loadConfig(configFilePath string) (*Config, error) {
log.Infof("Loading docker config from %v", configFilePath)
info, err := os.Stat(configFilePath)
if os.IsExist(err) && info.IsDir() {
return nil, fmt.Errorf("config file is a directory")
}
cfg := make(Config)
if os.IsNotExist(err) {
log.Infof("Config file does not exist, creating new one")
return &cfg, nil
}
readBytes, err := ioutil.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("unable to read config: %v", err)
}
reader := bytes.NewReader(readBytes)
if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
return nil, err
}
log.Infof("Successfully loaded config")
return &cfg, nil
}

2
test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so
View File

@ -1 +1 @@
# This is a dummy lib file to test nvidia-runtime-experimental
# This is a dummy lib file to test nvidia-runtime.experimental

8
test/container/toolkit_test.sh
View File

@ -47,11 +47,11 @@ testing::toolkit::install() {
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental.real"
grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"

4
tools/container/README.md
View File

@ -15,7 +15,7 @@ docker setup \
/run/nvidia/toolkit
```
Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
@ -48,7 +48,7 @@ containerd setup \
/run/nvidia/toolkit
```
Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
Adding the `--set-as-default` flag as follows:
```bash

114
tools/container/containerd/config.go
View File

@ -1,114 +0,0 @@
/**
# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package main
import (
"github.com/pelletier/go-toml"
)
// UpdateReverter defines the interface for applying and reverting configurations
type UpdateReverter interface {
Update(o *options) error
Revert(o *options) error
}
type config struct {
*toml.Tree
version int64
cri string
}
// update adds the specified runtime class to the the containerd config.
// if set-as default is specified, the runtime class is also set as the
// default runtime.
func (config *config) update(runtimeClass string, runtimeType string, runtimeBinary string, setAsDefault bool) {
config.Set("version", config.version)
runcPath := config.runcPath()
runtimeClassPath := config.runtimeClassPath(runtimeClass)
switch runc := config.GetPath(runcPath).(type) {
case *toml.Tree:
runc, _ = toml.Load(runc.String())
config.SetPath(runtimeClassPath, runc)
}
config.initRuntime(runtimeClassPath, runtimeType, "BinaryName", runtimeBinary)
if config.version == 1 {
config.initRuntime(runtimeClassPath, runtimeType, "Runtime", runtimeBinary)
}
if setAsDefault {
defaultRuntimeNamePath := config.defaultRuntimeNamePath()
config.SetPath(defaultRuntimeNamePath, runtimeClass)
}
}
// revert removes the configuration applied in an update call.
func (config *config) revert(runtimeClass string) {
runtimeClassPath := config.runtimeClassPath(runtimeClass)
defaultRuntimeNamePath := config.defaultRuntimeNamePath()
config.DeletePath(runtimeClassPath)
if runtime, ok := config.GetPath(defaultRuntimeNamePath).(string); ok {
if runtimeClass == runtime {
config.DeletePath(defaultRuntimeNamePath)
}
}
for i := 0; i < len(runtimeClassPath); i++ {
if runtimes, ok := config.GetPath(runtimeClassPath[:len(runtimeClassPath)-i]).(*toml.Tree); ok {
if len(runtimes.Keys()) == 0 {
config.DeletePath(runtimeClassPath[:len(runtimeClassPath)-i])
}
}
}
if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
config.Delete("version")
}
}
// initRuntime creates a runtime config if it does not exist and ensures that the
// runtimes binary path is specified.
func (config *config) initRuntime(path []string, runtimeType string, binaryKey string, binary string) {
if config.GetPath(path) == nil {
config.SetPath(append(path, "runtime_type"), runtimeType)
config.SetPath(append(path, "runtime_root"), "")
config.SetPath(append(path, "runtime_engine"), "")
config.SetPath(append(path, "privileged_without_host_devices"), false)
}
binaryPath := append(path, "options", binaryKey)
config.SetPath(binaryPath, binary)
}
func (config config) runcPath() []string {
return config.runtimeClassPath("runc")
}
func (config config) runtimeClassPath(runtimeClass string) []string {
return append(config.containerdPath(), "runtimes", runtimeClass)
}
func (config config) defaultRuntimeNamePath() []string {
return append(config.containerdPath(), "default_runtime_name")
}
func (config config) containerdPath() []string {
return []string{"plugins", config.cri, "containerd"}
}

134
tools/container/containerd/config_v1.go
View File

@ -1,134 +0,0 @@
/**
# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package main
import (
"path"
"github.com/pelletier/go-toml"
log "github.com/sirupsen/logrus"
)
// configV1 represents a V1 containerd config
type configV1 struct {
config
}
func newConfigV1(cfg *toml.Tree) UpdateReverter {
c := configV1{
config: config{
Tree: cfg,
version: 1,
cri: "cri",
},
}
return &c
}
// Update performs an update specific to v1 of the containerd config
func (config *configV1) Update(o *options) error {
// For v1 config, the `default_runtime_name` setting is only supported
// for containerd version at least v1.3
supportsDefaultRuntimeName := !o.useLegacyConfig
defaultRuntime := o.getDefaultRuntime()
for runtimeClass, runtimeBinary := range o.getRuntimeBinaries() {
isDefaultRuntime := runtimeClass == defaultRuntime
config.update(runtimeClass, o.runtimeType, runtimeBinary, isDefaultRuntime && supportsDefaultRuntimeName)
if !isDefaultRuntime {
continue
}
if supportsDefaultRuntimeName {
defaultRuntimePath := append(config.containerdPath(), "default_runtime")
if config.GetPath(defaultRuntimePath) != nil {
log.Warnf("The setting of default_runtime (%v) in containerd is deprecated", defaultRuntimePath)
}
continue
}
log.Warnf("Setting default_runtime is deprecated")
defaultRuntimePath := append(config.containerdPath(), "default_runtime")
config.initRuntime(defaultRuntimePath, o.runtimeType, "Runtime", runtimeBinary)
config.initRuntime(defaultRuntimePath, o.runtimeType, "BinaryName", runtimeBinary)
}
return nil
}
// Revert performs a revert specific to v1 of the containerd config
func (config *configV1) Revert(o *options) error {
defaultRuntimePath := append(config.containerdPath(), "default_runtime")
defaultRuntimeOptionsPath := append(defaultRuntimePath, "options")
if runtime, ok := config.GetPath(append(defaultRuntimeOptionsPath, "Runtime")).(string); ok {
for _, runtimeBinary := range o.getRuntimeBinaries() {
if path.Base(runtimeBinary) == path.Base(runtime) {
config.DeletePath(append(defaultRuntimeOptionsPath, "Runtime"))
break
}
}
}
if runtime, ok := config.GetPath(append(defaultRuntimeOptionsPath, "BinaryName")).(string); ok {
for _, runtimeBinary := range o.getRuntimeBinaries() {
if path.Base(runtimeBinary) == path.Base(runtime) {
config.DeletePath(append(defaultRuntimeOptionsPath, "BinaryName"))
break
}
}
}
if options, ok := config.GetPath(defaultRuntimeOptionsPath).(*toml.Tree); ok {
if len(options.Keys()) == 0 {
config.DeletePath(defaultRuntimeOptionsPath)
}
}
if runtime, ok := config.GetPath(defaultRuntimePath).(*toml.Tree); ok {
fields := []string{"runtime_type", "runtime_root", "runtime_engine", "privileged_without_host_devices"}
if len(runtime.Keys()) <= len(fields) {
matches := []string{}
for _, f := range fields {
e := runtime.Get(f)
if e != nil {
matches = append(matches, f)
}
}
if len(matches) == len(runtime.Keys()) {
for _, m := range matches {
runtime.Delete(m)
}
}
}
}
for i := 0; i < len(defaultRuntimePath); i++ {
if runtimes, ok := config.GetPath(defaultRuntimePath[:len(defaultRuntimePath)-i]).(*toml.Tree); ok {
if len(runtimes.Keys()) == 0 {
config.DeletePath(defaultRuntimePath[:len(defaultRuntimePath)-i])
}
}
}
for runtimeClass := range nvidiaRuntimeBinaries {
config.revert(runtimeClass)
}
return nil
}

82
tools/container/containerd/config_v1_test.go
View File

@ -17,8 +17,10 @@
package main
import (
"fmt"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/containerd"
"github.com/pelletier/go-toml"
"github.com/stretchr/testify/require"
)
@ -58,7 +60,7 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
setAsDefault: true,
runtimeClass: "nvidia-experimental",
expectedDefaultRuntimeName: nil,
expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime-experimental",
expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime.experimental",
},
{
legacyConfig: false,
@ -89,6 +91,7 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
useLegacyConfig: tc.legacyConfig,
setAsDefault: tc.setAsDefault,
@ -100,16 +103,24 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
config, err := toml.TreeFromMap(map[string]interface{}{})
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateV1Config(config, o)
v1 := &containerd.ConfigV1{
Tree: config,
UseDefaultRuntimeName: !tc.legacyConfig,
RuntimeType: runtimeType,
}
err = UpdateConfig(v1, o)
require.NoError(t, err, "%d: %v", i, tc)
defaultRuntimeName := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
defaultRuntimeName := v1.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
defaultRuntime := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"})
defaultRuntime := v1.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"})
if tc.expectedDefaultRuntimeBinary == nil {
require.Nil(t, defaultRuntime, "%d: %v", i, tc)
} else {
require.NotNil(t, defaultRuntime)
expected, err := defaultRuntimeTomlConfigV1(tc.expectedDefaultRuntimeBinary.(string))
require.NoError(t, err, "%d: %v", i, tc)
@ -119,6 +130,7 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, tc)
}
})
}
}
@ -128,7 +140,9 @@ func TestUpdateV1Config(t *testing.T) {
expectedBinaries := []string{
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime-experimental",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -137,19 +151,20 @@ func TestUpdateV1Config(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"NAME", "nvidia-experimental"},
expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
runtimeClass: tc.runtimeClass,
runtimeType: runtimeType,
@ -159,7 +174,13 @@ func TestUpdateV1Config(t *testing.T) {
config, err := toml.TreeFromMap(map[string]interface{}{})
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateV1Config(config, o)
v1 := &containerd.ConfigV1{
Tree: config,
UseDefaultRuntimeName: true,
RuntimeType: runtimeType,
}
err = UpdateConfig(v1, o)
require.NoError(t, err, "%d: %v", i, tc)
version, ok := config.Get("version").(int64)
@ -184,6 +205,7 @@ func TestUpdateV1Config(t *testing.T) {
require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
}
})
}
}
@ -195,7 +217,9 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
expectedBinaries := []string{
runcBinary,
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime-experimental",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -204,19 +228,20 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
runtimeClass: tc.runtimeClass,
runtimeType: runtimeType,
@ -226,14 +251,20 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
config, err := toml.TreeFromMap(runcConfigMapV1("/runc-binary"))
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateV1Config(config, o)
v1 := &containerd.ConfigV1{
Tree: config,
UseDefaultRuntimeName: true,
RuntimeType: runtimeType,
}
err = UpdateConfig(v1, o)
require.NoError(t, err, "%d: %v", i, tc)
version, ok := config.Get("version").(int64)
version, ok := v1.Get("version").(int64)
require.True(t, ok)
require.EqualValues(t, expectedVersion, version)
runtimes, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes"}).(*toml.Tree)
runtimes, ok := v1.GetPath([]string{"plugins", "cri", "containerd", "runtimes"}).(*toml.Tree)
require.True(t, ok)
runtimeClasses := runtimes.Keys()
@ -251,6 +282,7 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
}
})
}
}
@ -274,7 +306,9 @@ func TestRevertV1Config(t *testing.T) {
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime-experimental"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
},
},
},
@ -289,7 +323,9 @@ func TestRevertV1Config(t *testing.T) {
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime-experimental"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
},
"default_runtime": defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
"default_runtime_name": "nvidia",
@ -301,6 +337,7 @@ func TestRevertV1Config(t *testing.T) {
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
runtimeClass: "nvidia",
}
@ -311,13 +348,20 @@ func TestRevertV1Config(t *testing.T) {
expected, err := toml.TreeFromMap(tc.expected)
require.NoError(t, err, "%d: %v", i, tc)
err = RevertV1Config(config, o)
v1 := &containerd.ConfigV1{
Tree: config,
UseDefaultRuntimeName: true,
RuntimeType: runtimeType,
}
err = RevertConfig(v1, o)
require.NoError(t, err, "%d: %v", i, tc)
configContents, _ := toml.Marshal(config)
expectedContents, _ := toml.Marshal(expected)
require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
})
}
}

58
tools/container/containerd/config_v2.go
View File

@ -1,58 +0,0 @@
/**
# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package main
import (
"github.com/pelletier/go-toml"
)
// configV2 represents a V2 containerd config
type configV2 struct {
config
}
func newConfigV2(cfg *toml.Tree) UpdateReverter {
c := configV2{
config: config{
Tree: cfg,
version: 2,
cri: "io.containerd.grpc.v1.cri",
},
}
return &c
}
// Update performs an update specific to v2 of the containerd config
func (config *configV2) Update(o *options) error {
defaultRuntime := o.getDefaultRuntime()
for runtimeClass, runtimeBinary := range o.getRuntimeBinaries() {
setAsDefault := defaultRuntime == runtimeClass
config.update(runtimeClass, o.runtimeType, runtimeBinary, setAsDefault)
}
return nil
}
// Revert performs a revert specific to v2 of the containerd config
func (config *configV2) Revert(o *options) error {
for runtimeClass := range o.getRuntimeBinaries() {
config.revert(runtimeClass)
}
return nil
}

64
tools/container/containerd/config_v2_test.go
View File

@ -17,8 +17,10 @@
package main
import (
"fmt"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/containerd"
"github.com/pelletier/go-toml"
"github.com/stretchr/testify/require"
)
@ -69,6 +71,7 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
setAsDefault: tc.setAsDefault,
runtimeClass: tc.runtimeClass,
@ -78,11 +81,17 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
config, err := toml.TreeFromMap(map[string]interface{}{})
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateV2Config(config, o)
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
err = UpdateConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
})
}
}
@ -92,7 +101,9 @@ func TestUpdateV2Config(t *testing.T) {
expectedBinaries := []string{
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime-experimental",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -101,15 +112,15 @@ func TestUpdateV2Config(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"NAME", "nvidia-experimental"},
expectedRuntimes: []string{"NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
@ -123,7 +134,12 @@ func TestUpdateV2Config(t *testing.T) {
config, err := toml.TreeFromMap(map[string]interface{}{})
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateV2Config(config, o)
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
err = UpdateConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
version, ok := config.Get("version").(int64)
@ -160,7 +176,9 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
expectedBinaries := []string{
runcBinary,
"/test/runtime/dir/nvidia-container-runtime",
"/test/runtime/dir/nvidia-container-runtime-experimental",
"/test/runtime/dir/nvidia-container-runtime.experimental",
"/test/runtime/dir/nvidia-container-runtime.cdi",
"/test/runtime/dir/nvidia-container-runtime.legacy",
}
testCases := []struct {
@ -169,19 +187,20 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
}{
{
runtimeClass: "nvidia",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "NAME",
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
{
runtimeClass: "nvidia-experimental",
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental", "nvidia-cdi", "nvidia-legacy"},
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
runtimeClass: tc.runtimeClass,
runtimeType: runtimeType,
@ -191,14 +210,19 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
config, err := toml.TreeFromMap(runcConfigMapV2("/runc-binary"))
require.NoError(t, err, "%d: %v", i, tc)
err = UpdateV2Config(config, o)
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
err = UpdateConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
version, ok := config.Get("version").(int64)
version, ok := v2.Get("version").(int64)
require.True(t, ok)
require.EqualValues(t, expectedVersion, version)
runtimes, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}).(*toml.Tree)
runtimes, ok := v2.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}).(*toml.Tree)
require.True(t, ok, "%d: %v", i, tc)
runtimeClasses := runtimes.Keys()
@ -216,6 +240,7 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
}
})
}
}
@ -239,7 +264,7 @@ func TestRevertV2Config(t *testing.T) {
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime-experimental"),
"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
},
},
},
@ -254,7 +279,7 @@ func TestRevertV2Config(t *testing.T) {
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime-experimental"),
"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
},
"default_runtime_name": "nvidia",
},
@ -265,6 +290,7 @@ func TestRevertV2Config(t *testing.T) {
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
o := &options{
runtimeClass: "nvidia",
}
@ -275,13 +301,19 @@ func TestRevertV2Config(t *testing.T) {
expected, err := toml.TreeFromMap(tc.expected)
require.NoError(t, err, "%d: %v", i, tc)
err = RevertV2Config(config, o)
v2 := &containerd.Config{
Tree: config,
RuntimeType: runtimeType,
}
err = RevertConfig(v2, o)
require.NoError(t, err, "%d: %v", i, tc)
configContents, _ := toml.Marshal(config)
expectedContents, _ := toml.Marshal(expected)
require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
})
}
}

240
tools/container/containerd/containerd.go
View File

@ -21,11 +21,12 @@ import (
"net"
"os"
"os/exec"
"path/filepath"
"syscall"
"time"
toml "github.com/pelletier/go-toml"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/containerd"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
log "github.com/sirupsen/logrus"
cli "github.com/urfave/cli/v2"
)
@ -38,7 +39,7 @@ const (
nvidiaRuntimeName = "nvidia"
nvidiaRuntimeBinary = "nvidia-container-runtime"
nvidiaExperimentalRuntimeName = "nvidia-experimental"
nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime-experimental"
nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime.experimental"
defaultConfig = "/etc/containerd/config.toml"
defaultSocket = "/run/containerd/containerd.sock"
@ -194,25 +195,28 @@ func Setup(c *cli.Context, o *options) error {
}
o.runtimeDir = runtimeDir
cfg, err := LoadConfig(o.config)
cfg, err := containerd.New(
containerd.WithPath(o.config),
containerd.WithRuntimeType(o.runtimeType),
containerd.WithUseLegacyConfig(o.useLegacyConfig),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
version, err := ParseVersion(cfg, o.useLegacyConfig)
if err != nil {
return fmt.Errorf("unable to parse version: %v", err)
}
err = UpdateConfig(cfg, o, version)
err = UpdateConfig(cfg, o)
if err != nil {
return fmt.Errorf("unable to update config: %v", err)
}
err = FlushConfig(o.config, cfg)
log.Infof("Flushing containerd config to %v", o.config)
n, err := cfg.Save(o.config)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
log.Infof("Config file is empty, removed")
}
err = RestartContainerd(o)
if err != nil {
@ -233,25 +237,28 @@ func Cleanup(c *cli.Context, o *options) error {
return fmt.Errorf("unable to parse args: %v", err)
}
cfg, err := LoadConfig(o.config)
cfg, err := containerd.New(
containerd.WithPath(o.config),
containerd.WithRuntimeType(o.runtimeType),
containerd.WithUseLegacyConfig(o.useLegacyConfig),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
version, err := ParseVersion(cfg, o.useLegacyConfig)
if err != nil {
return fmt.Errorf("unable to parse version: %v", err)
}
err = RevertConfig(cfg, o, version)
err = RevertConfig(cfg, o)
if err != nil {
return fmt.Errorf("unable to update config: %v", err)
}
err = FlushConfig(o.config, cfg)
log.Infof("Flushing containerd config to %v", o.config)
n, err := cfg.Save(o.config)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
log.Infof("Config file is empty, removed")
}
err = RestartContainerd(o)
if err != nil {
@ -277,160 +284,36 @@ func ParseArgs(c *cli.Context) (string, error) {
return runtimeDir, nil
}
// LoadConfig loads the containerd config from disk
func LoadConfig(config string) (*toml.Tree, error) {
log.Infof("Loading config: %v", config)
info, err := os.Stat(config)
if os.IsExist(err) && info.IsDir() {
return nil, fmt.Errorf("config file is a directory")
}
configFile := config
if os.IsNotExist(err) {
configFile = "/dev/null"
log.Infof("Config file does not exist, creating new one")
}
cfg, err := toml.LoadFile(configFile)
if err != nil {
return nil, err
}
log.Infof("Successfully loaded config")
return cfg, nil
}
// ParseVersion parses the version field out of the containerd config
func ParseVersion(config *toml.Tree, useLegacyConfig bool) (int, error) {
var defaultVersion int
if !useLegacyConfig {
defaultVersion = 2
} else {
defaultVersion = 1
}
var version int
switch v := config.Get("version").(type) {
case nil:
switch len(config.Keys()) {
case 0: // No config exists, or the config file is empty, use version inferred from containerd
version = defaultVersion
default: // A config file exists, has content, and no version is set
version = 1
}
case int64:
version = int(v)
default:
return -1, fmt.Errorf("unsupported type for version field: %v", v)
}
log.Infof("Config version: %v", version)
if version == 1 {
log.Warnf("Support for containerd config version 1 is deprecated")
}
return version, nil
}
// UpdateConfig updates the containerd config to include the nvidia-container-runtime
func UpdateConfig(config *toml.Tree, o *options, version int) error {
var err error
log.Infof("Updating config")
switch version {
case 1:
err = UpdateV1Config(config, o)
case 2:
err = UpdateV2Config(config, o)
default:
err = fmt.Errorf("unsupported containerd config version: %v", version)
}
func UpdateConfig(cfg engine.Interface, o *options) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.runtimeClass),
operator.WithSetAsDefault(o.setAsDefault),
operator.WithRoot(o.runtimeDir),
)
for class, runtime := range runtimes {
err := cfg.AddRuntime(class, runtime.Path, runtime.SetAsDefault)
if err != nil {
return err
return fmt.Errorf("unable to update config for runtime class '%v': %v", class, err)
}
}
log.Infof("Successfully updated config")
return nil
}
// RevertConfig reverts the containerd config to remove the nvidia-container-runtime
func RevertConfig(config *toml.Tree, o *options, version int) error {
var err error
log.Infof("Reverting config")
switch version {
case 1:
err = RevertV1Config(config, o)
case 2:
err = RevertV2Config(config, o)
default:
err = fmt.Errorf("unsupported containerd config version: %v", version)
}
func RevertConfig(cfg engine.Interface, o *options) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.runtimeClass),
operator.WithSetAsDefault(o.setAsDefault),
operator.WithRoot(o.runtimeDir),
)
for class := range runtimes {
err := cfg.RemoveRuntime(class)
if err != nil {
return err
}
log.Infof("Successfully reverted config")
return nil
}
// UpdateV1Config performs an update specific to v1 of the containerd config
func UpdateV1Config(config *toml.Tree, o *options) error {
c := newConfigV1(config)
return c.Update(o)
}
// RevertV1Config performs a revert specific to v1 of the containerd config
func RevertV1Config(config *toml.Tree, o *options) error {
c := newConfigV1(config)
return c.Revert(o)
}
// UpdateV2Config performs an update specific to v2 of the containerd config
func UpdateV2Config(config *toml.Tree, o *options) error {
c := newConfigV2(config)
return c.Update(o)
}
// RevertV2Config performs a revert specific to v2 of the containerd config
func RevertV2Config(config *toml.Tree, o *options) error {
c := newConfigV2(config)
return c.Revert(o)
}
// FlushConfig flushes the updated/reverted config out to disk
func FlushConfig(config string, cfg *toml.Tree) error {
log.Infof("Flushing config")
output, err := cfg.ToTomlString()
if err != nil {
return fmt.Errorf("unable to convert to TOML: %v", err)
}
switch len(output) {
case 0:
err := os.Remove(config)
if err != nil {
return fmt.Errorf("unable to remove empty file: %v", err)
}
log.Infof("Config empty, removing file")
default:
f, err := os.Create(config)
if err != nil {
return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
}
defer f.Close()
_, err = f.WriteString(output)
if err != nil {
return fmt.Errorf("unable to write output: %v", err)
return fmt.Errorf("unable to revert config for runtime class '%v': %v", class, err)
}
}
log.Infof("Successfully flushed config")
return nil
}
@ -551,36 +434,3 @@ func RestartContainerdSystemd(hostRootMount string) error {
return nil
}
// getDefaultRuntime returns the default runtime for the configured options.
// If the configuration is invalid or the default runtimes should not be set
// the empty string is returned.
func (o options) getDefaultRuntime() string {
if o.setAsDefault {
if o.runtimeClass == nvidiaExperimentalRuntimeName {
return nvidiaExperimentalRuntimeName
}
if o.runtimeClass == "" {
return defaultRuntimeClass
}
return o.runtimeClass
}
return ""
}
// getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
// renaming of the `nvidia` runtime as per the --runtime-class command line flag.
func (o options) getRuntimeBinaries() map[string]string {
runtimeBinaries := make(map[string]string)
for rt, bin := range nvidiaRuntimeBinaries {
runtime := rt
if o.runtimeClass != "" && o.runtimeClass != nvidiaExperimentalRuntimeName && runtime == defaultRuntimeClass {
runtime = o.runtimeClass
}
runtimeBinaries[runtime] = filepath.Join(o.runtimeDir, bin)
}
return runtimeBinaries
}

106
tools/container/containerd/containerd_test.go
View File

@ -1,106 +0,0 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package main
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestOptions(t *testing.T) {
testCases := []struct {
options options
expectedDefaultRuntime string
expectedRuntimeBinaries map[string]string
}{
{
expectedRuntimeBinaries: map[string]string{
"nvidia": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
{
options: options{
setAsDefault: true,
},
expectedDefaultRuntime: "nvidia",
expectedRuntimeBinaries: map[string]string{
"nvidia": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
{
options: options{
setAsDefault: true,
runtimeClass: "nvidia",
},
expectedDefaultRuntime: "nvidia",
expectedRuntimeBinaries: map[string]string{
"nvidia": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
{
options: options{
setAsDefault: true,
runtimeClass: "NAME",
},
expectedDefaultRuntime: "NAME",
expectedRuntimeBinaries: map[string]string{
"NAME": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
{
options: options{
setAsDefault: false,
runtimeClass: "NAME",
},
expectedRuntimeBinaries: map[string]string{
"NAME": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
{
options: options{
setAsDefault: true,
runtimeClass: "nvidia-experimental",
},
expectedDefaultRuntime: "nvidia-experimental",
expectedRuntimeBinaries: map[string]string{
"nvidia": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
{
options: options{
setAsDefault: false,
runtimeClass: "nvidia-experimental",
},
expectedRuntimeBinaries: map[string]string{
"nvidia": "nvidia-container-runtime",
"nvidia-experimental": "nvidia-container-runtime-experimental",
},
},
}
for i, tc := range testCases {
require.Equal(t, tc.expectedDefaultRuntime, tc.options.getDefaultRuntime(), "%d: %v", i, tc)
require.EqualValues(t, tc.expectedRuntimeBinaries, tc.options.getRuntimeBinaries(), "%d: %v", i, tc)
}
}

57
tools/container/crio/crio.go
View File

@ -24,8 +24,9 @@ import (
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
"github.com/pelletier/go-toml"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/crio"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
log "github.com/sirupsen/logrus"
cli "github.com/urfave/cli/v2"
)
@ -213,7 +214,9 @@ func setupHook(o *options) error {
func setupConfig(o *options) error {
log.Infof("Updating config file")
cfg, err := crio.LoadConfig(o.config)
cfg, err := crio.New(
crio.WithPath(o.config),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
@ -223,10 +226,14 @@ func setupConfig(o *options) error {
return fmt.Errorf("unable to update config: %v", err)
}
err = crio.FlushConfig(o.config, cfg)
log.Infof("Flushing cri-o config to %v", o.config)
n, err := cfg.Save(o.config)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
log.Infof("Config file is empty, removed")
}
err = RestartCrio(o)
if err != nil {
@ -267,7 +274,9 @@ func cleanupHook(o *options) error {
func cleanupConfig(o *options) error {
log.Infof("Reverting config file modifications")
cfg, err := crio.LoadConfig(o.config)
cfg, err := crio.New(
crio.WithPath(o.config),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
@ -277,10 +286,14 @@ func cleanupConfig(o *options) error {
return fmt.Errorf("unable to update config: %v", err)
}
err = crio.FlushConfig(o.config, cfg)
log.Infof("Flushing cri-o config to %v", o.config)
n, err := cfg.Save(o.config)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
log.Infof("Config file is empty, removed")
}
err = RestartCrio(o)
if err != nil {
@ -345,14 +358,36 @@ func generateOciHook(toolkitDir string) podmanHook {
}
// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
func UpdateConfig(config *toml.Tree, o *options) error {
runtimePath := filepath.Join(o.runtimeDir, "nvidia-container-runtime")
return crio.UpdateConfig(config, o.runtimeClass, runtimePath, o.setAsDefault)
func UpdateConfig(cfg engine.Interface, o *options) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.runtimeClass),
operator.WithSetAsDefault(o.setAsDefault),
operator.WithRoot(o.runtimeDir),
)
for class, runtime := range runtimes {
err := cfg.AddRuntime(class, runtime.Path, runtime.SetAsDefault)
if err != nil {
return fmt.Errorf("unable to update config for runtime class '%v': %v", class, err)
}
}
return nil
}
// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
func RevertConfig(config *toml.Tree, o *options) error {
return crio.RevertConfig(config, o.runtimeClass)
func RevertConfig(cfg engine.Interface, o *options) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.runtimeClass),
operator.WithSetAsDefault(o.setAsDefault),
operator.WithRoot(o.runtimeDir),
)
for class := range runtimes {
err := cfg.RemoveRuntime(class)
if err != nil {
return fmt.Errorf("unable to revert config for runtime class '%v': %v", class, err)
}
}
return nil
}
// RestartCrio restarts crio depending on the value of restartModeFlag

106
tools/container/docker/docker.go
View File

@ -20,11 +20,12 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"syscall"
"time"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
log "github.com/sirupsen/logrus"
cli "github.com/urfave/cli/v2"
)
@ -36,7 +37,7 @@ const (
nvidiaRuntimeName = "nvidia"
nvidiaRuntimeBinary = "nvidia-container-runtime"
nvidiaExperimentalRuntimeName = "nvidia-experimental"
nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime-experimental"
nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime.experimental"
defaultConfig = "/etc/docker/daemon.json"
defaultSocket = "/var/run/docker.sock"
@ -170,7 +171,9 @@ func Setup(c *cli.Context, o *options) error {
}
o.runtimeDir = runtimeDir
cfg, err := LoadConfig(o.config)
cfg, err := docker.New(
docker.WithPath(o.config),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
@ -180,7 +183,8 @@ func Setup(c *cli.Context, o *options) error {
return fmt.Errorf("unable to update config: %v", err)
}
err = FlushConfig(cfg, o.config)
log.Infof("Flushing docker config to %v", o.config)
_, err = cfg.Save(o.config)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
@ -204,20 +208,26 @@ func Cleanup(c *cli.Context, o *options) error {
return fmt.Errorf("unable to parse args: %v", err)
}
cfg, err := LoadConfig(o.config)
cfg, err := docker.New(
docker.WithPath(o.config),
)
if err != nil {
return fmt.Errorf("unable to load config: %v", err)
}
err = RevertConfig(cfg)
err = RevertConfig(cfg, o)
if err != nil {
return fmt.Errorf("unable to update config: %v", err)
}
err = FlushConfig(cfg, o.config)
log.Infof("Flushing docker config to %v", o.config)
n, err := cfg.Save(o.config)
if err != nil {
return fmt.Errorf("unable to flush config: %v", err)
}
if n == 0 {
log.Infof("Config file is empty, removed")
}
err = RestartDocker(o)
if err != nil {
@ -243,52 +253,40 @@ func ParseArgs(c *cli.Context) (string, error) {
return runtimeDir, nil
}
// LoadConfig loads the docker config from disk
func LoadConfig(config string) (map[string]interface{}, error) {
return docker.LoadConfig(config)
}
// UpdateConfig updates the docker config to include the nvidia runtimes
func UpdateConfig(config map[string]interface{}, o *options) error {
for runtimeName, runtimePath := range o.getRuntimeBinaries() {
setAsDefault := runtimeName == o.getDefaultRuntime()
err := docker.UpdateConfig(config, runtimeName, runtimePath, setAsDefault)
func UpdateConfig(cfg engine.Interface, o *options) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.runtimeName),
operator.WithSetAsDefault(o.setAsDefault),
operator.WithRoot(o.runtimeDir),
)
for name, runtime := range runtimes {
err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault)
if err != nil {
return fmt.Errorf("failed to update runtime %q: %v", runtimeName, err)
return fmt.Errorf("failed to update runtime %q: %v", name, err)
}
}
return nil
}
//RevertConfig reverts the docker config to remove the nvidia runtime
func RevertConfig(config map[string]interface{}) error {
if _, exists := config["default-runtime"]; exists {
defaultRuntime := config["default-runtime"].(string)
if _, exists := nvidiaRuntimeBinaries[defaultRuntime]; exists {
config["default-runtime"] = defaultDockerRuntime
// RevertConfig reverts the docker config to remove the nvidia runtime
func RevertConfig(cfg engine.Interface, o *options) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.runtimeName),
operator.WithSetAsDefault(o.setAsDefault),
operator.WithRoot(o.runtimeDir),
)
for name := range runtimes {
err := cfg.RemoveRuntime(name)
if err != nil {
return fmt.Errorf("failed to remove runtime %q: %v", name, err)
}
}
if _, exists := config["runtimes"]; exists {
runtimes := config["runtimes"].(map[string]interface{})
for name := range nvidiaRuntimeBinaries {
delete(runtimes, name)
}
if len(runtimes) == 0 {
delete(config, "runtimes")
}
}
return nil
}
// FlushConfig flushes the updated/reverted config out to disk
func FlushConfig(cfg map[string]interface{}, config string) error {
return docker.FlushConfig(cfg, config)
}
// RestartDocker restarts docker depending on the value of restartModeFlag
func RestartDocker(o *options) error {
switch o.restartMode {
@ -385,31 +383,3 @@ func SignalDocker(socket string) error {
return nil
}
// getDefaultRuntime returns the default runtime for the configured options.
// If the configuration is invalid or the default runtimes should not be set
// the empty string is returned.
func (o options) getDefaultRuntime() string {
if o.setAsDefault == false {
return ""
}
return o.runtimeName
}
// getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
// renaming of the `nvidia` runtime as per the --runtime-class command line flag.
func (o options) getRuntimeBinaries() map[string]string {
runtimeBinaries := make(map[string]string)
for rt, bin := range nvidiaRuntimeBinaries {
runtime := rt
if o.runtimeName != "" && o.runtimeName != nvidiaExperimentalRuntimeName && runtime == defaultRuntimeName {
runtime = o.runtimeName
}
runtimeBinaries[runtime] = filepath.Join(o.runtimeDir, bin)
}
return runtimeBinaries
}

159
tools/container/docker/docker_test.go
View File

@ -20,6 +20,7 @@ import (
"encoding/json"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
"github.com/stretchr/testify/require"
)
@ -60,9 +61,9 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
runtimeDir: runtimeDir,
}
config := map[string]interface{}{}
config := docker.Config(map[string]interface{}{})
err := UpdateConfig(config, o)
err := UpdateConfig(&config, o)
require.NoError(t, err, "%d: %v", i, tc)
defaultRuntimeName := config["default-runtime"]
@ -74,7 +75,7 @@ func TestUpdateConfig(t *testing.T) {
const runtimeDir = "/test/runtime/dir"
testCases := []struct {
config map[string]interface{}
config docker.Config
setAsDefault bool
runtimeName string
expectedConfig map[string]interface{}
@ -89,7 +90,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -106,7 +115,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -123,7 +140,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -146,7 +171,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -172,7 +205,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -192,7 +233,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -212,7 +261,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -240,7 +297,15 @@ func TestUpdateConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -254,7 +319,8 @@ func TestUpdateConfig(t *testing.T) {
runtimeName: tc.runtimeName,
runtimeDir: runtimeDir,
}
err := UpdateConfig(tc.config, options)
err := UpdateConfig(&tc.config, options)
require.NoError(t, err, "%d: %v", i, tc)
configContent, err := json.MarshalIndent(tc.config, "", " ")
@ -269,7 +335,7 @@ func TestUpdateConfig(t *testing.T) {
func TestRevertConfig(t *testing.T) {
testCases := []struct {
config map[string]interface{}
config docker.Config
expectedConfig map[string]interface{}
}{
{
@ -306,7 +372,30 @@ func TestRevertConfig(t *testing.T) {
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
},
},
expectedConfig: map[string]interface{}{},
},
{
config: map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
@ -368,7 +457,7 @@ func TestRevertConfig(t *testing.T) {
}
for i, tc := range testCases {
err := RevertConfig(tc.config)
err := RevertConfig(&tc.config, &options{})
require.NoError(t, err, "%d: %v", i, tc)
@ -381,43 +470,3 @@ func TestRevertConfig(t *testing.T) {
require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
}
}
func TestFlagsDefaultRuntime(t *testing.T) {
testCases := []struct {
setAsDefault bool
runtimeName string
expected string
}{
{
expected: "",
},
{
runtimeName: "not-bool",
expected: "",
},
{
setAsDefault: false,
runtimeName: "nvidia",
expected: "",
},
{
setAsDefault: true,
runtimeName: "nvidia",
expected: "nvidia",
},
{
setAsDefault: true,
runtimeName: "nvidia-experimental",
expected: "nvidia-experimental",
},
}
for i, tc := range testCases {
f := options{
setAsDefault: tc.setAsDefault,
runtimeName: tc.runtimeName,
}
require.Equal(t, tc.expected, f.getDefaultRuntime(), "%d: %v", i, tc)
}
}

135
tools/container/operator/operator.go Normal file
View File

@ -0,0 +1,135 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package operator
import "path/filepath"
const (
defaultRuntimeName = "nvidia"
experimentalRuntimeName = "nvidia-experimental"
defaultRoot = "/usr/bin"
)
// Runtime defines a runtime to be configured.
// The path and whether the runtime is the default runtime can be specfied
type Runtime struct {
name string
Path string
SetAsDefault bool
}
// Runtimes defines a set of runtimes to be configure for use in the GPU Operator
type Runtimes map[string]Runtime
type config struct {
root string
nvidiaRuntimeName string
setAsDefault bool
}
// GetRuntimes returns the set of runtimes to be configured for use with the GPU Operator.
func GetRuntimes(opts ...Option) Runtimes {
c := &config{}
for _, opt := range opts {
opt(c)
}
if c.root == "" {
c.root = defaultRoot
}
if c.nvidiaRuntimeName == "" {
c.nvidiaRuntimeName = defaultRuntimeName
}
runtimes := make(Runtimes)
runtimes.add(c.nvidiaRuntime())
modes := []string{"experimental", "cdi", "legacy"}
for _, mode := range modes {
runtimes.add(c.modeRuntime(mode))
}
return runtimes
}
// DefaultRuntimeName returns the name of the default runtime.
func (r Runtimes) DefaultRuntimeName() string {
for _, runtime := range r {
if runtime.SetAsDefault {
return runtime.name
}
}
return ""
}
// Add a runtime to the set of runtimes.
func (r *Runtimes) add(runtime Runtime) {
(*r)[runtime.name] = runtime
}
// nvidiaRuntime creates a runtime that corresponds to the nvidia runtime.
// If the nvidiaRuntimeName is specified, this name us used unless this is exactly equal to nvidia-experimental.
func (c config) nvidiaRuntime() Runtime {
predefinedRuntimes := map[string]struct{}{
"nvidia-experimental": {},
"nvidia-cdi": {},
"nvidia-legacy": {},
}
name := c.nvidiaRuntimeName
if _, isPredefinedRuntime := predefinedRuntimes[name]; isPredefinedRuntime {
name = defaultRuntimeName
}
return c.newRuntime(name, "nvidia-container-runtime")
}
// modeRuntime creates a runtime for the specified mode.
func (c config) modeRuntime(mode string) Runtime {
return c.newRuntime("nvidia-"+mode, "nvidia-container-runtime."+mode)
}
// newRuntime creates a runtime based on the configuration
func (c config) newRuntime(name string, binary string) Runtime {
return Runtime{
name: name,
Path: filepath.Join(c.root, binary),
SetAsDefault: c.setAsDefault && name == c.nvidiaRuntimeName,
}
}
// Option is a functional option for configuring set of runtimes.
type Option func(*config)
// WithRoot sets the root directory for the runtime binaries.
func WithRoot(root string) Option {
return func(c *config) {
c.root = root
}
}
// WithNvidiaRuntimeName sets the name of the nvidia runtime.
func WithNvidiaRuntimeName(name string) Option {
return func(c *config) {
c.nvidiaRuntimeName = name
}
}
// WithSetAsDefault sets the default runtime to the nvidia runtime.
func WithSetAsDefault(set bool) Option {
return func(c *config) {
c.setAsDefault = set
}
}

207
tools/container/operator/operator_test.go Normal file
View File

@ -0,0 +1,207 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package operator
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestOptions(t *testing.T) {
testCases := []struct {
setAsDefault bool
nvidiaRuntimeName string
root string
expectedDefaultRuntime string
expectedRuntimes Runtimes
}{
{
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: true,
expectedDefaultRuntime: "nvidia",
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
SetAsDefault: true,
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: true,
nvidiaRuntimeName: "nvidia",
expectedDefaultRuntime: "nvidia",
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
SetAsDefault: true,
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: true,
nvidiaRuntimeName: "NAME",
expectedDefaultRuntime: "NAME",
expectedRuntimes: Runtimes{
"NAME": Runtime{
name: "NAME",
Path: "/usr/bin/nvidia-container-runtime",
SetAsDefault: true,
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: false,
nvidiaRuntimeName: "NAME",
expectedRuntimes: Runtimes{
"NAME": Runtime{
name: "NAME",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: true,
nvidiaRuntimeName: "nvidia-experimental",
expectedDefaultRuntime: "nvidia-experimental",
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
SetAsDefault: true,
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: false,
nvidiaRuntimeName: "nvidia-experimental",
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
runtimes := GetRuntimes(
WithNvidiaRuntimeName(tc.nvidiaRuntimeName),
WithSetAsDefault(tc.setAsDefault),
WithRoot(tc.root),
)
require.EqualValues(t, tc.expectedRuntimes, runtimes)
require.Equal(t, tc.expectedDefaultRuntime, runtimes.DefaultRuntimeName())
})
}
}

38
tools/container/toolkit/runtime.go
View File

@ -21,31 +21,34 @@ import (
"path/filepath"
"strings"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
log "github.com/sirupsen/logrus"
)
const (
nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
nvidiaContainerRuntimeTarget = "nvidia-container-runtime.real"
nvidiaContainerRuntimeWrapper = "nvidia-container-runtime"
nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental"
nvidiaExperimentalContainerRuntimeTarget = nvidiaExperimentalContainerRuntimeSource
nvidiaExperimentalContainerRuntimeWrapper = "nvidia-container-runtime-experimental"
)
// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
// and implementing the required wrapper
func installContainerRuntimes(toolkitDir string, driverRoot string) error {
r := newNvidiaContainerRuntimeInstaller()
runtimes := operator.GetRuntimes()
for _, runtime := range runtimes {
if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource {
continue
}
r := newNvidiaContainerRuntimeInstaller(runtime.Path)
_, err := r.install(toolkitDir)
if err != nil {
return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
}
}
// Install the experimental runtime and treat failures as non-fatal.
err = installExperimentalRuntime(toolkitDir, driverRoot)
err := installExperimentalRuntime(toolkitDir, driverRoot)
if err != nil {
log.Warnf("Could not install experimental runtime: %v", err)
}
@ -70,25 +73,34 @@ func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
return nil
}
func newNvidiaContainerRuntimeInstaller() *executable {
// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime.
// This installer will copy the specified source exectuable to the toolkit directory.
// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is
// created to allow for the configuration of the runtime environment.
func newNvidiaContainerRuntimeInstaller(source string) *executable {
wrapperName := filepath.Base(source)
dotfileName := wrapperName + ".real"
target := executableTarget{
dotfileName: nvidiaContainerRuntimeTarget,
wrapperName: nvidiaContainerRuntimeWrapper,
dotfileName: dotfileName,
wrapperName: wrapperName,
}
return newRuntimeInstaller(nvidiaContainerRuntimeSource, target, nil)
return newRuntimeInstaller(source, target, nil)
}
func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {
source := nvidiaExperimentalContainerRuntimeSource
wrapperName := filepath.Base(source)
dotfileName := wrapperName + ".real"
target := executableTarget{
dotfileName: nvidiaExperimentalContainerRuntimeTarget,
wrapperName: nvidiaExperimentalContainerRuntimeWrapper,
dotfileName: dotfileName,
wrapperName: wrapperName,
}
env := make(map[string]string)
if libraryRoot != "" {
env["LD_LIBRARY_PATH"] = strings.Join([]string{libraryRoot, "$LD_LIBRARY_PATH"}, ":")
}
return newRuntimeInstaller(nvidiaExperimentalContainerRuntimeSource, target, env)
return newRuntimeInstaller(source, target, env)
}
func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {

2
tools/container/toolkit/runtime_test.go
View File

@ -25,7 +25,7 @@ import (
)
func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
r := newNvidiaContainerRuntimeInstaller()
r := newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource)
const shebang = "#! /bin/sh"
const destFolder = "/dest/folder"