UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-06-26 18:18:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for containerd version 3 config

Browse Source 
This change adds support for containerd configs with version=3.
From the perspective of the runtime configuration the contents of the
config are the same. This means that we just have to load the new
version and ensure that this is propagated to the generated config.

Note that v3 config also requires a switch to the 'io.containerd.cri.v1.runtime'
CRI runtime plugin. See:
https://github.com/containerd/containerd/blob/v2.0.0/docs/PLUGINS.md
https://github.com/containerd/containerd/issues/10132

Note that we still use a default config of version=2 since we need to
ensure compatibility with older containerd versions (1.6.x and 1.7.x).

Signed-off-by: Sam Lockart <sam.lockart@zendesk.com>
Signed-off-by: Evan Lezar <elezar@nvidia.com>
Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
This commit is contained in:
Sam Lockart
2024-11-18 16:20:55 +11:00
committed by Evan Lezar
parent ec182705f1
commit 95bda3ad72
8 changed files with 262 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
pkg/config/engine/containerd/config_v2.go → pkg/config/engine/containerd/config.go
View File

@@ -30,40 +30,40 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
}
config := *c.Tree
config.Set("version", int64(2))
config.Set("version", c.Version)
runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
for _, r := range runtimeNamesForConfig {
options := config.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", r})
options := config.GetSubtreeByPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", r})
if options == nil {
continue
}
c.Logger.Debugf("using options from runtime %v: %v", r, options)
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, options.Copy())
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name}, options.Copy())
break
}
if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
if config.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name}) == nil {
c.Logger.Warningf("could not infer options from runtimes %v; using defaults", runtimeNamesForConfig)
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "runtime_root"}, "")
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "runtime_engine"}, "")
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
}
if len(c.ContainerAnnotations) > 0 {
annotations, err := c.getRuntimeAnnotations([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"})
annotations, err := c.getRuntimeAnnotations([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "container_annotations"})
if err != nil {
return err
}
annotations = append(c.ContainerAnnotations, annotations...)
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "container_annotations"}, annotations)
}
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "options", "BinaryName"}, path)
if setAsDefault {
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}, name)
}
*c.Tree = config
@@ -99,13 +99,13 @@ func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
// Set sets the specified containerd option.
func (c *Config) Set(key string, value interface{}) {
config := *c.Tree
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", key}, value)
config.SetPath([]string{"plugins", c.CRIRuntimePluginName, key}, value)
*c.Tree = config
}
// DefaultRuntime returns the default runtime for the cri-o config
func (c Config) DefaultRuntime() string {
if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
if runtime, ok := c.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}).(string); ok {
return runtime
}
return ""
@@ -119,14 +119,14 @@ func (c *Config) RemoveRuntime(name string) error {
config := *c.Tree
config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
config.DeletePath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name})
if runtime, ok := config.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}).(string); ok {
if runtime == name {
config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
config.DeletePath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"})
}
}
runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
runtimePath := []string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name}
for i := 0; i < len(runtimePath); i++ {
if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
if len(runtimes.Keys()) == 0 {

62
pkg/config/engine/containerd/config_v2_test.go → pkg/config/engine/containerd/config_test.go
View File

@@ -195,6 +195,68 @@ func TestAddRuntime(t *testing.T) {
SystemdCgroup = false
`,
},
{
description: "empty v3 spec is supported",
config: `
version = 3
`,
expectedConfig: `
version = 3
[plugins]
[plugins."io.containerd.cri.v1.runtime"]
[plugins."io.containerd.cri.v1.runtime".containerd]
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test]
privileged_without_host_devices = false
runtime_engine = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test.options]
BinaryName = "/usr/bin/test"
`,
expectedError: nil,
},
{
description: "v3 spec is supported",
config: `
version = 3
[plugins]
[plugins."io.containerd.cri.v1.runtime"]
[plugins."io.containerd.cri.v1.runtime".containerd]
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
privileged_without_host_devices = true
runtime_engine = "engine"
runtime_root = "root"
runtime_type = "type"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
`,
expectedConfig: `
version = 3
[plugins]
[plugins."io.containerd.cri.v1.runtime"]
[plugins."io.containerd.cri.v1.runtime".containerd]
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
privileged_without_host_devices = true
runtime_engine = "engine"
runtime_root = "root"
runtime_type = "type"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test]
privileged_without_host_devices = true
runtime_engine = "engine"
runtime_root = "root"
runtime_type = "type"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test.options]
BinaryName = "/usr/bin/test"
SystemdCgroup = true
`,
},
}
for _, tc := range testCases {

75
pkg/config/engine/containerd/containerd.go
View File

@@ -24,9 +24,15 @@ import (
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
)
const (
defaultConfigVersion = 2
defaultRuntimeType = "io.containerd.runc.v2"
)
// Config represents the containerd config
type Config struct {
*toml.Tree
Version int64
Logger logger.Interface
RuntimeType string
ContainerAnnotations []string
@@ -36,6 +42,10 @@ type Config struct {
// This was deprecated in v1.4 in favour of containerd.default_runtime_name.
// Support for this section has been removed in v2.0.
UseLegacyConfig bool
// CRIRuntimePluginName represents the fully qualified name of the containerd plugin
// for the CRI runtime service. The name of this plugin was changed in v3 of the
// containerd configuration file.
CRIRuntimePluginName string
}
var _ engine.Interface = (*Config)(nil)
@@ -60,7 +70,8 @@ func (c *containerdCfgRuntime) GetBinaryPath() string {
// New creates a containerd config with the specified options
func New(opts ...Option) (engine.Interface, error) {
b := &builder{
runtimeType: defaultRuntimeType,
configVersion: defaultConfigVersion,
runtimeType: defaultRuntimeType,
}
for _, opt := range opts {
opt(b)
@@ -77,56 +88,74 @@ func New(opts ...Option) (engine.Interface, error) {
return nil, fmt.Errorf("failed to load config: %v", err)
}
configVersion, err := b.parseVersion(tomlConfig)
if err != nil {
return nil, fmt.Errorf("failed to parse config version: %w", err)
}
b.logger.Infof("Using config version %v", configVersion)
criRuntimePluginName, err := b.criRuntimePluginName(configVersion)
if err != nil {
return nil, fmt.Errorf("failed to get CRI runtime plugin name: %w", err)
}
b.logger.Infof("Using CRI runtime plugin name %q", criRuntimePluginName)
cfg := &Config{
Tree: tomlConfig,
Version: configVersion,
CRIRuntimePluginName: criRuntimePluginName,
Logger: b.logger,
RuntimeType: b.runtimeType,
ContainerAnnotations: b.containerAnnotations,
UseLegacyConfig: b.useLegacyConfig,
ContainerAnnotations: b.containerAnnotations,
}
version, err := cfg.parseVersion()
if err != nil {
return nil, fmt.Errorf("failed to parse config version: %v", err)
}
switch version {
switch configVersion {
case 1:
return (*ConfigV1)(cfg), nil
case 2:
default:
return cfg, nil
}
return nil, fmt.Errorf("unsupported config version: %v", version)
}
// parseVersion returns the version of the config
func (c *Config) parseVersion() (int, error) {
defaultVersion := 2
// For legacy configs, we default to v1 configs.
if c.UseLegacyConfig {
defaultVersion = 1
func (b *builder) parseVersion(c *toml.Tree) (int64, error) {
if c == nil || len(c.Keys()) == 0 {
// No config exists, or the config file is empty.
if b.useLegacyConfig {
// If a legacy config is explicitly requested, we default to a v1 config.
return 1, nil
}
// Use the requested version.
return int64(b.configVersion), nil
}
switch v := c.Get("version").(type) {
case nil:
switch len(c.Keys()) {
case 0: // No config exists, or the config file is empty, use version inferred from containerd
return defaultVersion, nil
default: // A config file exists, has content, and no version is set
return 1, nil
}
return 1, nil
case int64:
return int(v), nil
return v, nil
default:
return -1, fmt.Errorf("unsupported type for version field: %v", v)
}
}
func (b *builder) criRuntimePluginName(configVersion int64) (string, error) {
switch configVersion {
case 1:
return "cri", nil
case 2:
return "io.containerd.grpc.v1.cri", nil
default:
return "io.containerd.cri.v1.runtime", nil
}
}
func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
if c == nil || c.Tree == nil {
return nil, fmt.Errorf("config is nil")
}
runtimeData := c.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
runtimeData := c.GetSubtreeByPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name})
return &containerdCfgRuntime{
tree: runtimeData,
}, nil

16
pkg/config/engine/containerd/option.go
View File

@@ -21,16 +21,13 @@ import (
"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
)
const (
defaultRuntimeType = "io.containerd.runc.v2"
)
type builder struct {
logger logger.Interface
configSource toml.Loader
configVersion int
useLegacyConfig bool
path string
runtimeType string
useLegacyConfig bool
containerAnnotations []string
}
@@ -65,13 +62,20 @@ func WithRuntimeType(runtimeType string) Option {
}
}
// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder.
func WithUseLegacyConfig(useLegacyConfig bool) Option {
return func(b *builder) {
b.useLegacyConfig = useLegacyConfig
}
}
// WithConfigVersion sets the config version for the config builder
func WithConfigVersion(configVersion int) Option {
return func(b *builder) {
b.configVersion = configVersion
}
}
// WithContainerAnnotations sets the container annotations for the config builder
func WithContainerAnnotations(containerAnnotations ...string) Option {
return func(b *builder) {