UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2024-11-22 00:08:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #497 from elezar/systemdcgroup

Browse Source 
Add option to set additional containerd configs per runtime
This commit is contained in:
Evan Lezar 2024-05-21 12:05:51 +02:00 committed by GitHub
commit 2d7b2360d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 313 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
cmd/nvidia-ctk/runtime/configure/configure.go
View File

@ -17,6 +17,7 @@
package configure
import (
"encoding/json"
"fmt"
"path/filepath"
@ -66,6 +67,8 @@ type config struct {
mode string
hookFilePath string
runtimeConfigOverrideJSON string
nvidiaRuntime struct {
name string
path string
@ -153,6 +156,13 @@ func (m command) build() *cli.Command {
Usage: "Enable CDI in the configured runtime",
Destination: &config.cdi.enabled,
},
&cli.StringFlag{
Name: "runtime-config-override",
Destination: &config.runtimeConfigOverrideJSON,
Usage: "specify additional runtime options as a JSON string. The paths are relative to the runtime config.",
Value: "{}",
EnvVars: []string{"RUNTIME_CONFIG_OVERRIDE"},
},
}
return &configure
@ -194,6 +204,11 @@ func (m command) validateFlags(c *cli.Context, config *config) error {
config.cdi.enabled = false
}
if config.runtimeConfigOverrideJSON != "" && config.runtime != "containerd" {
m.logger.Warningf("Ignoring runtime-config-override flag for %v", config.runtime)
config.runtimeConfigOverrideJSON = ""
}
return nil
}
@ -237,10 +252,16 @@ func (m command) configureConfigFile(c *cli.Context, config *config) error {
return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
}
runtimeConfigOverride, err := config.runtimeConfigOverride()
if err != nil {
return fmt.Errorf("unable to parse config overrides: %w", err)
}
err = cfg.AddRuntime(
config.nvidiaRuntime.name,
config.nvidiaRuntime.path,
config.nvidiaRuntime.setAsDefault,
runtimeConfigOverride,
)
if err != nil {
return fmt.Errorf("unable to update config: %v", err)
@ -293,6 +314,20 @@ func (c *config) getOuputConfigPath() string {
return c.resolveConfigFilePath()
}
// runtimeConfigOverride converts the specified runtimeConfigOverride JSON string to a map.
func (o *config) runtimeConfigOverride() (map[string]interface{}, error) {
if o.runtimeConfigOverrideJSON == "" {
return nil, nil
}
runtimeOptions := make(map[string]interface{})
if err := json.Unmarshal([]byte(o.runtimeConfigOverrideJSON), &runtimeOptions); err != nil {
return nil, fmt.Errorf("failed to read %v as JSON: %w", o.runtimeConfigOverrideJSON, err)
}
return runtimeOptions, nil
}
// configureOCIHook creates and configures the OCI hook for the NVIDIA runtime
func (m *command) configureOCIHook(c *cli.Context, config *config) error {
err := ocihook.CreateHook(config.hookFilePath, config.nvidiaRuntime.hookPath)

2
pkg/config/engine/api.go
View File

@ -19,7 +19,7 @@ package engine
// Interface defines the API for a runtime config updater.
type Interface interface {
DefaultRuntime() string
AddRuntime(string, string, bool) error
AddRuntime(string, string, bool, ...map[string]interface{}) error
Set(string, interface{})
RemoveRuntime(string) error
Save(string) (int64, error)

12
pkg/config/engine/containerd/config_v1.go
View File

@ -30,7 +30,7 @@ type ConfigV1 Config
var _ engine.Interface = (*ConfigV1)(nil)
// AddRuntime adds a runtime to the containerd config
func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool, configOverrides ...map[string]interface{}) error {
if c == nil || c.Tree == nil {
return fmt.Errorf("config is nil")
}
@ -75,6 +75,16 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error
}
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
defaultRuntimeSubtree := subtreeAtPath(config, "plugins", "cri", "containerd", "default_runtime")
if err := defaultRuntimeSubtree.applyOverrides(configOverrides...); err != nil {
return fmt.Errorf("failed to apply config overrides to default_runtime: %w", err)
}
}
runtimeSubtree := subtreeAtPath(config, "plugins", "cri", "containerd", "runtimes", name)
if err := runtimeSubtree.applyOverrides(configOverrides...); err != nil {
return fmt.Errorf("failed to apply config overrides: %w", err)
}
*c.Tree = config

7
pkg/config/engine/containerd/config_v2.go
View File

@ -25,7 +25,7 @@ import (
)
// AddRuntime adds a runtime to the containerd config
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
func (c *Config) AddRuntime(name string, path string, setAsDefault bool, configOverrides ...map[string]interface{}) error {
if c == nil || c.Tree == nil {
return fmt.Errorf("config is nil")
}
@ -60,6 +60,11 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
}
runtimeSubtree := subtreeAtPath(config, "plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name)
if err := runtimeSubtree.applyOverrides(configOverrides...); err != nil {
return fmt.Errorf("failed to apply config overrides: %w", err)
}
*c.Tree = config
return nil
}

97
pkg/config/engine/containerd/config_v2_test.go Normal file
View File

@ -0,0 +1,97 @@
/**
# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package containerd
import (
"testing"
"github.com/pelletier/go-toml"
"github.com/stretchr/testify/require"
)
func TestAddRuntime(t *testing.T) {
testCases := []struct {
description string
config string
setAsDefault bool
configOverrides []map[string]interface{}
expectedConfig string
expectedError error
}{
{
description: "empty config not default runtime",
expectedConfig: `
version = 2
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
[plugins."io.containerd.grpc.v1.cri".containerd]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
privileged_without_host_devices = false
runtime_engine = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
BinaryName = "/usr/bin/test"
`,
expectedError: nil,
},
{
description: "empty config not default runtime with overrides",
configOverrides: []map[string]interface{}{
{
"options": map[string]interface{}{
"SystemdCgroup": true,
},
},
},
expectedConfig: `
version = 2
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
[plugins."io.containerd.grpc.v1.cri".containerd]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
privileged_without_host_devices = false
runtime_engine = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
BinaryName = "/usr/bin/test"
SystemdCgroup = true
`,
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
config, err := toml.Load(tc.config)
require.NoError(t, err)
expectedConfig, err := toml.Load(tc.expectedConfig)
require.NoError(t, err)
c := &Config{
Tree: config,
}
err = c.AddRuntime("test", "/usr/bin/test", tc.setAsDefault, tc.configOverrides...)
require.NoError(t, err)
require.EqualValues(t, expectedConfig.String(), config.String())
})
}
}

56
pkg/config/engine/containerd/toml.go Normal file
View File

@ -0,0 +1,56 @@
/**
# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package containerd
import (
"fmt"
"github.com/pelletier/go-toml"
)
// tomlTree is an alias for toml.Tree that allows for extensions.
type tomlTree toml.Tree
func subtreeAtPath(c toml.Tree, path ...string) *tomlTree {
tree := c.GetPath(path).(*toml.Tree)
return (*tomlTree)(tree)
}
func (t *tomlTree) insert(other map[string]interface{}) error {
for key, value := range other {
if insertsubtree, ok := value.(map[string]interface{}); ok {
subtree := (*toml.Tree)(t).Get(key).(*toml.Tree)
return (*tomlTree)(subtree).insert(insertsubtree)
}
(*toml.Tree)(t).Set(key, value)
}
return nil
}
func (t *tomlTree) applyOverrides(overrides ...map[string]interface{}) error {
for _, override := range overrides {
subconfig, err := toml.TreeFromMap(override)
if err != nil {
return fmt.Errorf("invalid toml config: %w", err)
}
if err := t.insert(subconfig.ToMap()); err != nil {
return err
}
}
return nil
}

2
pkg/config/engine/crio/crio.go
View File

@ -40,7 +40,7 @@ func New(opts ...Option) (engine.Interface, error) {
}
// AddRuntime adds a new runtime to the crio config
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
func (c *Config) AddRuntime(name string, path string, setAsDefault bool, _ ...map[string]interface{}) error {
if c == nil {
return fmt.Errorf("config is nil")
}

2
pkg/config/engine/docker/docker.go
View File

@ -49,7 +49,7 @@ func New(opts ...Option) (engine.Interface, error) {
}
// AddRuntime adds a new runtime to the docker config
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
func (c *Config) AddRuntime(name string, path string, setAsDefault bool, _ ...map[string]interface{}) error {
if c == nil {
return fmt.Errorf("config is nil")
}

8
tools/container/container.go
View File

@ -67,8 +67,8 @@ func ParseArgs(c *cli.Context, o *Options) error {
}
// Configure applies the options to the specified config
func (o Options) Configure(cfg engine.Interface) error {
err := o.UpdateConfig(cfg)
func (o Options) Configure(cfg engine.Interface, configOverrides ...map[string]interface{}) error {
err := o.UpdateConfig(cfg, configOverrides...)
if err != nil {
return fmt.Errorf("unable to update config: %v", err)
}
@ -98,14 +98,14 @@ func (o Options) flush(cfg engine.Interface) error {
}
// UpdateConfig updates the specified config to include the nvidia runtimes
func (o Options) UpdateConfig(cfg engine.Interface) error {
func (o Options) UpdateConfig(cfg engine.Interface, configOverrides ...map[string]interface{}) error {
runtimes := operator.GetRuntimes(
operator.WithNvidiaRuntimeName(o.RuntimeName),
operator.WithSetAsDefault(o.SetAsDefault),
operator.WithRoot(o.RuntimeDir),
)
for name, runtime := range runtimes {
err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault)
err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault, configOverrides...)
if err != nil {
return fmt.Errorf("failed to update runtime %q: %v", name, err)
}

31
tools/container/containerd/containerd.go
View File

@ -17,6 +17,7 @@
package main
import (
"encoding/json"
"fmt"
"os"
@ -47,6 +48,8 @@ type options struct {
runtimeType string
ContainerRuntimeModesCDIAnnotationPrefixes cli.StringSlice
runtimeConfigOverrideJSON string
}
func main() {
@ -162,6 +165,13 @@ func main() {
Destination: &options.ContainerRuntimeModesCDIAnnotationPrefixes,
EnvVars: []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES"},
},
&cli.StringFlag{
Name: "runtime-config-override",
Destination: &options.runtimeConfigOverrideJSON,
Usage: "specify additional runtime options as a JSON string. The paths are relative to the runtime config.",
Value: "{}",
EnvVars: []string{"RUNTIME_CONFIG_OVERRIDE", "CONTAINERD_RUNTIME_CONFIG_OVERRIDE"},
},
}
// Update the subcommand flags with the common subcommand flags
@ -170,7 +180,7 @@ func main() {
// Run the top-level CLI
if err := c.Run(os.Args); err != nil {
log.Fatal(fmt.Errorf("Error: %v", err))
log.Fatal(fmt.Errorf("error: %v", err))
}
}
@ -188,7 +198,11 @@ func Setup(c *cli.Context, o *options) error {
return fmt.Errorf("unable to load config: %v", err)
}
err = o.Configure(cfg)
runtimeConfigOverride, err := o.runtimeConfigOverride()
if err != nil {
return fmt.Errorf("unable to parse config overrides: %w", err)
}
err = o.Configure(cfg, runtimeConfigOverride)
if err != nil {
return fmt.Errorf("unable to configure containerd: %v", err)
}
@ -246,3 +260,16 @@ func (o *options) containerAnnotationsFromCDIPrefixes() []string {
return annotations
}
func (o *options) runtimeConfigOverride() (map[string]interface{}, error) {
if o.runtimeConfigOverrideJSON == "" {
return nil, nil
}
runtimeOptions := make(map[string]interface{})
if err := json.Unmarshal([]byte(o.runtimeConfigOverrideJSON), &runtimeOptions); err != nil {
return nil, fmt.Errorf("failed to read %v as JSON: %w", o.runtimeConfigOverrideJSON, err)
}
return runtimeOptions, nil
}

72
tools/container/containerd/containerd_test.go Normal file
View File

@ -0,0 +1,72 @@
/**
# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package main
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestRuntimeOptions(t *testing.T) {
testCases := []struct {
description string
options options
expected map[string]interface{}
expectedError error
}{
{
description: "empty is nil",
},
{
description: "empty json",
options: options{
runtimeConfigOverrideJSON: "{}",
},
expected: map[string]interface{}{},
expectedError: nil,
},
{
description: "SystemdCgroup is true",
options: options{
runtimeConfigOverrideJSON: "{\"SystemdCgroup\": true}",
},
expected: map[string]interface{}{
"SystemdCgroup": true,
},
expectedError: nil,
},
{
description: "SystemdCgroup is false",
options: options{
runtimeConfigOverrideJSON: "{\"SystemdCgroup\": false}",
},
expected: map[string]interface{}{
"SystemdCgroup": false,
},
expectedError: nil,
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
runtimeOptions, err := tc.options.runtimeConfigOverride()
require.ErrorIs(t, tc.expectedError, err)
require.EqualValues(t, tc.expected, runtimeOptions)
})
}
}