UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-06-26 18:18:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move cmd/nvidia-container-runtime/modifier package to internal/modifier

Browse Source 
Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
2022-06-15 14:12:17 +02:00
parent d66c00dd1d
commit e8843c38f2
9 changed files with 267 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/nvidia-container-runtime/main_test.go
View File

@@ -10,7 +10,7 @@ import (
"strings"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/require"

154
cmd/nvidia-container-runtime/modifier/csv.go
View File

@@ -1,154 +0,0 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package modifier
import (
"fmt"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
"github.com/NVIDIA/nvidia-container-toolkit/internal/cuda"
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements"
"github.com/sirupsen/logrus"
)
// csvMode represents the modifications as performed by the csv runtime mode
type csvMode struct {
logger *logrus.Logger
discoverer discover.Discover
}
const (
visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
visibleDevicesVoid = "void"
nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
)
// NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
// The modifications are defined by CSV MountSpecs.
func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
rawSpec, err := ociSpec.Load()
if err != nil {
return nil, fmt.Errorf("failed to load OCI spec: %v", err)
}
// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
// if no modification is required.
visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
return nil, nil
}
logger.Infof("Constructing modifier from config: %+v", *cfg)
config := &discover.Config{
Root: cfg.NVIDIAContainerCLIConfig.Root,
NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
}
// TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the
// visible devices are checked.
image, err := image.NewCUDAImageFromSpec(rawSpec)
if err != nil {
return nil, err
}
if err := checkRequirements(logger, &image); err != nil {
return nil, fmt.Errorf("requirements not met: %v", err)
}
csvFiles, err := csv.GetFileList(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.MountSpecPath)
if err != nil {
return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
}
nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
if nvidiaRequireJetpack != "csv-mounts=all" {
csvFiles = csv.BaseFilesOnly(csvFiles)
}
csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
if err != nil {
return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
}
ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
if err != nil {
return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
}
createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config)
if err != nil {
return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
}
d := discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
return newCSVModifierFromDiscoverer(logger, d)
}
// newCSVModifierFromDiscoverer is used to test with dependency injection
func newCSVModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
discoverModifier, err := modifier.NewModifierFromDiscoverer(logger, d)
if err != nil {
return nil, fmt.Errorf("failed to construct modifier: %v", err)
}
modifiers := modifier.Merge(
nvidiaContainerRuntimeHookRemover{logger},
discoverModifier,
)
return modifiers, nil
}
func checkRequirements(logger *logrus.Logger, image *image.CUDA) error {
if image.HasDisableRequire() {
// TODO: We could print the real value here instead
logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true)
return nil
}
imageRequirements, err := image.GetRequirements()
if err != nil {
// TODO: Should we treat this as a failure, or just issue a warning?
return fmt.Errorf("failed to get image requirements: %v", err)
}
r := requirements.New(logger, imageRequirements)
cudaVersion, err := cuda.Version()
if err != nil {
logger.Warnf("Failed to get CUDA version: %v", err)
} else {
r.AddVersionProperty(requirements.CUDA, cudaVersion)
}
compteCapability, err := cuda.ComputeCapability(0)
if err != nil {
logger.Warnf("Failed to get CUDA Compute Capability: %v", err)
} else {
r.AddVersionProperty(requirements.ARCH, compteCapability)
}
return r.Assert()
}

284
cmd/nvidia-container-runtime/modifier/csv_test.go
View File

@@ -1,284 +0,0 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package modifier
import (
"fmt"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
"github.com/opencontainers/runtime-spec/specs-go"
testlog "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
)
func TestNewCSVModifier(t *testing.T) {
logger, _ := testlog.NewNullLogger()
testCases := []struct {
description string
cfg *config.Config
spec oci.Spec
visibleDevices string
expectedError error
expectedNil bool
}{
{
description: "spec load error returns error",
spec: &oci.SpecMock{
LoadFunc: func() (*specs.Spec, error) {
return nil, fmt.Errorf("load failed")
},
},
expectedError: fmt.Errorf("load failed"),
},
{
description: "visible devices not set returns nil",
visibleDevices: "NOT_SET",
expectedNil: true,
},
{
description: "visible devices empty returns nil",
visibleDevices: "",
expectedNil: true,
},
{
description: "visible devices 'void' returns nil",
visibleDevices: "void",
expectedNil: true,
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
spec := tc.spec
if spec == nil {
spec = &oci.SpecMock{
LookupEnvFunc: func(s string) (string, bool) {
if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
return tc.visibleDevices, true
}
return "", false
},
}
}
m, err := NewCSVModifier(logger, tc.cfg, spec)
if tc.expectedError != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
if tc.expectedNil || tc.expectedError != nil {
require.Nil(t, m)
} else {
require.NotNil(t, m)
}
})
}
}
func TestExperimentalModifier(t *testing.T) {
logger, _ := testlog.NewNullLogger()
testCases := []struct {
description string
discover *discover.DiscoverMock
spec *specs.Spec
expectedError error
expectedSpec *specs.Spec
}{
{
description: "empty discoverer does not modify spec",
discover: &discover.DiscoverMock{},
},
{
description: "failed hooks discoverer returns error",
discover: &discover.DiscoverMock{
HooksFunc: func() ([]discover.Hook, error) {
return nil, fmt.Errorf("discover.Hooks error")
},
},
expectedError: fmt.Errorf("discover.Hooks error"),
},
{
description: "discovered hooks are injected into spec",
spec: &specs.Spec{},
discover: &discover.DiscoverMock{
HooksFunc: func() ([]discover.Hook, error) {
hooks := []discover.Hook{
{
Lifecycle: "prestart",
Path: "/hook/a",
Args: []string{"/hook/a", "arga"},
},
{
Lifecycle: "createContainer",
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
}
return hooks, nil
},
},
expectedSpec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/hook/a",
Args: []string{"/hook/a", "arga"},
},
},
CreateContainer: []specs.Hook{
{
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
},
},
},
},
{
description: "existing hooks are maintained",
spec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/hook/a",
Args: []string{"/hook/a", "arga"},
},
},
},
},
discover: &discover.DiscoverMock{
HooksFunc: func() ([]discover.Hook, error) {
hooks := []discover.Hook{
{
Lifecycle: "prestart",
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
}
return hooks, nil
},
},
expectedSpec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/hook/a",
Args: []string{"/hook/a", "arga"},
},
{
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
},
},
},
},
{
description: "modification removes existing nvidia-container-runtime-hook",
spec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/path/to/nvidia-container-runtime-hook",
Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
},
},
},
},
discover: &discover.DiscoverMock{
HooksFunc: func() ([]discover.Hook, error) {
hooks := []discover.Hook{
{
Lifecycle: "prestart",
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
}
return hooks, nil
},
},
expectedSpec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
},
},
},
},
{
description: "modification removes existing nvidia-container-toolkit",
spec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/path/to/nvidia-container-toolkit",
Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
},
},
},
},
discover: &discover.DiscoverMock{
HooksFunc: func() ([]discover.Hook, error) {
hooks := []discover.Hook{
{
Lifecycle: "prestart",
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
}
return hooks, nil
},
},
expectedSpec: &specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "/hook/b",
Args: []string{"/hook/b", "argb"},
},
},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
m, err := newCSVModifierFromDiscoverer(logger, tc.discover)
require.NoError(t, err)
err = m.Modify(tc.spec)
if tc.expectedError != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.EqualValues(t, tc.expectedSpec, tc.spec)
})
}
}

79
cmd/nvidia-container-runtime/modifier/hook_remover.go
View File

@@ -1,79 +0,0 @@
/**
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package modifier
import (
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
// nvidiaContainerRuntimeHookRemover is a spec modifer that detects and removes inserted nvidia-container-runtime hooks
type nvidiaContainerRuntimeHookRemover struct {
logger *logrus.Logger
}
var _ oci.SpecModifier = (*nvidiaContainerRuntimeHookRemover)(nil)
// Modify removes any NVIDIA Container Runtime hooks from the provided spec
func (m nvidiaContainerRuntimeHookRemover) Modify(spec *specs.Spec) error {
if spec == nil {
return nil
}
if spec.Hooks == nil {
return nil
}
if len(spec.Hooks.Prestart) == 0 {
return nil
}
var newPrestart []specs.Hook
for _, hook := range spec.Hooks.Prestart {
if isNVIDIAContainerRuntimeHook(&hook) {
m.logger.Debugf("Removing hook %v", hook)
continue
}
newPrestart = append(newPrestart, hook)
}
if len(newPrestart) != len(spec.Hooks.Prestart) {
m.logger.Debugf("Updating 'prestart' hooks to %v", newPrestart)
spec.Hooks.Prestart = newPrestart
}
return nil
}
// isNVIDIAContainerRuntimeHook checks if the provided hook is an nvidia-container-runtime-hook
// or nvidia-container-toolkit hook. These are included, for example, by the non-experimental
// nvidia-container-runtime or docker when specifying the --gpus flag.
func isNVIDIAContainerRuntimeHook(hook *specs.Hook) bool {
bins := map[string]struct{}{
config.NVIDIAContainerRuntimeHookExecutable: {},
config.NVIDIAContainerToolkitExecutable: {},
}
_, exists := bins[filepath.Base(hook.Path)]
return exists
}

77
cmd/nvidia-container-runtime/modifier/stable.go
View File

@@ -1,77 +0,0 @@
/*
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package modifier
import (
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
// NewStableRuntimeModifier creates an OCI spec modifier that inserts the NVIDIA Container Runtime Hook into an OCI
// spec. The specified logger is used to capture log output.
func NewStableRuntimeModifier(logger *logrus.Logger) oci.SpecModifier {
m := stableRuntimeModifier{logger: logger}
return &m
}
// stableRuntimeModifier modifies an OCI spec inplace, inserting the nvidia-container-runtime-hook as a
// prestart hook. If the hook is already present, no modification is made.
type stableRuntimeModifier struct {
logger *logrus.Logger
}
// Modify applies the required modification to the incoming OCI spec, inserting the nvidia-container-runtime-hook
// as a prestart hook.
func (m stableRuntimeModifier) Modify(spec *specs.Spec) error {
path, err := exec.LookPath(config.NVIDIAContainerRuntimeHookExecutable)
if err != nil {
path = filepath.Join(config.DefaultExecutableDir, config.NVIDIAContainerRuntimeHookExecutable)
_, err = os.Stat(path)
if err != nil {
return err
}
}
m.logger.Infof("Using prestart hook path: %s", path)
args := []string{path}
if spec.Hooks == nil {
spec.Hooks = &specs.Hooks{}
} else if len(spec.Hooks.Prestart) != 0 {
for _, hook := range spec.Hooks.Prestart {
if strings.Contains(hook.Path, config.NVIDIAContainerRuntimeHookExecutable) {
m.logger.Infof("existing nvidia prestart hook found in OCI spec")
return nil
}
}
}
spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
Path: path,
Args: append(args, "prestart"),
})
return nil
}

170
cmd/nvidia-container-runtime/modifier/stable_test.go
View File

@@ -1,170 +0,0 @@
/*
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package modifier
import (
"os"
"path/filepath"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
testlog "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
)
type testConfig struct {
root string
binPath string
}
var cfg *testConfig
func TestMain(m *testing.M) {
// TEST SETUP
// Determine the module root and the test binary path
var err error
moduleRoot, err := test.GetModuleRoot()
if err != nil {
logrus.Fatalf("error in test setup: could not get module root: %v", err)
}
testBinPath := filepath.Join(moduleRoot, "test", "bin")
// Set the environment variables for the test
os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
// Store the root and binary paths in the test Config
cfg = &testConfig{
root: moduleRoot,
binPath: testBinPath,
}
// RUN TESTS
exitCode := m.Run()
os.Exit(exitCode)
}
func TestAddHookModifier(t *testing.T) {
logger, logHook := testlog.NewNullLogger()
testHookPath := filepath.Join(cfg.binPath, "nvidia-container-runtime-hook")
testCases := []struct {
description string
spec specs.Spec
expectedError error
expectedSpec specs.Spec
}{
{
description: "empty spec adds hook",
spec: specs.Spec{},
expectedSpec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: testHookPath,
Args: []string{testHookPath, "prestart"},
},
},
},
},
},
{
description: "spec with empty hooks adds hook",
spec: specs.Spec{
Hooks: &specs.Hooks{},
},
expectedSpec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: testHookPath,
Args: []string{testHookPath, "prestart"},
},
},
},
},
},
{
description: "hook is not replaced",
spec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "nvidia-container-runtime-hook",
},
},
},
},
expectedSpec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "nvidia-container-runtime-hook",
},
},
},
},
},
{
description: "other hooks are not replaced",
spec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "some-hook",
},
},
},
},
expectedSpec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{
{
Path: "some-hook",
},
{
Path: testHookPath,
Args: []string{testHookPath, "prestart"},
},
},
},
},
},
}
for _, tc := range testCases {
logHook.Reset()
t.Run(tc.description, func(t *testing.T) {
m := NewStableRuntimeModifier(logger)
err := m.Modify(&tc.spec)
if tc.expectedError != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.EqualValues(t, tc.expectedSpec, tc.spec)
})
}
}

2
cmd/nvidia-container-runtime/runtime_factory.go
View File

@@ -19,9 +19,9 @@ package main
import (
"fmt"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
"github.com/sirupsen/logrus"