UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2024-11-22 08:18:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'fix-symlink-discovery' into 'main'

Browse Source 
Fix bug in creating symlinks in containers on Tegra-based systems

See merge request nvidia/container-toolkit/container-toolkit!479
This commit is contained in:
Evan Lezar 2023-09-25 10:01:08 +00:00
commit 807c87e057
12 changed files with 345 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,5 +1,9 @@
# NVIDIA Container Toolkit Changelog
## v1.14.2
* Fix bug on Tegra-based systems where symlinks were not created in containers.
* Add --csv.ignore-pattern command line option to nvidia-ctk cdi generate command.
## v1.14.1
* Fixed bug where contents of `/etc/nvidia-container-runtime/config.toml` is ignored by the NVIDIA Container Runtime Hook.

11
cmd/nvidia-ctk/cdi/generate/generate.go
View File

@ -53,7 +53,8 @@ type options struct {
librarySearchPaths cli.StringSlice
csv struct {
files cli.StringSlice
files cli.StringSlice
ignorePatterns cli.StringSlice
}
}
@ -141,6 +142,11 @@ func (m command) build() *cli.Command {
Value: cli.NewStringSlice(csv.DefaultFileList()...),
Destination: &opts.csv.files,
},
&cli.StringSliceFlag{
Name: "csv.ignore-pattern",
Usage: "Specify a pattern the CSV mount specifications.",
Destination: &opts.csv.ignorePatterns,
},
}
return &c
@ -233,8 +239,9 @@ func (m command) generateSpec(opts *options) (spec.Interface, error) {
nvcdi.WithNVIDIACTKPath(opts.nvidiaCTKPath),
nvcdi.WithDeviceNamer(deviceNamer),
nvcdi.WithMode(string(opts.mode)),
nvcdi.WithCSVFiles(opts.csv.files.Value()),
nvcdi.WithLibrarySearchPaths(opts.librarySearchPaths.Value()),
nvcdi.WithCSVFiles(opts.csv.files.Value()),
nvcdi.WithCSVIgnorePatterns(opts.csv.ignorePatterns.Value()),
)
if err != nil {
return nil, fmt.Errorf("failed to create CDI library: %v", err)

50
internal/platform-support/tegra/csv.go
View File

@ -28,51 +28,45 @@ import (
// newDiscovererFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
// The constructed discoverer is comprised of a list, with each element in the list being associated with a
// single CSV files.
func newDiscovererFromCSVFiles(logger logger.Interface, files []string, driverRoot string, nvidiaCTKPath string, librarySearchPaths []string) (discover.Discover, error) {
if len(files) == 0 {
logger.Warningf("No CSV files specified")
func (o tegraOptions) newDiscovererFromCSVFiles() (discover.Discover, error) {
if len(o.csvFiles) == 0 {
o.logger.Warningf("No CSV files specified")
return discover.None{}, nil
}
targetsByType := getTargetsFromCSVFiles(logger, files)
targetsByType := getTargetsFromCSVFiles(o.logger, o.csvFiles)
devices := discover.NewDeviceDiscoverer(
logger,
lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
driverRoot,
o.logger,
lookup.NewCharDeviceLocator(lookup.WithLogger(o.logger), lookup.WithRoot(o.driverRoot)),
o.driverRoot,
targetsByType[csv.MountSpecDev],
)
directories := discover.NewMounts(
logger,
lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
driverRoot,
o.logger,
lookup.NewDirectoryLocator(lookup.WithLogger(o.logger), lookup.WithRoot(o.driverRoot)),
o.driverRoot,
targetsByType[csv.MountSpecDir],
)
// Libraries and symlinks use the same locator.
searchPaths := append(librarySearchPaths, "/")
symlinkLocator := lookup.NewSymlinkLocator(
lookup.WithLogger(logger),
lookup.WithRoot(driverRoot),
lookup.WithSearchPaths(searchPaths...),
)
libraries := discover.NewMounts(
logger,
symlinkLocator,
driverRoot,
o.logger,
o.symlinkLocator,
o.driverRoot,
targetsByType[csv.MountSpecLib],
)
nonLibSymlinks := ignoreFilenamePatterns{"*.so", "*.so.[0-9]"}.Apply(targetsByType[csv.MountSpecSym]...)
logger.Debugf("Non-lib symlinks: %v", nonLibSymlinks)
symlinkTargets := o.ignorePatterns.Apply(targetsByType[csv.MountSpecSym]...)
o.logger.Debugf("Filtered symlink targets: %v", symlinkTargets)
symlinks := discover.NewMounts(
logger,
symlinkLocator,
driverRoot,
nonLibSymlinks,
o.logger,
o.symlinkLocator,
o.driverRoot,
symlinkTargets,
)
createSymlinks := createCSVSymlinkHooks(logger, nonLibSymlinks, libraries, nvidiaCTKPath)
createSymlinks := o.createCSVSymlinkHooks(symlinkTargets, libraries)
d := discover.Merge(
devices,
@ -87,7 +81,9 @@ func newDiscovererFromCSVFiles(logger logger.Interface, files []string, driverRo
// getTargetsFromCSVFiles returns the list of mount specs from the specified CSV files.
// These are aggregated by mount spec type.
func getTargetsFromCSVFiles(logger logger.Interface, files []string) map[csv.MountSpecType][]string {
// TODO: We use a function variable here to allow this to be overridden for testing.
// This should be properly mocked.
var getTargetsFromCSVFiles = func(logger logger.Interface, files []string) map[csv.MountSpecType][]string {
targetsByType := make(map[csv.MountSpecType][]string)
for _, filename := range files {
targets, err := loadCSVFile(logger, filename)

206
internal/platform-support/tegra/csv_test.go
View File

@ -15,3 +15,209 @@
**/
package tegra
import (
"fmt"
"testing"
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
testlog "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
)
func TestDiscovererFromCSVFiles(t *testing.T) {
logger, _ := testlog.NewNullLogger()
testCases := []struct {
description string
moutSpecs map[csv.MountSpecType][]string
ignorePatterns []string
symlinkLocator lookup.Locator
symlinkChainLocator lookup.Locator
symlinkResolver func(string) (string, error)
expectedError error
expectedMounts []discover.Mount
expectedMountsError error
expectedHooks []discover.Hook
expectedHooksError error
}{
{
// TODO: This current resolves to two mounts that are the same.
// These are deduplicated at a later stage. We could consider deduplicating earlier in the pipeline.
description: "symlink is resolved to target; mounts and symlink are created",
moutSpecs: map[csv.MountSpecType][]string{
"lib": {"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"},
"sym": {"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so"},
},
symlinkLocator: &lookup.LocatorMock{
LocateFunc: func(path string) ([]string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return []string{"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, nil
}
return []string{path}, nil
},
},
symlinkChainLocator: &lookup.LocatorMock{
LocateFunc: func(path string) ([]string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return []string{"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so", "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, nil
}
return nil, fmt.Errorf("Unexpected path: %v", path)
},
},
symlinkResolver: func(path string) (string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so", nil
}
return path, nil
},
expectedMounts: []discover.Mount{
{
Path: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
HostPath: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
Options: []string{"ro", "nosuid", "nodev", "bind"},
},
{
Path: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
HostPath: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
Options: []string{"ro", "nosuid", "nodev", "bind"},
},
},
expectedHooks: []discover.Hook{
{
Lifecycle: "createContainer",
Path: "/usr/bin/nvidia-ctk",
Args: []string{
"nvidia-ctk",
"hook",
"create-symlinks",
"--link",
"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so::/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so",
},
},
},
},
{
// TODO: This current resolves to two mounts that are the same.
// These are deduplicated at a later stage. We could consider deduplicating earlier in the pipeline.
description: "single glob filter does not remove symlink mounts",
moutSpecs: map[csv.MountSpecType][]string{
"lib": {"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"},
"sym": {"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so"},
},
ignorePatterns: []string{"*.so"},
symlinkLocator: &lookup.LocatorMock{
LocateFunc: func(path string) ([]string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return []string{"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, nil
}
return []string{path}, nil
},
},
symlinkChainLocator: &lookup.LocatorMock{
LocateFunc: func(path string) ([]string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return []string{"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so", "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, nil
}
return nil, fmt.Errorf("Unexpected path: %v", path)
},
},
symlinkResolver: func(path string) (string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so", nil
}
return path, nil
},
expectedMounts: []discover.Mount{
{
Path: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
HostPath: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
Options: []string{"ro", "nosuid", "nodev", "bind"},
},
{
Path: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
HostPath: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
Options: []string{"ro", "nosuid", "nodev", "bind"},
},
},
expectedHooks: []discover.Hook{
{
Lifecycle: "createContainer",
Path: "/usr/bin/nvidia-ctk",
Args: []string{
"nvidia-ctk",
"hook",
"create-symlinks",
"--link",
"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so::/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so",
},
},
},
},
{
description: "** filter removes symlink mounts",
moutSpecs: map[csv.MountSpecType][]string{
"lib": {"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"},
"sym": {"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so"},
},
symlinkLocator: &lookup.LocatorMock{
LocateFunc: func(path string) ([]string, error) {
if path == "/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so" {
return []string{"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, nil
}
return []string{path}, nil
},
},
ignorePatterns: []string{"**/*.so"},
expectedMounts: []discover.Mount{
{
Path: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
HostPath: "/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so",
Options: []string{"ro", "nosuid", "nodev", "bind"},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
defer setGetTargetsFromCSVFiles(tc.moutSpecs)()
o := tegraOptions{
logger: logger,
nvidiaCTKPath: "/usr/bin/nvidia-ctk",
csvFiles: []string{"dummy"},
ignorePatterns: tc.ignorePatterns,
symlinkLocator: tc.symlinkLocator,
symlinkChainLocator: tc.symlinkChainLocator,
resolveSymlink: tc.symlinkResolver,
}
d, err := o.newDiscovererFromCSVFiles()
require.ErrorIs(t, err, tc.expectedError)
hooks, err := d.Hooks()
require.ErrorIs(t, err, tc.expectedHooksError)
require.EqualValues(t, tc.expectedHooks, hooks)
mounts, err := d.Mounts()
require.ErrorIs(t, err, tc.expectedMountsError)
require.EqualValues(t, tc.expectedMounts, mounts)
})
}
}
func setGetTargetsFromCSVFiles(ovverride map[csv.MountSpecType][]string) func() {
original := getTargetsFromCSVFiles
getTargetsFromCSVFiles = func(logger logger.Interface, files []string) map[csv.MountSpecType][]string {
return ovverride
}
return func() {
getTargetsFromCSVFiles = original
}
}

18
internal/platform-support/tegra/filter.go
View File

@ -16,20 +16,28 @@
package tegra
import "path/filepath"
import (
"path/filepath"
"strings"
)
type ignoreFilenamePatterns []string
type ignoreMountSpecPatterns []string
func (d ignoreFilenamePatterns) Match(name string) bool {
func (d ignoreMountSpecPatterns) Match(name string) bool {
for _, pattern := range d {
if match, _ := filepath.Match(pattern, filepath.Base(name)); match {
target := name
if strings.HasPrefix(pattern, "**/") {
target = filepath.Base(name)
pattern = strings.TrimPrefix(pattern, "**/")
}
if match, _ := filepath.Match(pattern, target); match {
return true
}
}
return false
}
func (d ignoreFilenamePatterns) Apply(input ...string) []string {
func (d ignoreMountSpecPatterns) Apply(input ...string) []string {
var filtered []string
for _, name := range input {
if d.Match(name) {

32
internal/platform-support/tegra/filter_test.go
View File

@ -23,7 +23,35 @@ import (
)
func TestIgnorePatterns(t *testing.T) {
filtered := ignoreFilenamePatterns{"*.so", "*.so.[0-9]"}.Apply("/foo/bar/libsomething.so", "libsometing.so", "libsometing.so.1", "libsometing.so.1.2.3")
testCases := []struct {
description string
blockedFilter []string
input []string
expected []string
}{
{
description: "nil slice",
input: []string{"something", "somethingelse"},
expected: []string{"something", "somethingelse"},
},
{
description: "match libraries full path and so symlinks using globs",
blockedFilter: []string{"*.so", "*.so.[0-9]"},
input: []string{"/foo/bar/libsomething.so", "libsometing.so", "libsometing.so.1", "libsometing.so.1.2.3"},
expected: []string{"/foo/bar/libsomething.so", "libsometing.so.1.2.3"},
},
{
description: "match libraries full path and so symlinks using globs with any path prefix",
blockedFilter: []string{"**/*.so", "**/*.so.[0-9]"},
input: []string{"/foo/bar/libsomething.so", "libsometing.so", "libsometing.so.1", "libsometing.so.1.2.3"},
expected: []string{"libsometing.so.1.2.3"},
},
}
require.ElementsMatch(t, []string{"libsometing.so.1.2.3"}, filtered)
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
filtered := ignoreMountSpecPatterns(tc.blockedFilter).Apply(tc.input...)
require.ElementsMatch(t, tc.expected, filtered)
})
}
}

27
internal/platform-support/tegra/symlinks.go
View File

@ -24,25 +24,29 @@ import (
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
)
type symlinkHook struct {
discover.None
logger logger.Interface
driverRoot string
nvidiaCTKPath string
targets []string
mountsFrom discover.Discover
// The following can be overridden for testing
symlinkChainLocator lookup.Locator
resolveSymlink func(string) (string, error)
}
// createCSVSymlinkHooks creates a discoverer for a hook that creates required symlinks in the container
func createCSVSymlinkHooks(logger logger.Interface, targets []string, mounts discover.Discover, nvidiaCTKPath string) discover.Discover {
func (o tegraOptions) createCSVSymlinkHooks(targets []string, mounts discover.Discover) discover.Discover {
return symlinkHook{
logger: logger,
nvidiaCTKPath: nvidiaCTKPath,
targets: targets,
mountsFrom: mounts,
logger: o.logger,
nvidiaCTKPath: o.nvidiaCTKPath,
targets: targets,
mountsFrom: mounts,
symlinkChainLocator: o.symlinkChainLocator,
resolveSymlink: o.resolveSymlink,
}
}
@ -105,14 +109,9 @@ func (d symlinkHook) getSpecificLinks() ([]string, error) {
// getSymlinkCandidates returns a list of symlinks that are candidates for being created.
func (d symlinkHook) getSymlinkCandidates() []string {
chainLocator := lookup.NewSymlinkChainLocator(
lookup.WithLogger(d.logger),
lookup.WithRoot(d.driverRoot),
)
var candidates []string
for _, target := range d.targets {
reslovedSymlinkChain, err := chainLocator.Locate(target)
reslovedSymlinkChain, err := d.symlinkChainLocator.Locate(target)
if err != nil {
d.logger.Warningf("Failed to locate symlink %v", target)
continue
@ -127,7 +126,7 @@ func (d symlinkHook) getCSVFileSymlinks() []string {
created := make(map[string]bool)
// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
for _, candidate := range d.getSymlinkCandidates() {
target, err := symlinks.Resolve(candidate)
target, err := d.resolveSymlink(candidate)
if err != nil {
d.logger.Debugf("Skipping invalid link: %v", err)
continue

37
internal/platform-support/tegra/tegra.go
View File

@ -22,6 +22,7 @@ import (
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
)
type tegraOptions struct {
@ -30,6 +31,13 @@ type tegraOptions struct {
driverRoot string
nvidiaCTKPath string
librarySearchPaths []string
ignorePatterns ignoreMountSpecPatterns
// The following can be overridden for testing
symlinkLocator lookup.Locator
symlinkChainLocator lookup.Locator
// TODO: This should be replaced by a regular mock
resolveSymlink func(string) (string, error)
}
// Option defines a functional option for configuring a Tegra discoverer.
@ -42,7 +50,27 @@ func New(opts ...Option) (discover.Discover, error) {
opt(o)
}
csvDiscoverer, err := newDiscovererFromCSVFiles(o.logger, o.csvFiles, o.driverRoot, o.nvidiaCTKPath, o.librarySearchPaths)
if o.symlinkLocator == nil {
searchPaths := append(o.librarySearchPaths, "/")
o.symlinkLocator = lookup.NewSymlinkLocator(
lookup.WithLogger(o.logger),
lookup.WithRoot(o.driverRoot),
lookup.WithSearchPaths(searchPaths...),
)
}
if o.symlinkChainLocator == nil {
o.symlinkChainLocator = lookup.NewSymlinkChainLocator(
lookup.WithLogger(o.logger),
lookup.WithRoot(o.driverRoot),
)
}
if o.resolveSymlink == nil {
o.resolveSymlink = symlinks.Resolve
}
csvDiscoverer, err := o.newDiscovererFromCSVFiles()
if err != nil {
return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
}
@ -105,3 +133,10 @@ func WithLibrarySearchPaths(librarySearchPaths ...string) Option {
o.librarySearchPaths = librarySearchPaths
}
}
// WithIngorePatterns sets patterns to ignore in the CSV files
func WithIngorePatterns(ignorePatterns ...string) Option {
return func(o *tegraOptions) {
o.ignorePatterns = ignoreMountSpecPatterns(ignorePatterns)
}
}

1
pkg/nvcdi/lib-csv.go
View File

@ -45,6 +45,7 @@ func (l *csvlib) GetAllDeviceSpecs() ([]specs.Device, error) {
tegra.WithNVIDIACTKPath(l.nvidiaCTKPath),
tegra.WithCSVFiles(l.csvFiles),
tegra.WithLibrarySearchPaths(l.librarySearchPaths...),
tegra.WithIngorePatterns(l.csvIgnorePatterns...),
)
if err != nil {
return nil, fmt.Errorf("failed to create discoverer for CSV files: %v", err)

3
pkg/nvcdi/lib.go
View File

@ -47,7 +47,8 @@ type nvcdilib struct {
nvidiaCTKPath string
librarySearchPaths []string
csvFiles []string
csvFiles []string
csvIgnorePatterns []string
vendor string
class string

7
pkg/nvcdi/options.go
View File

@ -104,6 +104,13 @@ func WithCSVFiles(csvFiles []string) Option {
}
}
// WithCSVIgnorePatterns sets the ignore patterns for entries in the CSV files.
func WithCSVIgnorePatterns(csvIgnorePatterns []string) Option {
return func(o *nvcdilib) {
o.csvIgnorePatterns = csvIgnorePatterns
}
}
// WithLibrarySearchPaths sets the library search paths.
// This is currently only used for CSV-mode.
func WithLibrarySearchPaths(paths []string) Option {

2
versions.mk
View File

@ -13,7 +13,7 @@
# limitations under the License.
LIB_NAME := nvidia-container-toolkit
LIB_VERSION := 1.14.1
LIB_VERSION := 1.14.2
LIB_TAG :=
# The package version is the combination of the library version and tag.