UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-01-23 19:17:02 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Merge branch 'specify-nvidia-ctk-path' into 'main'

Browse Source 
Make handling of nvidia-ctk path consistent

See merge request nvidia/container-toolkit/container-toolkit!261
This commit is contained in:
Evan Lezar 2023-01-19 10:20:25 +00:00
commit f50efca73f
14 changed files with 179 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/nvidia-ctk/cdi/generate/common.go
View File

@ -27,7 +27,7 @@ import (
// NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device. // NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
// This includes driver libraries and meta devices, for example. // This includes driver libraries and meta devices, for example.
func NewCommonDiscoverer(logger *logrus.Logger, root string, nvmllib nvml.Interface) (discover.Discover, error) { func NewCommonDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
metaDevices := discover.NewDeviceDiscoverer( metaDevices := discover.NewDeviceDiscoverer(
logger, logger,
lookup.NewCharDeviceLocator(logger, root), lookup.NewCharDeviceLocator(logger, root),
@ -45,7 +45,7 @@ func NewCommonDiscoverer(logger *logrus.Logger, root string, nvmllib nvml.Interf
return nil, fmt.Errorf("error constructing discoverer for graphics mounts: %v", err) return nil, fmt.Errorf("error constructing discoverer for graphics mounts: %v", err)
} }
driverFiles, err := NewDriverDiscoverer(logger, root, nvmllib) driverFiles, err := NewDriverDiscoverer(logger, root, nvidiaCTKPath, nvmllib)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err) return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err)
} }

23
cmd/nvidia-ctk/cdi/generate/device-folder-permissions.go
View File

@ -20,15 +20,15 @@ import (
"path/filepath" "path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/container-orchestrated-devices/container-device-interface/specs-go" "github.com/container-orchestrated-devices/container-device-interface/specs-go"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
type deviceFolderPermissions struct { type deviceFolderPermissions struct {
logger *logrus.Logger logger *logrus.Logger
root string root string
folders []string nvidiaCTKPath string
folders []string
} }
var _ discover.Discover = (*deviceFolderPermissions)(nil) var _ discover.Discover = (*deviceFolderPermissions)(nil)
@ -39,7 +39,7 @@ var _ discover.Discover = (*deviceFolderPermissions)(nil)
// The nested devices that are applicable to the NVIDIA GPU devices are: // The nested devices that are applicable to the NVIDIA GPU devices are:
// - DRM devices at /dev/dri/* // - DRM devices at /dev/dri/*
// - NVIDIA Caps devices at /dev/nvidia-caps/* // - NVIDIA Caps devices at /dev/nvidia-caps/*
func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string, deviceSpecs []specs.Device) (discover.Discover, error) { func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, deviceSpecs []specs.Device) (discover.Discover, error) {
var folders []string var folders []string
seen := make(map[string]bool) seen := make(map[string]bool)
for _, device := range deviceSpecs { for _, device := range deviceSpecs {
@ -65,9 +65,10 @@ func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string,
} }
d := &deviceFolderPermissions{ d := &deviceFolderPermissions{
logger: logger, logger: logger,
root: root, root: root,
folders: folders, nvidiaCTKPath: nvidiaCTKPath,
folders: folders,
} }
return d, nil return d, nil
@ -88,11 +89,9 @@ func (d *deviceFolderPermissions) Hooks() ([]discover.Hook, error) {
for _, folder := range d.folders { for _, folder := range d.folders {
args = append(args, "--path", folder) args = append(args, "--path", folder)
} }
hook := discover.CreateNvidiaCTKHook( hook := discover.CreateNvidiaCTKHook(
d.logger, d.nvidiaCTKPath,
lookup.NewExecutableLocator(d.logger, d.root),
nvidiaCTKExecutable,
nvidiaCTKDefaultFilePath,
"chmod", "chmod",
args..., args...,
) )

27
cmd/nvidia-ctk/cdi/generate/driver.go
View File

@ -29,22 +29,23 @@ import (
) )
type driverLibraries struct { type driverLibraries struct {
logger *logrus.Logger logger *logrus.Logger
root string root string
libraries []string nvidiaCTKPath string
libraries []string
} }
var _ discover.Discover = (*driverLibraries)(nil) var _ discover.Discover = (*driverLibraries)(nil)
// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation. // NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
// The supplied NVML Library is used to query the expected driver version. // The supplied NVML Library is used to query the expected driver version.
func NewDriverDiscoverer(logger *logrus.Logger, root string, nvmllib nvml.Interface) (discover.Discover, error) { func NewDriverDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
version, r := nvmllib.SystemGetDriverVersion() version, r := nvmllib.SystemGetDriverVersion()
if r != nvml.SUCCESS { if r != nvml.SUCCESS {
return nil, fmt.Errorf("failed to determine driver version: %v", r) return nil, fmt.Errorf("failed to determine driver version: %v", r)
} }
libraries, err := NewDriverLibraryDiscoverer(logger, root, version) libraries, err := NewDriverLibraryDiscoverer(logger, root, nvidiaCTKPath, version)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err) return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
} }
@ -63,16 +64,17 @@ func NewDriverDiscoverer(logger *logrus.Logger, root string, nvmllib nvml.Interf
} }
// NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version. // NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version.
func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, version string) (discover.Discover, error) { func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath, version string) (discover.Discover, error) {
libraries, err := findVersionLibs(logger, root, version) libraries, err := findVersionLibs(logger, root, version)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get libraries for driver version: %v", err) return nil, fmt.Errorf("failed to get libraries for driver version: %v", err)
} }
d := driverLibraries{ d := driverLibraries{
logger: logger, logger: logger,
root: root, root: root,
libraries: libraries, nvidiaCTKPath: nvidiaCTKPath,
libraries: libraries,
} }
return &d, nil return &d, nil
@ -129,13 +131,8 @@ func (d *driverLibraries) Mounts() ([]discover.Mount, error) {
// Hooks returns a hook that updates the LDCache for the specified driver library paths. // Hooks returns a hook that updates the LDCache for the specified driver library paths.
func (d *driverLibraries) Hooks() ([]discover.Hook, error) { func (d *driverLibraries) Hooks() ([]discover.Hook, error) {
locator := lookup.NewExecutableLocator(d.logger, d.root)
hook := discover.CreateLDCacheUpdateHook( hook := discover.CreateLDCacheUpdateHook(
d.logger, d.nvidiaCTKPath,
locator,
nvidiaCTKExecutable,
nvidiaCTKDefaultFilePath,
d.libraries, d.libraries,
) )

33
cmd/nvidia-ctk/cdi/generate/full-gpu.go
View File

@ -31,15 +31,16 @@ import (
// byPathHookDiscoverer discovers the entities required for injecting by-path DRM device links // byPathHookDiscoverer discovers the entities required for injecting by-path DRM device links
type byPathHookDiscoverer struct { type byPathHookDiscoverer struct {
logger *logrus.Logger logger *logrus.Logger
root string root string
pciBusID string nvidiaCTKPath string
pciBusID string
} }
var _ discover.Discover = (*byPathHookDiscoverer)(nil) var _ discover.Discover = (*byPathHookDiscoverer)(nil)
// NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device. // NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device.
func NewFullGPUDiscoverer(logger *logrus.Logger, root string, d device.Device) (discover.Discover, error) { func NewFullGPUDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, d device.Device) (discover.Discover, error) {
// TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface. // TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface.
// This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin. // This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin.
minor, ret := d.GetMinorNumber() minor, ret := d.GetMinorNumber()
@ -68,9 +69,10 @@ func NewFullGPUDiscoverer(logger *logrus.Logger, root string, d device.Device) (
) )
byPathHooks := &byPathHookDiscoverer{ byPathHooks := &byPathHookDiscoverer{
logger: logger, logger: logger,
root: root, root: root,
pciBusID: pciBusID, nvidiaCTKPath: nvidiaCTKPath,
pciBusID: pciBusID,
} }
dd := discover.Merge( dd := discover.Merge(
@ -98,21 +100,18 @@ func (d *byPathHookDiscoverer) Hooks() ([]discover.Hook, error) {
return nil, nil return nil, nil
} }
hookPath := "nvidia-ctk" var args []string
args := []string{hookPath, "hook", "create-symlinks"}
for _, l := range links { for _, l := range links {
args = append(args, "--link", l) args = append(args, "--link", l)
} }
var hooks []discover.Hook hook := discover.CreateNvidiaCTKHook(
hook := discover.Hook{ d.nvidiaCTKPath,
Lifecycle: "createContainer", "create-symlinks",
Path: hookPath, args...,
Args: args, )
}
hooks = append(hooks, hook)
return hooks, nil return []discover.Hook{hook}, nil
} }
// Mounts returns an empty slice for a full GPU // Mounts returns an empty slice for a full GPU

31
cmd/nvidia-ctk/cdi/generate/generate.go
View File

@ -35,9 +35,6 @@ import (
) )
const ( const (
nvidiaCTKExecutable = "nvidia-ctk"
nvidiaCTKDefaultFilePath = "/usr/bin/" + nvidiaCTKExecutable
formatJSON = "json" formatJSON = "json"
formatYAML = "yaml" formatYAML = "yaml"
) )
@ -47,9 +44,10 @@ type command struct {
} }
type config struct { type config struct {
output string output string
format string format string
root string root string
nvidiaCTKPath string
} }
// NewCommand constructs a generate-cdi command with the specified logger // NewCommand constructs a generate-cdi command with the specified logger
@ -93,6 +91,11 @@ func (m command) build() *cli.Command {
Usage: "Specify the root to use when discovering the entities that should be included in the CDI specification.", Usage: "Specify the root to use when discovering the entities that should be included in the CDI specification.",
Destination: &cfg.root, Destination: &cfg.root,
}, },
&cli.StringFlag{
Name: "nvidia-ctk-path",
Usage: "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
Destination: &cfg.nvidiaCTKPath,
},
} }
return &c return &c
@ -111,7 +114,7 @@ func (m command) validateFlags(r *cli.Context, cfg *config) error {
} }
func (m command) run(c *cli.Context, cfg *config) error { func (m command) run(c *cli.Context, cfg *config) error {
spec, err := m.generateSpec(cfg.root) spec, err := m.generateSpec(cfg.root, cfg.nvidiaCTKPath)
if err != nil { if err != nil {
return fmt.Errorf("failed to generate CDI spec: %v", err) return fmt.Errorf("failed to generate CDI spec: %v", err)
} }
@ -190,7 +193,7 @@ func writeToOutput(format string, data []byte, output io.Writer) error {
return nil return nil
} }
func (m command) generateSpec(root string) (*specs.Spec, error) { func (m command) generateSpec(root string, nvidiaCTKPath string) (*specs.Spec, error) {
nvmllib := nvml.New() nvmllib := nvml.New()
if r := nvmllib.Init(); r != nvml.SUCCESS { if r := nvmllib.Init(); r != nvml.SUCCESS {
return nil, r return nil, r
@ -199,7 +202,9 @@ func (m command) generateSpec(root string) (*specs.Spec, error) {
devicelib := device.New(device.WithNvml(nvmllib)) devicelib := device.New(device.WithNvml(nvmllib))
deviceSpecs, err := m.generateDeviceSpecs(devicelib, root) useNvidiaCTKPath := discover.FindNvidiaCTK(m.logger, nvidiaCTKPath)
deviceSpecs, err := m.generateDeviceSpecs(devicelib, root, useNvidiaCTKPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create device CDI specs: %v", err) return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
} }
@ -226,12 +231,12 @@ func (m command) generateSpec(root string) (*specs.Spec, error) {
allEdits.Append(ipcEdits) allEdits.Append(ipcEdits)
common, err := NewCommonDiscoverer(m.logger, root, nvmllib) common, err := NewCommonDiscoverer(m.logger, root, nvidiaCTKPath, nvmllib)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err) return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
} }
deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, root, deviceSpecs) deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, root, nvidiaCTKPath, deviceSpecs)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err) return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
} }
@ -255,7 +260,7 @@ func (m command) generateSpec(root string) (*specs.Spec, error) {
return &spec, nil return &spec, nil
} }
func (m command) generateDeviceSpecs(devicelib device.Interface, root string) ([]specs.Device, error) { func (m command) generateDeviceSpecs(devicelib device.Interface, root string, nvidiaCTKPath string) ([]specs.Device, error) {
var deviceSpecs []specs.Device var deviceSpecs []specs.Device
err := devicelib.VisitDevices(func(i int, d device.Device) error { err := devicelib.VisitDevices(func(i int, d device.Device) error {
@ -266,7 +271,7 @@ func (m command) generateDeviceSpecs(devicelib device.Interface, root string) ([
if isMigEnabled { if isMigEnabled {
return nil return nil
} }
device, err := NewFullGPUDiscoverer(m.logger, root, d) device, err := NewFullGPUDiscoverer(m.logger, root, nvidiaCTKPath, d)
if err != nil { if err != nil {
return fmt.Errorf("failed to create device: %v", err) return fmt.Errorf("failed to create device: %v", err)
} }

7
internal/discover/discover.go
View File

@ -18,8 +18,8 @@ package discover
// Config represents the configuration options for discovery // Config represents the configuration options for discovery
type Config struct { type Config struct {
Root string Root string
NVIDIAContainerToolkitCLIExecutablePath string NvidiaCTKPath string
} }
// Device represents a discovered character device. // Device represents a discovered character device.
@ -41,8 +41,9 @@ type Hook struct {
Args []string Args []string
} }
//go:generate moq -stub -out discover_mock.go . Discover
// Discover defines an interface for discovering the devices, mounts, and hooks available on a system // Discover defines an interface for discovering the devices, mounts, and hooks available on a system
//
//go:generate moq -stub -out discover_mock.go . Discover
type Discover interface { type Discover interface {
Devices() ([]Device, error) Devices() ([]Device, error)
Mounts() ([]Mount, error) Mounts() ([]Mount, error)

45
internal/discover/graphics.go
View File

@ -25,7 +25,6 @@ import (
"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm" "github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc" "github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -95,21 +94,19 @@ func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover,
type drmDevicesByPath struct { type drmDevicesByPath struct {
None None
logger *logrus.Logger logger *logrus.Logger
lookup lookup.Locator nvidiaCTKPath string
nvidiaCTKExecutablePath string root string
root string devicesFrom Discover
devicesFrom Discover
} }
// newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer // newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover { func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover {
d := drmDevicesByPath{ d := drmDevicesByPath{
logger: logger, logger: logger,
lookup: lookup.NewExecutableLocator(logger, cfg.Root), nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath, root: cfg.Root,
root: cfg.Root, devicesFrom: devices,
devicesFrom: devices,
} }
return &d return &d
@ -132,30 +129,18 @@ func (d drmDevicesByPath) Hooks() ([]Hook, error) {
return nil, nil return nil, nil
} }
hookPath := nvidiaCTKDefaultFilePath var args []string
targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
if err != nil {
d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
} else if len(targets) == 0 {
d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
} else {
d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
hookPath = targets[0]
}
d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
args := []string{hookPath, "hook", "create-symlinks"}
for _, l := range links { for _, l := range links {
args = append(args, "--link", l) args = append(args, "--link", l)
} }
h := Hook{ hook := CreateNvidiaCTKHook(
Lifecycle: cdi.CreateContainerHook, d.nvidiaCTKPath,
Path: hookPath, "create-symlinks",
Args: args, args...,
} )
return []Hook{h}, nil return []Hook{hook}, nil
} }
// getSpecificLinkArgs returns the required specic links that need to be created // getSpecificLinkArgs returns the required specic links that need to be created

63
internal/discover/hooks.go Normal file
View File

@ -0,0 +1,63 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package discover
import (
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
"github.com/sirupsen/logrus"
)
const (
nvidiaCTKExecutable = "nvidia-ctk"
nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
)
// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
func CreateNvidiaCTKHook(executable string, hookName string, additionalArgs ...string) Hook {
return Hook{
Lifecycle: cdi.CreateContainerHook,
Path: executable,
Args: append([]string{filepath.Base(executable), "hook", hookName}, additionalArgs...),
}
}
// FindNvidiaCTK locates the nvidia-ctk executable to be used in hooks.
// If an override is specified, this is used instead.
func FindNvidiaCTK(logger *logrus.Logger, override string) string {
if override != "" {
logger.Debugf("Using specified NVIDIA Container Toolkit CLI path %v", override)
return override
}
lookup := lookup.NewExecutableLocator(logger, "")
hookPath := nvidiaCTKDefaultFilePath
targets, err := lookup.Locate(nvidiaCTKExecutable)
if err != nil {
logger.Warnf("Failed to locate %v: %v", nvidiaCTKExecutable, err)
} else if len(targets) == 0 {
logger.Warnf("%v not found", nvidiaCTKExecutable)
} else {
logger.Debugf("Found %v candidates: %v", nvidiaCTKExecutable, targets)
hookPath = targets[0]
}
logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
return hookPath
}

52
internal/discover/ldconfig.go
View File

@ -22,32 +22,27 @@ import (
"strings" "strings"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified // NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) { func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
d := ldconfig{ d := ldconfig{
logger: logger, logger: logger,
mountsFrom: mounts, mountsFrom: mounts,
lookup: lookup.NewExecutableLocator(logger, cfg.Root), lookup: lookup.NewExecutableLocator(logger, cfg.Root),
nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath, nvidiaCTKPath: cfg.NvidiaCTKPath,
} }
return &d, nil return &d, nil
} }
const (
nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
)
type ldconfig struct { type ldconfig struct {
None None
logger *logrus.Logger logger *logrus.Logger
mountsFrom Discover mountsFrom Discover
lookup lookup.Locator lookup lookup.Locator
nvidiaCTKExecutablePath string nvidiaCTKPath string
} }
// Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths. // Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
@ -57,27 +52,21 @@ func (d ldconfig) Hooks() ([]Hook, error) {
return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err) return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
} }
h := CreateLDCacheUpdateHook( h := CreateLDCacheUpdateHook(
d.logger, d.nvidiaCTKPath,
d.lookup,
d.nvidiaCTKExecutablePath,
nvidiaCTKDefaultFilePath,
getLibraryPaths(mounts), getLibraryPaths(mounts),
) )
return []Hook{h}, nil return []Hook{h}, nil
} }
// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache // CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
func CreateLDCacheUpdateHook(logger *logrus.Logger, lookup lookup.Locator, executable string, defaultPath string, libraries []string) Hook { func CreateLDCacheUpdateHook(executable string, libraries []string) Hook {
var args []string var args []string
for _, f := range uniqueFolders(libraries) { for _, f := range uniqueFolders(libraries) {
args = append(args, "--folder", f) args = append(args, "--folder", f)
} }
hook := CreateNvidiaCTKHook( hook := CreateNvidiaCTKHook(
logger,
lookup,
executable, executable,
defaultPath,
"update-ldcache", "update-ldcache",
args..., args...,
) )
@ -86,27 +75,6 @@ func CreateLDCacheUpdateHook(logger *logrus.Logger, lookup lookup.Locator, execu
} }
// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
func CreateNvidiaCTKHook(logger *logrus.Logger, lookup lookup.Locator, executable string, defaultPath string, hookName string, additionalArgs ...string) Hook {
hookPath := defaultPath
targets, err := lookup.Locate(executable)
if err != nil {
logger.Warnf("Failed to locate %v: %v", executable, err)
} else if len(targets) == 0 {
logger.Warnf("%v not found", executable)
} else {
logger.Debugf("Found %v candidates: %v", executable, targets)
hookPath = targets[0]
}
logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
return Hook{
Lifecycle: cdi.CreateContainerHook,
Path: hookPath,
Args: append([]string{filepath.Base(hookPath), "hook", hookName}, additionalArgs...),
}
}
// getLibraryPaths extracts the library dirs from the specified mounts // getLibraryPaths extracts the library dirs from the specified mounts
func getLibraryPaths(mounts []Mount) []string { func getLibraryPaths(mounts []Mount) []string {
var paths []string var paths []string

10
internal/discover/ldconfig_test.go
View File

@ -24,12 +24,16 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
const (
testNvidiaCTKPath = "/foo/bar/nvidia-ctk"
)
func TestLDCacheUpdateHook(t *testing.T) { func TestLDCacheUpdateHook(t *testing.T) {
logger, _ := testlog.NewNullLogger() logger, _ := testlog.NewNullLogger()
cfg := Config{ cfg := Config{
Root: "/", Root: "/",
NVIDIAContainerToolkitCLIExecutablePath: "/foo/bar/nvidia-ctk", NvidiaCTKPath: testNvidiaCTKPath,
} }
testCases := []struct { testCases := []struct {
@ -86,7 +90,7 @@ func TestLDCacheUpdateHook(t *testing.T) {
}, },
} }
expectedHook := Hook{ expectedHook := Hook{
Path: "/usr/bin/nvidia-ctk", Path: testNvidiaCTKPath,
Args: tc.expectedArgs, Args: tc.expectedArgs,
Lifecycle: "createContainer", Lifecycle: "createContainer",
} }

46
internal/discover/symlinks.go
View File

@ -21,28 +21,24 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
type symlinks struct { type symlinks struct {
None None
logger *logrus.Logger logger *logrus.Logger
lookup lookup.Locator nvidiaCTKPath string
nvidiaCTKExecutablePath string csvFiles []string
csvFiles []string mountsFrom Discover
mountsFrom Discover
} }
// NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container // NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container
func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) { func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) {
d := symlinks{ d := symlinks{
logger: logger, logger: logger,
lookup: lookup.NewExecutableLocator(logger, cfg.Root), nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath, csvFiles: csvFiles,
csvFiles: csvFiles, mountsFrom: mounts,
mountsFrom: mounts,
} }
return &d, nil return &d, nil
@ -50,19 +46,7 @@ func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Disc
// Hooks returns a hook to create the symlinks from the required CSV files // Hooks returns a hook to create the symlinks from the required CSV files
func (d symlinks) Hooks() ([]Hook, error) { func (d symlinks) Hooks() ([]Hook, error) {
hookPath := nvidiaCTKDefaultFilePath var args []string
targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
if err != nil {
d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
} else if len(targets) == 0 {
d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
} else {
d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
hookPath = targets[0]
}
d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
args := []string{hookPath, "hook", "create-symlinks"}
for _, f := range d.csvFiles { for _, f := range d.csvFiles {
args = append(args, "--csv-filename", f) args = append(args, "--csv-filename", f)
} }
@ -73,13 +57,13 @@ func (d symlinks) Hooks() ([]Hook, error) {
} }
args = append(args, links...) args = append(args, links...)
h := Hook{ hook := CreateNvidiaCTKHook(
Lifecycle: cdi.CreateContainerHook, d.nvidiaCTKPath,
Path: hookPath, "create-symlinks",
Args: args, args...,
} )
return []Hook{h}, nil return []Hook{hook}, nil
} }
// getSpecificLinkArgs returns the required specic links that need to be created // getSpecificLinkArgs returns the required specic links that need to be created

4
internal/modifier/csv.go
View File

@ -62,8 +62,8 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
logger.Infof("Constructing modifier from config: %+v", *cfg) logger.Infof("Constructing modifier from config: %+v", *cfg)
config := &discover.Config{ config := &discover.Config{
Root: cfg.NVIDIAContainerCLIConfig.Root, Root: cfg.NVIDIAContainerCLIConfig.Root,
NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path, NvidiaCTKPath: cfg.NVIDIACTKConfig.Path,
} }
if err := checkRequirements(logger, image); err != nil { if err := checkRequirements(logger, image); err != nil {

4
internal/modifier/graphics.go
View File

@ -45,8 +45,8 @@ func NewGraphicsModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.
} }
config := &discover.Config{ config := &discover.Config{
Root: cfg.NVIDIAContainerCLIConfig.Root, Root: cfg.NVIDIAContainerCLIConfig.Root,
NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path, NvidiaCTKPath: cfg.NVIDIACTKConfig.Path,
} }
d, err := discover.NewGraphicsDiscoverer( d, err := discover.NewGraphicsDiscoverer(
logger, logger,

3
test/release/Makefile
View File

@ -14,7 +14,7 @@
WORKFLOW ?= nvidia-docker WORKFLOW ?= nvidia-docker
DISTRIBUTIONS := ubuntu18.04 centos8 fedora35 DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 centos8 fedora35
IMAGE_TARGETS := $(patsubst %,image-%, $(DISTRIBUTIONS)) IMAGE_TARGETS := $(patsubst %,image-%, $(DISTRIBUTIONS))
RUN_TARGETS := $(patsubst %,run-%, $(DISTRIBUTIONS)) RUN_TARGETS := $(patsubst %,run-%, $(DISTRIBUTIONS))
@ -33,6 +33,7 @@ $(IMAGE_TARGETS): image-%: $(DOCKERFILE)
$(shell dirname $(DOCKERFILE)) $(shell dirname $(DOCKERFILE))
%-ubuntu20.04: ARCH ?= amd64
%-ubuntu18.04: ARCH ?= amd64 %-ubuntu18.04: ARCH ?= amd64
%-centos8: ARCH ?= x86_64 %-centos8: ARCH ?= x86_64
%-fedora35: ARCH ?= x86_64 %-fedora35: ARCH ?= x86_64