UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-06-12 09:31:17 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Rename root to driverRoot for CDI generation

Browse Source 
This makes the intent of the command line argument clearer since this
relates specifically to the root where the NVIDIA driver is installed.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar 2023-02-02 15:42:01 +01:00
parent 03cdf3b5d7
commit cfa2647260
7 changed files with 50 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/nvidia-ctk/cdi/generate/common.go
View File

@ -27,14 +27,14 @@ import (
// NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device. // NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
// This includes driver libraries and meta devices, for example. // This includes driver libraries and meta devices, for example.
func NewCommonDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) { func NewCommonDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
metaDevices := discover.NewDeviceDiscoverer( metaDevices := discover.NewDeviceDiscoverer(
logger, logger,
lookup.NewCharDeviceLocator( lookup.NewCharDeviceLocator(
lookup.WithLogger(logger), lookup.WithLogger(logger),
lookup.WithRoot(root), lookup.WithRoot(driverRoot),
), ),
root, driverRoot,
[]string{ []string{
"/dev/nvidia-modeset", "/dev/nvidia-modeset",
"/dev/nvidia-uvm-tools", "/dev/nvidia-uvm-tools",
@ -43,12 +43,12 @@ func NewCommonDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath strin
}, },
) )
graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(logger, root) graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(logger, driverRoot)
if err != nil { if err != nil {
return nil, fmt.Errorf("error constructing discoverer for graphics mounts: %v", err) return nil, fmt.Errorf("error constructing discoverer for graphics mounts: %v", err)
} }
driverFiles, err := NewDriverDiscoverer(logger, root, nvidiaCTKPath, nvmllib) driverFiles, err := NewDriverDiscoverer(logger, driverRoot, nvidiaCTKPath, nvmllib)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err) return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err)
} }

6
cmd/nvidia-ctk/cdi/generate/device-folder-permissions.go
View File

@ -26,7 +26,7 @@ import (
type deviceFolderPermissions struct { type deviceFolderPermissions struct {
logger *logrus.Logger logger *logrus.Logger
root string driverRoot string
nvidiaCTKPath string nvidiaCTKPath string
folders []string folders []string
} }
@ -39,7 +39,7 @@ var _ discover.Discover = (*deviceFolderPermissions)(nil)
// The nested devices that are applicable to the NVIDIA GPU devices are: // The nested devices that are applicable to the NVIDIA GPU devices are:
// - DRM devices at /dev/dri/* // - DRM devices at /dev/dri/*
// - NVIDIA Caps devices at /dev/nvidia-caps/* // - NVIDIA Caps devices at /dev/nvidia-caps/*
func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, deviceSpecs []specs.Device) (discover.Discover, error) { func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, deviceSpecs []specs.Device) (discover.Discover, error) {
var folders []string var folders []string
seen := make(map[string]bool) seen := make(map[string]bool)
for _, device := range deviceSpecs { for _, device := range deviceSpecs {
@ -66,7 +66,7 @@ func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string,
d := &deviceFolderPermissions{ d := &deviceFolderPermissions{
logger: logger, logger: logger,
root: root, driverRoot: driverRoot,
nvidiaCTKPath: nvidiaCTKPath, nvidiaCTKPath: nvidiaCTKPath,
folders: folders, folders: folders,
} }

40
cmd/nvidia-ctk/cdi/generate/driver.go
View File

@ -30,20 +30,20 @@ import (
// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation. // NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
// The supplied NVML Library is used to query the expected driver version. // The supplied NVML Library is used to query the expected driver version.
func NewDriverDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) { func NewDriverDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
version, r := nvmllib.SystemGetDriverVersion() version, r := nvmllib.SystemGetDriverVersion()
if r != nvml.SUCCESS { if r != nvml.SUCCESS {
return nil, fmt.Errorf("failed to determine driver version: %v", r) return nil, fmt.Errorf("failed to determine driver version: %v", r)
} }
libraries, err := NewDriverLibraryDiscoverer(logger, root, nvidiaCTKPath, version) libraries, err := NewDriverLibraryDiscoverer(logger, driverRoot, nvidiaCTKPath, version)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err) return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
} }
firmwares := NewDriverFirmwareDiscoverer(logger, root, version) firmwares := NewDriverFirmwareDiscoverer(logger, driverRoot, version)
binaries := NewDriverBinariesDiscoverer(logger, root) binaries := NewDriverBinariesDiscoverer(logger, driverRoot)
d := discover.Merge( d := discover.Merge(
libraries, libraries,
@ -55,8 +55,8 @@ func NewDriverDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath strin
} }
// NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version. // NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version.
func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, version string) (discover.Discover, error) { func NewDriverLibraryDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, version string) (discover.Discover, error) {
libraryPaths, err := getVersionLibs(logger, root, version) libraryPaths, err := getVersionLibs(logger, driverRoot, version)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get libraries for driver version: %v", err) return nil, fmt.Errorf("failed to get libraries for driver version: %v", err)
} }
@ -65,14 +65,14 @@ func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPat
logger, logger,
lookup.NewFileLocator( lookup.NewFileLocator(
lookup.WithLogger(logger), lookup.WithLogger(logger),
lookup.WithRoot(root), lookup.WithRoot(driverRoot),
), ),
root, driverRoot,
libraryPaths, libraryPaths,
) )
cfg := &discover.Config{ cfg := &discover.Config{
Root: root, Root: driverRoot,
NvidiaCTKPath: nvidiaCTKPath, NvidiaCTKPath: nvidiaCTKPath,
} }
hooks, _ := discover.NewLDCacheUpdateHook(logger, libraries, cfg) hooks, _ := discover.NewLDCacheUpdateHook(logger, libraries, cfg)
@ -86,25 +86,25 @@ func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPat
} }
// NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version. // NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version.
func NewDriverFirmwareDiscoverer(logger *logrus.Logger, root string, version string) discover.Discover { func NewDriverFirmwareDiscoverer(logger *logrus.Logger, driverRoot string, version string) discover.Discover {
gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp.bin") gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp.bin")
return discover.NewMounts( return discover.NewMounts(
logger, logger,
lookup.NewFileLocator( lookup.NewFileLocator(
lookup.WithLogger(logger), lookup.WithLogger(logger),
lookup.WithRoot(root), lookup.WithRoot(driverRoot),
), ),
root, driverRoot,
[]string{gspFirmwarePath}, []string{gspFirmwarePath},
) )
} }
// NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver. // NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver.
func NewDriverBinariesDiscoverer(logger *logrus.Logger, root string) discover.Discover { func NewDriverBinariesDiscoverer(logger *logrus.Logger, driverRoot string) discover.Discover {
return discover.NewMounts( return discover.NewMounts(
logger, logger,
lookup.NewExecutableLocator(logger, root), lookup.NewExecutableLocator(logger, driverRoot),
root, driverRoot,
[]string{ []string{
"nvidia-smi", /* System management interface */ "nvidia-smi", /* System management interface */
"nvidia-debugdump", /* GPU coredump utility */ "nvidia-debugdump", /* GPU coredump utility */
@ -116,12 +116,12 @@ func NewDriverBinariesDiscoverer(logger *logrus.Logger, root string) discover.Di
} }
// getVersionLibs checks the LDCache for libraries ending in the specified driver version. // getVersionLibs checks the LDCache for libraries ending in the specified driver version.
// Although the ldcache at the specified root is queried, the paths are returned relative to this root. // Although the ldcache at the specified driverRoot is queried, the paths are returned relative to this driverRoot.
// This allows the standard mount location logic to be used for resolving the mounts. // This allows the standard mount location logic to be used for resolving the mounts.
func getVersionLibs(logger *logrus.Logger, root string, version string) ([]string, error) { func getVersionLibs(logger *logrus.Logger, driverRoot string, version string) ([]string, error) {
logger.Infof("Using driver version %v", version) logger.Infof("Using driver version %v", version)
cache, err := ldcache.New(logger, root) cache, err := ldcache.New(logger, driverRoot)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to load ldcache: %v", err) return nil, fmt.Errorf("failed to load ldcache: %v", err)
} }
@ -143,13 +143,13 @@ func getVersionLibs(logger *logrus.Logger, root string, version string) ([]strin
} }
} }
if root == "/" || root == "" { if driverRoot == "/" || driverRoot == "" {
return libs, nil return libs, nil
} }
var relative []string var relative []string
for _, l := range libs { for _, l := range libs {
relative = append(relative, strings.TrimPrefix(l, root)) relative = append(relative, strings.TrimPrefix(l, driverRoot))
} }
return relative, nil return relative, nil

10
cmd/nvidia-ctk/cdi/generate/full-gpu.go
View File

@ -32,7 +32,7 @@ import (
// byPathHookDiscoverer discovers the entities required for injecting by-path DRM device links // byPathHookDiscoverer discovers the entities required for injecting by-path DRM device links
type byPathHookDiscoverer struct { type byPathHookDiscoverer struct {
logger *logrus.Logger logger *logrus.Logger
root string driverRoot string
nvidiaCTKPath string nvidiaCTKPath string
pciBusID string pciBusID string
} }
@ -40,7 +40,7 @@ type byPathHookDiscoverer struct {
var _ discover.Discover = (*byPathHookDiscoverer)(nil) var _ discover.Discover = (*byPathHookDiscoverer)(nil)
// NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device. // NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device.
func NewFullGPUDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, d device.Device) (discover.Discover, error) { func NewFullGPUDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, d device.Device) (discover.Discover, error) {
// TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface. // TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface.
// This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin. // This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin.
minor, ret := d.GetMinorNumber() minor, ret := d.GetMinorNumber()
@ -65,12 +65,12 @@ func NewFullGPUDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath stri
deviceNodes := discover.NewCharDeviceDiscoverer( deviceNodes := discover.NewCharDeviceDiscoverer(
logger, logger,
deviceNodePaths, deviceNodePaths,
root, driverRoot,
) )
byPathHooks := &byPathHookDiscoverer{ byPathHooks := &byPathHookDiscoverer{
logger: logger, logger: logger,
root: root, driverRoot: driverRoot,
nvidiaCTKPath: nvidiaCTKPath, nvidiaCTKPath: nvidiaCTKPath,
pciBusID: pciBusID, pciBusID: pciBusID,
} }
@ -127,7 +127,7 @@ func (d *byPathHookDiscoverer) deviceNodeLinks() ([]string, error) {
var links []string var links []string
for _, c := range candidates { for _, c := range candidates {
linkPath := filepath.Join(d.root, c) linkPath := filepath.Join(d.driverRoot, c)
device, err := os.Readlink(linkPath) device, err := os.Readlink(linkPath)
if err != nil { if err != nil {
d.logger.Warningf("Failed to evaluate symlink %v; ignoring", linkPath) d.logger.Warningf("Failed to evaluate symlink %v; ignoring", linkPath)

24
cmd/nvidia-ctk/cdi/generate/generate.go
View File

@ -47,7 +47,7 @@ type config struct {
output string output string
format string format string
deviceNameStrategy string deviceNameStrategy string
root string driverRoot string
nvidiaCTKPath string nvidiaCTKPath string
} }
@ -94,9 +94,9 @@ func (m command) build() *cli.Command {
Destination: &cfg.deviceNameStrategy, Destination: &cfg.deviceNameStrategy,
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "root", Name: "driver-root",
Usage: "Specify the root to use when discovering the entities that should be included in the CDI specification.", Usage: "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
Destination: &cfg.root, Destination: &cfg.driverRoot,
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "nvidia-ctk-path", Name: "nvidia-ctk-path",
@ -132,7 +132,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
} }
spec, err := m.generateSpec( spec, err := m.generateSpec(
cfg.root, cfg.driverRoot,
discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath), discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath),
deviceNamer, deviceNamer,
) )
@ -214,7 +214,7 @@ func writeToOutput(format string, data []byte, output io.Writer) error {
return nil return nil
} }
func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNamer) (*specs.Spec, error) { func (m command) generateSpec(driverRoot string, nvidiaCTKPath string, namer deviceNamer) (*specs.Spec, error) {
nvmllib := nvml.New() nvmllib := nvml.New()
if r := nvmllib.Init(); r != nvml.SUCCESS { if r := nvmllib.Init(); r != nvml.SUCCESS {
return nil, r return nil, r
@ -223,7 +223,7 @@ func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNam
devicelib := device.New(device.WithNvml(nvmllib)) devicelib := device.New(device.WithNvml(nvmllib))
deviceSpecs, err := m.generateDeviceSpecs(devicelib, root, nvidiaCTKPath, namer) deviceSpecs, err := m.generateDeviceSpecs(devicelib, driverRoot, nvidiaCTKPath, namer)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create device CDI specs: %v", err) return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
} }
@ -234,7 +234,7 @@ func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNam
allEdits := edits.NewContainerEdits() allEdits := edits.NewContainerEdits()
ipcs, err := NewIPCDiscoverer(m.logger, root) ipcs, err := NewIPCDiscoverer(m.logger, driverRoot)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err) return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
} }
@ -250,12 +250,12 @@ func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNam
allEdits.Append(ipcEdits) allEdits.Append(ipcEdits)
common, err := NewCommonDiscoverer(m.logger, root, nvidiaCTKPath, nvmllib) common, err := NewCommonDiscoverer(m.logger, driverRoot, nvidiaCTKPath, nvmllib)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err) return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
} }
deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, root, nvidiaCTKPath, deviceSpecs) deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, driverRoot, nvidiaCTKPath, deviceSpecs)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err) return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
} }
@ -286,7 +286,7 @@ func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNam
return &spec, nil return &spec, nil
} }
func (m command) generateDeviceSpecs(devicelib device.Interface, root string, nvidiaCTKPath string, namer deviceNamer) ([]specs.Device, error) { func (m command) generateDeviceSpecs(devicelib device.Interface, driverRoot string, nvidiaCTKPath string, namer deviceNamer) ([]specs.Device, error) {
var deviceSpecs []specs.Device var deviceSpecs []specs.Device
err := devicelib.VisitDevices(func(i int, d device.Device) error { err := devicelib.VisitDevices(func(i int, d device.Device) error {
@ -297,7 +297,7 @@ func (m command) generateDeviceSpecs(devicelib device.Interface, root string, nv
if isMigEnabled { if isMigEnabled {
return nil return nil
} }
device, err := NewFullGPUDiscoverer(m.logger, root, nvidiaCTKPath, d) device, err := NewFullGPUDiscoverer(m.logger, driverRoot, nvidiaCTKPath, d)
if err != nil { if err != nil {
return fmt.Errorf("failed to create device: %v", err) return fmt.Errorf("failed to create device: %v", err)
} }

6
cmd/nvidia-ctk/cdi/generate/ipc.go
View File

@ -23,14 +23,14 @@ import (
) )
// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets. // NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
func NewIPCDiscoverer(logger *logrus.Logger, root string) (discover.Discover, error) { func NewIPCDiscoverer(logger *logrus.Logger, driverRoot string) (discover.Discover, error) {
d := discover.NewMounts( d := discover.NewMounts(
logger, logger,
lookup.NewFileLocator( lookup.NewFileLocator(
lookup.WithLogger(logger), lookup.WithLogger(logger),
lookup.WithRoot(root), lookup.WithRoot(driverRoot),
), ),
root, driverRoot,
[]string{ []string{
"/var/run/nvidia-persistenced/socket", "/var/run/nvidia-persistenced/socket",
"/var/run/nvidia-fabricmanager/socket", "/var/run/nvidia-fabricmanager/socket",

4
cmd/nvidia-ctk/cdi/generate/mig-device.go
View File

@ -27,7 +27,7 @@ import (
) )
// NewMigDeviceDiscoverer creates a discoverer for the specified mig device and its parent. // NewMigDeviceDiscoverer creates a discoverer for the specified mig device and its parent.
func NewMigDeviceDiscoverer(logger *logrus.Logger, root string, parent device.Device, d device.MigDevice) (discover.Discover, error) { func NewMigDeviceDiscoverer(logger *logrus.Logger, driverRoot string, parent device.Device, d device.MigDevice) (discover.Discover, error) {
minor, ret := parent.GetMinorNumber() minor, ret := parent.GetMinorNumber()
if ret != nvml.SUCCESS { if ret != nvml.SUCCESS {
return nil, fmt.Errorf("error getting GPU device minor number: %v", ret) return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
@ -68,7 +68,7 @@ func NewMigDeviceDiscoverer(logger *logrus.Logger, root string, parent device.De
giCapDevicePath, giCapDevicePath,
ciCapDevicePath, ciCapDevicePath,
}, },
root, driverRoot,
) )
return deviceNodes, nil return deviceNodes, nil