UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-06-26 18:18:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add driver root abstraction

Browse Source 
This change adds a driver root abstraction that defines how
libraries are located relative to the root. This allows for
this driver root to be constructed once and passed to discovery
code.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
2023-11-21 16:08:16 +01:00
parent f20ab793a2
commit bbd9222206
9 changed files with 119 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/nvcdi/common-nvml.go
View File

@@ -36,12 +36,12 @@ func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) {
},
)
graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath)
graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driver, l.nvidiaCTKPath)
if err != nil {
l.logger.Warningf("failed to create discoverer for graphics mounts: %v", err)
}
driverFiles, err := NewDriverDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath, l.nvmllib)
driverFiles, err := NewDriverDiscoverer(l.logger, l.driver, l.nvidiaCTKPath, l.nvmllib)
if err != nil {
return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err)
}

32
pkg/nvcdi/driver-nvml.go
View File

@@ -27,12 +27,13 @@ import (
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root"
"golang.org/x/sys/unix"
)
// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
// The supplied NVML Library is used to query the expected driver version.
func NewDriverDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
func NewDriverDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
if r := nvmllib.Init(); r != nvml.SUCCESS {
return nil, fmt.Errorf("failed to initialize NVML: %v", r)
}
@@ -47,26 +48,26 @@ func NewDriverDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPa
return nil, fmt.Errorf("failed to determine driver version: %v", r)
}
return newDriverVersionDiscoverer(logger, driverRoot, nvidiaCTKPath, version)
return newDriverVersionDiscoverer(logger, driver, nvidiaCTKPath, version)
}
func newDriverVersionDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string, version string) (discover.Discover, error) {
libraries, err := NewDriverLibraryDiscoverer(logger, driverRoot, nvidiaCTKPath, version)
func newDriverVersionDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string, version string) (discover.Discover, error) {
libraries, err := NewDriverLibraryDiscoverer(logger, driver, nvidiaCTKPath, version)
if err != nil {
return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
}
ipcs, err := discover.NewIPCDiscoverer(logger, driverRoot)
ipcs, err := discover.NewIPCDiscoverer(logger, driver.Root)
if err != nil {
return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
}
firmwares, err := NewDriverFirmwareDiscoverer(logger, driverRoot, version)
firmwares, err := NewDriverFirmwareDiscoverer(logger, driver.Root, version)
if err != nil {
return nil, fmt.Errorf("failed to create discoverer for GSP firmware: %v", err)
}
binaries := NewDriverBinariesDiscoverer(logger, driverRoot)
binaries := NewDriverBinariesDiscoverer(logger, driver.Root)
d := discover.Merge(
libraries,
@@ -79,8 +80,8 @@ func newDriverVersionDiscoverer(logger logger.Interface, driverRoot string, nvid
}
// NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version.
func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string, version string) (discover.Discover, error) {
libraryPaths, err := getVersionLibs(logger, driverRoot, version)
func NewDriverLibraryDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string, version string) (discover.Discover, error) {
libraryPaths, err := getVersionLibs(logger, driver, version)
if err != nil {
return nil, fmt.Errorf("failed to get libraries for driver version: %v", err)
}
@@ -89,9 +90,9 @@ func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvid
logger,
lookup.NewFileLocator(
lookup.WithLogger(logger),
lookup.WithRoot(driverRoot),
lookup.WithRoot(driver.Root),
),
driverRoot,
driver.Root,
libraryPaths,
)
@@ -185,12 +186,11 @@ func NewDriverBinariesDiscoverer(logger logger.Interface, driverRoot string) dis
// getVersionLibs checks the LDCache for libraries ending in the specified driver version.
// Although the ldcache at the specified driverRoot is queried, the paths are returned relative to this driverRoot.
// This allows the standard mount location logic to be used for resolving the mounts.
func getVersionLibs(logger logger.Interface, driverRoot string, version string) ([]string, error) {
func getVersionLibs(logger logger.Interface, driver *root.Driver, version string) ([]string, error) {
logger.Infof("Using driver version %v", version)
libCudaPaths, err := cuda.New(
cuda.WithLogger(logger),
cuda.WithDriverRoot(driverRoot),
driver.Libraries(),
).Locate("." + version)
if err != nil {
return nil, fmt.Errorf("failed to locate libcuda.so.%v: %v", version, err)
@@ -208,13 +208,13 @@ func getVersionLibs(logger logger.Interface, driverRoot string, version string)
return nil, fmt.Errorf("failed to locate libraries for driver version %v: %v", version, err)
}
if driverRoot == "/" || driverRoot == "" {
if driver.Root == "/" || driver.Root == "" {
return libs, nil
}
var relative []string
for _, l := range libs {
relative = append(relative, strings.TrimPrefix(l, driverRoot))
relative = append(relative, strings.TrimPrefix(l, driver.Root))
}
return relative, nil

5
pkg/nvcdi/lib.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/NVIDIA/go-nvlib/pkg/nvlib/info"
"github.com/NVIDIA/go-nvlib/pkg/nvml"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root"
"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
@@ -54,6 +55,7 @@ type nvcdilib struct {
vendor string
class string
driver *root.Driver
infolib info.Interface
mergedDeviceOptions []transform.MergedDeviceOption
@@ -87,6 +89,9 @@ func New(opts ...Option) (Interface, error) {
l.infolib = info.New()
}
// TODO: We need to improve the construction of this driver root.
l.driver = root.New(l.logger, l.driverRoot, l.librarySearchPaths)
var lib Interface
switch l.resolveMode() {
case ModeCSV:

5
pkg/nvcdi/management.go
View File

@@ -65,7 +65,7 @@ func (m *managementlib) GetCommonEdits() (*cdi.ContainerEdits, error) {
return nil, fmt.Errorf("failed to get CUDA version: %v", err)
}
driver, err := newDriverVersionDiscoverer(m.logger, m.driverRoot, m.nvidiaCTKPath, version)
driver, err := newDriverVersionDiscoverer(m.logger, m.driver, m.nvidiaCTKPath, version)
if err != nil {
return nil, fmt.Errorf("failed to create driver library discoverer: %v", err)
}
@@ -86,8 +86,7 @@ func (m *managementlib) getCudaVersion() (string, error) {
}
libCudaPaths, err := cuda.New(
cuda.WithLogger(m.logger),
cuda.WithDriverRoot(m.driverRoot),
m.driver.Libraries(),
).Locate(".*.*")
if err != nil {
return "", fmt.Errorf("failed to locate libcuda.so: %v", err)