Refactor the way we create CDI Hooks

Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>
This commit is contained in:
Carlos Eduardo Arango Gutierrez
2025-05-16 18:44:59 +02:00
parent ac8f190c99
commit a3ca7fd98c
30 changed files with 201 additions and 210 deletions

View File

@@ -36,7 +36,7 @@ func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) {
},
)
graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driver, l.nvidiaCDIHookPath)
graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driver, l.hookCreator)
if err != nil {
l.logger.Warningf("failed to create discoverer for graphics mounts: %v", err)
}

View File

@@ -102,17 +102,17 @@ func (l *nvcdilib) NewDriverLibraryDiscoverer(version string) (discover.Discover
driverDotSoSymlinksDiscoverer := discover.WithDriverDotSoSymlinks(
libraries,
version,
l.nvidiaCDIHookPath,
l.hookCreator,
)
discoverers = append(discoverers, driverDotSoSymlinksDiscoverer)
if l.HookIsSupported(HookEnableCudaCompat) {
// TODO: The following should use the version directly.
cudaCompatLibHookDiscoverer := discover.NewCUDACompatHookDiscoverer(l.logger, l.nvidiaCDIHookPath, l.driver)
cudaCompatLibHookDiscoverer := discover.NewCUDACompatHookDiscoverer(l.logger, l.hookCreator, l.driver)
discoverers = append(discoverers, cudaCompatLibHookDiscoverer)
}
updateLDCache, _ := discover.NewLDCacheUpdateHook(l.logger, libraries, l.nvidiaCDIHookPath, l.ldconfigPath)
updateLDCache, _ := discover.NewLDCacheUpdateHook(l.logger, libraries, l.hookCreator, l.ldconfigPath)
discoverers = append(discoverers, updateLDCache)
d := discover.Merge(discoverers...)

View File

@@ -39,7 +39,7 @@ var requiredDriverStoreFiles = []string{
}
// newWSLDriverDiscoverer returns a Discoverer for WSL2 drivers.
func newWSLDriverDiscoverer(logger logger.Interface, driverRoot string, nvidiaCDIHookPath, ldconfigPath string) (discover.Discover, error) {
func newWSLDriverDiscoverer(logger logger.Interface, driverRoot string, hookCreator discover.HookCreator, ldconfigPath string) (discover.Discover, error) {
err := dxcore.Init()
if err != nil {
return nil, fmt.Errorf("failed to initialize dxcore: %v", err)
@@ -56,11 +56,11 @@ func newWSLDriverDiscoverer(logger logger.Interface, driverRoot string, nvidiaCD
}
logger.Infof("Using WSL driver store paths: %v", driverStorePaths)
return newWSLDriverStoreDiscoverer(logger, driverRoot, nvidiaCDIHookPath, ldconfigPath, driverStorePaths)
return newWSLDriverStoreDiscoverer(logger, driverRoot, hookCreator, ldconfigPath, driverStorePaths)
}
// newWSLDriverStoreDiscoverer returns a Discoverer for WSL2 drivers in the driver store associated with a dxcore adapter.
func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, nvidiaCDIHookPath string, ldconfigPath string, driverStorePaths []string) (discover.Discover, error) {
func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, hookCreator discover.HookCreator, ldconfigPath string, driverStorePaths []string) (discover.Discover, error) {
var searchPaths []string
seen := make(map[string]bool)
for _, path := range driverStorePaths {
@@ -88,12 +88,12 @@ func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, nvi
)
symlinkHook := nvidiaSMISimlinkHook{
logger: logger,
mountsFrom: libraries,
nvidiaCDIHookPath: nvidiaCDIHookPath,
logger: logger,
mountsFrom: libraries,
hookCreator: hookCreator,
}
ldcacheHook, _ := discover.NewLDCacheUpdateHook(logger, libraries, nvidiaCDIHookPath, ldconfigPath)
ldcacheHook, _ := discover.NewLDCacheUpdateHook(logger, libraries, hookCreator, ldconfigPath)
d := discover.Merge(
libraries,
@@ -106,9 +106,9 @@ func newWSLDriverStoreDiscoverer(logger logger.Interface, driverRoot string, nvi
type nvidiaSMISimlinkHook struct {
discover.None
logger logger.Interface
mountsFrom discover.Discover
nvidiaCDIHookPath string
logger logger.Interface
mountsFrom discover.Discover
hookCreator discover.HookCreator
}
// Hooks returns a hook that creates a symlink to nvidia-smi in the driver store.
@@ -135,7 +135,7 @@ func (m nvidiaSMISimlinkHook) Hooks() ([]discover.Hook, error) {
}
link := "/usr/bin/nvidia-smi"
links := []string{fmt.Sprintf("%s::%s", target, link)}
symlinkHook := discover.CreateCreateSymlinkHook(m.nvidiaCDIHookPath, links)
symlinkHook := m.hookCreator.Create("create-symlinks", links...)
return symlinkHook.Hooks()
}

View File

@@ -29,6 +29,7 @@ import (
func TestNvidiaSMISymlinkHook(t *testing.T) {
logger, _ := testlog.NewNullLogger()
hookCreator := discover.NewHookCreator("nvidia-cdi-hook")
errMounts := errors.New("mounts error")
@@ -143,9 +144,9 @@ func TestNvidiaSMISymlinkHook(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
m := nvidiaSMISimlinkHook{
logger: logger,
mountsFrom: tc.mounts,
nvidiaCDIHookPath: "nvidia-cdi-hook",
logger: logger,
mountsFrom: tc.mounts,
hookCreator: hookCreator,
}
devices, err := m.Devices()

View File

@@ -71,7 +71,7 @@ func (l *nvmllib) newFullGPUDiscoverer(d device.Device) (discover.Discover, erro
deviceNodes, err := dgpu.NewForDevice(d,
dgpu.WithDevRoot(l.devRoot),
dgpu.WithLogger(l.logger),
dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath),
dgpu.WithHookCreator(l.hookCreator),
dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib),
)
if err != nil {
@@ -81,7 +81,7 @@ func (l *nvmllib) newFullGPUDiscoverer(d device.Device) (discover.Discover, erro
deviceFolderPermissionHooks := newDeviceFolderPermissionHookDiscoverer(
l.logger,
l.devRoot,
l.nvidiaCDIHookPath,
l.hookCreator,
deviceNodes,
)

View File

@@ -44,7 +44,7 @@ func (l *csvlib) GetAllDeviceSpecs() ([]specs.Device, error) {
tegra.WithLogger(l.logger),
tegra.WithDriverRoot(l.driverRoot),
tegra.WithDevRoot(l.devRoot),
tegra.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath),
tegra.WithHookCreator(l.hookCreator),
tegra.WithLdconfigPath(l.ldconfigPath),
tegra.WithCSVFiles(l.csvFiles),
tegra.WithLibrarySearchPaths(l.librarySearchPaths...),

View File

@@ -54,7 +54,7 @@ func (l *wsllib) GetAllDeviceSpecs() ([]specs.Device, error) {
// GetCommonEdits generates a CDI specification that can be used for ANY devices
func (l *wsllib) GetCommonEdits() (*cdi.ContainerEdits, error) {
driver, err := newWSLDriverDiscoverer(l.logger, l.driverRoot, l.nvidiaCDIHookPath, l.ldconfigPath)
driver, err := newWSLDriverDiscoverer(l.logger, l.driverRoot, l.hookCreator, l.ldconfigPath)
if err != nil {
return nil, fmt.Errorf("failed to create discoverer for WSL driver: %v", err)
}

View File

@@ -23,6 +23,7 @@ import (
"github.com/NVIDIA/go-nvlib/pkg/nvlib/info"
"github.com/NVIDIA/go-nvml/pkg/nvml"
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root"
"github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils"
@@ -56,6 +57,7 @@ type nvcdilib struct {
mergedDeviceOptions []transform.MergedDeviceOption
disabledHooks disabledHooks
hookCreator discover.HookCreator
}
// New creates a new nvcdi library
@@ -79,6 +81,9 @@ func New(opts ...Option) (Interface, error) {
if l.nvidiaCDIHookPath == "" {
l.nvidiaCDIHookPath = "/usr/bin/nvidia-cdi-hook"
}
// create hookCreator
l.hookCreator = discover.NewHookCreator(l.nvidiaCDIHookPath)
if l.driverRoot == "" {
l.driverRoot = "/"
}

View File

@@ -138,7 +138,7 @@ func (m *managementlib) newManagementDeviceDiscoverer() (discover.Discover, erro
deviceFolderPermissionHooks := newDeviceFolderPermissionHookDiscoverer(
m.logger,
m.devRoot,
m.nvidiaCDIHookPath,
m.hookCreator,
deviceNodes,
)

View File

@@ -54,7 +54,7 @@ func (l *nvmllib) GetMIGDeviceEdits(parent device.Device, mig device.MigDevice)
deviceNodes, err := dgpu.NewForMigDevice(parent, mig,
dgpu.WithDevRoot(l.devRoot),
dgpu.WithLogger(l.logger),
dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath),
dgpu.WithHookCreator(l.hookCreator),
dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib),
)
if err != nil {

View File

@@ -25,10 +25,10 @@ import (
)
type deviceFolderPermissions struct {
logger logger.Interface
devRoot string
nvidiaCDIHookPath string
devices discover.Discover
logger logger.Interface
devRoot string
devices discover.Discover
hookCreator discover.HookCreator
}
var _ discover.Discover = (*deviceFolderPermissions)(nil)
@@ -39,12 +39,12 @@ var _ discover.Discover = (*deviceFolderPermissions)(nil)
// The nested devices that are applicable to the NVIDIA GPU devices are:
// - DRM devices at /dev/dri/*
// - NVIDIA Caps devices at /dev/nvidia-caps/*
func newDeviceFolderPermissionHookDiscoverer(logger logger.Interface, devRoot string, nvidiaCDIHookPath string, devices discover.Discover) discover.Discover {
func newDeviceFolderPermissionHookDiscoverer(logger logger.Interface, devRoot string, hookCreator discover.HookCreator, devices discover.Discover) discover.Discover {
d := &deviceFolderPermissions{
logger: logger,
devRoot: devRoot,
nvidiaCDIHookPath: nvidiaCDIHookPath,
devices: devices,
logger: logger,
devRoot: devRoot,
hookCreator: hookCreator,
devices: devices,
}
return d
@@ -70,13 +70,9 @@ func (d *deviceFolderPermissions) Hooks() ([]discover.Hook, error) {
args = append(args, "--path", folder)
}
hook := discover.CreateNvidiaCDIHook(
d.nvidiaCDIHookPath,
"chmod",
args...,
)
hook := d.hookCreator.Create("chmod", args...)
return []discover.Hook{hook}, nil
return []discover.Hook{*hook}, nil
}
func (d *deviceFolderPermissions) getDeviceSubfolders() ([]string, error) {