From bbd92222063d2211e33c5393b97415f60d23c021 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 21 Nov 2023 16:08:16 +0100 Subject: [PATCH] Add driver root abstraction This change adds a driver root abstraction that defines how libraries are located relative to the root. This allows for this driver root to be constructed once and passed to discovery code. Signed-off-by: Evan Lezar --- internal/discover/graphics.go | 34 +++++++--------- internal/lookup/cuda/cuda.go | 41 ++----------------- internal/lookup/cuda/cuda_test.go | 6 ++- internal/lookup/root/root.go | 65 +++++++++++++++++++++++++++++++ internal/modifier/graphics.go | 10 +++-- pkg/nvcdi/common-nvml.go | 4 +- pkg/nvcdi/driver-nvml.go | 32 +++++++-------- pkg/nvcdi/lib.go | 5 +++ pkg/nvcdi/management.go | 5 +-- 9 files changed, 119 insertions(+), 83 deletions(-) create mode 100644 internal/lookup/root/root.go diff --git a/internal/discover/graphics.go b/internal/discover/graphics.go index aeaabe69..76badbf0 100644 --- a/internal/discover/graphics.go +++ b/internal/discover/graphics.go @@ -28,6 +28,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" ) // NewDRMNodesDiscoverer returns a discoverrer for the DRM device nodes associated with the specified visible devices. @@ -48,15 +49,11 @@ func NewDRMNodesDiscoverer(logger logger.Interface, devices image.VisibleDevices } // NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan. -func NewGraphicsMountsDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) { - locator := lookup.NewLibraryLocator( - lookup.WithLogger(logger), - lookup.WithRoot(driverRoot), - ) +func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string) (Discover, error) { libraries := NewMounts( logger, - locator, - driverRoot, + driver.Libraries(), + driver.Root, []string{ "libnvidia-egl-gbm.so", }, @@ -66,10 +63,10 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driverRoot string, nvi logger, lookup.NewFileLocator( lookup.WithLogger(logger), - lookup.WithRoot(driverRoot), + lookup.WithRoot(driver.Root), lookup.WithSearchPaths("/etc", "/usr/share"), ), - driverRoot, + driver.Root, []string{ "glvnd/egl_vendor.d/10_nvidia.json", "vulkan/icd.d/nvidia_icd.json", @@ -79,7 +76,7 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driverRoot string, nvi }, ) - xorg := optionalXorgDiscoverer(logger, driverRoot, nvidiaCTKPath) + xorg := optionalXorgDiscoverer(logger, driver, nvidiaCTKPath) discover := Merge( libraries, @@ -247,8 +244,8 @@ var _ Discover = (*xorgHooks)(nil) // optionalXorgDiscoverer creates a discoverer for Xorg libraries. // If the creation of the discoverer fails, a None discoverer is returned. -func optionalXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) Discover { - xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath) +func optionalXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string) Discover { + xorg, err := newXorgDiscoverer(logger, driver, nvidiaCTKPath) if err != nil { logger.Warningf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err) return None{} @@ -256,10 +253,9 @@ func optionalXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCT return xorg } -func newXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) { +func newXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string) (Discover, error) { libCudaPaths, err := cuda.New( - cuda.WithLogger(logger), - cuda.WithDriverRoot(driverRoot), + driver.Libraries(), ).Locate(".*.*") if err != nil { return nil, fmt.Errorf("failed to locate libcuda.so: %v", err) @@ -276,11 +272,11 @@ func newXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath logger, lookup.NewFileLocator( lookup.WithLogger(logger), - lookup.WithRoot(driverRoot), + lookup.WithRoot(driver.Root), lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"), lookup.WithCount(1), ), - driverRoot, + driver.Root, []string{ "nvidia/xorg/nvidia_drv.so", fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version), @@ -296,10 +292,10 @@ func newXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath logger, lookup.NewFileLocator( lookup.WithLogger(logger), - lookup.WithRoot(driverRoot), + lookup.WithRoot(driver.Root), lookup.WithSearchPaths("/usr/share"), ), - driverRoot, + driver.Root, []string{"X11/xorg.conf.d/10-nvidia.conf"}, ) diff --git a/internal/lookup/cuda/cuda.go b/internal/lookup/cuda/cuda.go index 0a8620bc..68c4db35 100644 --- a/internal/lookup/cuda/cuda.go +++ b/internal/lookup/cuda/cuda.go @@ -17,52 +17,19 @@ package cuda import ( - "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" ) type cudaLocator struct { lookup.Locator - logger logger.Interface - driverRoot string -} - -// Options is a function that configures a cudaLocator. -type Options func(*cudaLocator) - -// WithLogger is an option that configures the logger used by the locator. -func WithLogger(logger logger.Interface) Options { - return func(c *cudaLocator) { - c.logger = logger - } -} - -// WithDriverRoot is an option that configures the driver root used by the locator. -func WithDriverRoot(driverRoot string) Options { - return func(c *cudaLocator) { - c.driverRoot = driverRoot - } } // New creates a new CUDA library locator. -func New(opts ...Options) lookup.Locator { - c := &cudaLocator{} - for _, opt := range opts { - opt(c) +func New(libraries lookup.Locator) lookup.Locator { + c := cudaLocator{ + Locator: libraries, } - - if c.logger == nil { - c.logger = logger.New() - } - if c.driverRoot == "" { - c.driverRoot = "/" - } - - c.Locator = lookup.NewLibraryLocator( - lookup.WithLogger(c.logger), - lookup.WithRoot(c.driverRoot), - ) - return c + return &c } // Locate returns the path to the libcuda.so.RMVERSION file. diff --git a/internal/lookup/cuda/cuda_test.go b/internal/lookup/cuda/cuda_test.go index 7151acdb..5db69c25 100644 --- a/internal/lookup/cuda/cuda_test.go +++ b/internal/lookup/cuda/cuda_test.go @@ -57,8 +57,10 @@ func TestLocate(t *testing.T) { require.NoError(t, err) l := New( - WithLogger(logger), - WithDriverRoot(driverRoot), + lookup.NewLibraryLocator( + lookup.WithLogger(logger), + lookup.WithRoot(driverRoot), + ), ) candidates, err := l.Locate(".*") diff --git a/internal/lookup/root/root.go b/internal/lookup/root/root.go new file mode 100644 index 00000000..f96e6b99 --- /dev/null +++ b/internal/lookup/root/root.go @@ -0,0 +1,65 @@ +/** +# Copyright 2023 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package root + +import ( + "path/filepath" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" +) + +// Driver represents a filesystem in which a set of drivers or devices is defined. +type Driver struct { + logger logger.Interface + // Root represents the root from the perspective of the driver libraries and binaries. + Root string + // librarySearchPaths specifies explicit search paths for discovering libraries. + librarySearchPaths []string +} + +// New creates a new Driver root at the specified path. +// TODO: Use functional options here. +func New(logger logger.Interface, path string, librarySearchPaths []string) *Driver { + return &Driver{ + logger: logger, + Root: path, + librarySearchPaths: normalizeSearchPaths(librarySearchPaths...), + } +} + +// Drivers returns a Locator for driver libraries. +func (r *Driver) Libraries() lookup.Locator { + return lookup.NewLibraryLocator( + lookup.WithLogger(r.logger), + lookup.WithRoot(r.Root), + lookup.WithSearchPaths(r.librarySearchPaths...), + ) +} + +// normalizeSearchPaths takes a list of paths and normalized these. +// Each of the elements in the list is expanded if it is a path list and the +// resultant list is returned. +// This allows, for example, for the contents of `PATH` or `LD_LIBRARY_PATH` to +// be passed as a search path directly. +func normalizeSearchPaths(paths ...string) []string { + var normalized []string + for _, path := range paths { + normalized = append(normalized, filepath.SplitList(path)...) + } + return normalized +} diff --git a/internal/modifier/graphics.go b/internal/modifier/graphics.go index b4cbb1f6..c0a98d1c 100644 --- a/internal/modifier/graphics.go +++ b/internal/modifier/graphics.go @@ -23,6 +23,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" ) @@ -34,20 +35,21 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, image imag return nil, nil } - driverRoot := cfg.NVIDIAContainerCLIConfig.Root + // TODO: We should not just pass `nil` as the search path here. + driver := root.New(logger, cfg.NVIDIAContainerCLIConfig.Root, nil) nvidiaCTKPath := cfg.NVIDIACTKConfig.Path mounts, err := discover.NewGraphicsMountsDiscoverer( logger, - driverRoot, + driver, nvidiaCTKPath, ) if err != nil { return nil, fmt.Errorf("failed to create mounts discoverer: %v", err) } - // In standard usage, the devRoot is the same as the driverRoot. - devRoot := driverRoot + // In standard usage, the devRoot is the same as the driver.Root. + devRoot := driver.Root drmNodes, err := discover.NewDRMNodesDiscoverer( logger, image.DevicesFromEnvvars(visibleDevicesEnvvar), diff --git a/pkg/nvcdi/common-nvml.go b/pkg/nvcdi/common-nvml.go index a33cfc61..4c634a72 100644 --- a/pkg/nvcdi/common-nvml.go +++ b/pkg/nvcdi/common-nvml.go @@ -36,12 +36,12 @@ func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) { }, ) - graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath) + graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driver, l.nvidiaCTKPath) if err != nil { l.logger.Warningf("failed to create discoverer for graphics mounts: %v", err) } - driverFiles, err := NewDriverDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath, l.nvmllib) + driverFiles, err := NewDriverDiscoverer(l.logger, l.driver, l.nvidiaCTKPath, l.nvmllib) if err != nil { return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err) } diff --git a/pkg/nvcdi/driver-nvml.go b/pkg/nvcdi/driver-nvml.go index cbc892af..28bd0704 100644 --- a/pkg/nvcdi/driver-nvml.go +++ b/pkg/nvcdi/driver-nvml.go @@ -27,12 +27,13 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" "golang.org/x/sys/unix" ) // NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation. // The supplied NVML Library is used to query the expected driver version. -func NewDriverDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) { +func NewDriverDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) { if r := nvmllib.Init(); r != nvml.SUCCESS { return nil, fmt.Errorf("failed to initialize NVML: %v", r) } @@ -47,26 +48,26 @@ func NewDriverDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPa return nil, fmt.Errorf("failed to determine driver version: %v", r) } - return newDriverVersionDiscoverer(logger, driverRoot, nvidiaCTKPath, version) + return newDriverVersionDiscoverer(logger, driver, nvidiaCTKPath, version) } -func newDriverVersionDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string, version string) (discover.Discover, error) { - libraries, err := NewDriverLibraryDiscoverer(logger, driverRoot, nvidiaCTKPath, version) +func newDriverVersionDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string, version string) (discover.Discover, error) { + libraries, err := NewDriverLibraryDiscoverer(logger, driver, nvidiaCTKPath, version) if err != nil { return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err) } - ipcs, err := discover.NewIPCDiscoverer(logger, driverRoot) + ipcs, err := discover.NewIPCDiscoverer(logger, driver.Root) if err != nil { return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err) } - firmwares, err := NewDriverFirmwareDiscoverer(logger, driverRoot, version) + firmwares, err := NewDriverFirmwareDiscoverer(logger, driver.Root, version) if err != nil { return nil, fmt.Errorf("failed to create discoverer for GSP firmware: %v", err) } - binaries := NewDriverBinariesDiscoverer(logger, driverRoot) + binaries := NewDriverBinariesDiscoverer(logger, driver.Root) d := discover.Merge( libraries, @@ -79,8 +80,8 @@ func newDriverVersionDiscoverer(logger logger.Interface, driverRoot string, nvid } // NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version. -func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string, version string) (discover.Discover, error) { - libraryPaths, err := getVersionLibs(logger, driverRoot, version) +func NewDriverLibraryDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string, version string) (discover.Discover, error) { + libraryPaths, err := getVersionLibs(logger, driver, version) if err != nil { return nil, fmt.Errorf("failed to get libraries for driver version: %v", err) } @@ -89,9 +90,9 @@ func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvid logger, lookup.NewFileLocator( lookup.WithLogger(logger), - lookup.WithRoot(driverRoot), + lookup.WithRoot(driver.Root), ), - driverRoot, + driver.Root, libraryPaths, ) @@ -185,12 +186,11 @@ func NewDriverBinariesDiscoverer(logger logger.Interface, driverRoot string) dis // getVersionLibs checks the LDCache for libraries ending in the specified driver version. // Although the ldcache at the specified driverRoot is queried, the paths are returned relative to this driverRoot. // This allows the standard mount location logic to be used for resolving the mounts. -func getVersionLibs(logger logger.Interface, driverRoot string, version string) ([]string, error) { +func getVersionLibs(logger logger.Interface, driver *root.Driver, version string) ([]string, error) { logger.Infof("Using driver version %v", version) libCudaPaths, err := cuda.New( - cuda.WithLogger(logger), - cuda.WithDriverRoot(driverRoot), + driver.Libraries(), ).Locate("." + version) if err != nil { return nil, fmt.Errorf("failed to locate libcuda.so.%v: %v", version, err) @@ -208,13 +208,13 @@ func getVersionLibs(logger logger.Interface, driverRoot string, version string) return nil, fmt.Errorf("failed to locate libraries for driver version %v: %v", version, err) } - if driverRoot == "/" || driverRoot == "" { + if driver.Root == "/" || driver.Root == "" { return libs, nil } var relative []string for _, l := range libs { - relative = append(relative, strings.TrimPrefix(l, driverRoot)) + relative = append(relative, strings.TrimPrefix(l, driver.Root)) } return relative, nil diff --git a/pkg/nvcdi/lib.go b/pkg/nvcdi/lib.go index 8a60ac6b..3839697c 100644 --- a/pkg/nvcdi/lib.go +++ b/pkg/nvcdi/lib.go @@ -23,6 +23,7 @@ import ( "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" "github.com/NVIDIA/go-nvlib/pkg/nvml" "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform" @@ -54,6 +55,7 @@ type nvcdilib struct { vendor string class string + driver *root.Driver infolib info.Interface mergedDeviceOptions []transform.MergedDeviceOption @@ -87,6 +89,9 @@ func New(opts ...Option) (Interface, error) { l.infolib = info.New() } + // TODO: We need to improve the construction of this driver root. + l.driver = root.New(l.logger, l.driverRoot, l.librarySearchPaths) + var lib Interface switch l.resolveMode() { case ModeCSV: diff --git a/pkg/nvcdi/management.go b/pkg/nvcdi/management.go index 9352e110..460a4873 100644 --- a/pkg/nvcdi/management.go +++ b/pkg/nvcdi/management.go @@ -65,7 +65,7 @@ func (m *managementlib) GetCommonEdits() (*cdi.ContainerEdits, error) { return nil, fmt.Errorf("failed to get CUDA version: %v", err) } - driver, err := newDriverVersionDiscoverer(m.logger, m.driverRoot, m.nvidiaCTKPath, version) + driver, err := newDriverVersionDiscoverer(m.logger, m.driver, m.nvidiaCTKPath, version) if err != nil { return nil, fmt.Errorf("failed to create driver library discoverer: %v", err) } @@ -86,8 +86,7 @@ func (m *managementlib) getCudaVersion() (string, error) { } libCudaPaths, err := cuda.New( - cuda.WithLogger(m.logger), - cuda.WithDriverRoot(m.driverRoot), + m.driver.Libraries(), ).Locate(".*.*") if err != nil { return "", fmt.Errorf("failed to locate libcuda.so: %v", err)