From 0a37f8798a4e79fe2ff5fb8b0cb1004833b5004d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 7 Jul 2023 15:09:12 +0200 Subject: [PATCH 1/3] Add firmware search paths when generating CDI specifications Path to locate the GSP firmware is explicitly set to /lib/firmware/nvidia. Users may chose to install the GSP firmware in alternate locations where the kernel would look for firmware on the root filesystem. Add locate functionality which looks for the GSP firmware files in the same location as the kernel would (https://docs.kernel.org/driver-api/firmware/fw_search_path.html). The paths searched in order are: - path described in /sys/module/firmware_class/parameters/path - /lib/firmware/updates/UTS_RELEASE/ - /lib/firmware/updates/ - /lib/firmware/UTS_RELEASE/ - /lib/firmware/ Signed-off-by: Evan Lezar --- pkg/nvcdi/driver-nvml.go | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/pkg/nvcdi/driver-nvml.go b/pkg/nvcdi/driver-nvml.go index 17456779..0343abc6 100644 --- a/pkg/nvcdi/driver-nvml.go +++ b/pkg/nvcdi/driver-nvml.go @@ -18,6 +18,7 @@ package nvcdi import ( "fmt" + "os" "path/filepath" "strings" @@ -26,6 +27,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda" "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml" + "golang.org/x/sys/unix" ) // NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation. @@ -55,7 +57,10 @@ func newDriverVersionDiscoverer(logger logger.Interface, driverRoot string, nvid return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err) } - firmwares := NewDriverFirmwareDiscoverer(logger, driverRoot, version) + firmwares, err := NewDriverFirmwareDiscoverer(logger, driverRoot, version) + if err != nil { + return nil, fmt.Errorf("failed to create discoverer for GSP firmware: %v", err) + } binaries := NewDriverBinariesDiscoverer(logger, driverRoot) @@ -96,18 +101,54 @@ func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvid return d, nil } +func getUTSRelease(logger logger.Interface) (string, error) { + utsname := &unix.Utsname{} + if err := unix.Uname(utsname); err != nil { + return "", err + } + return unix.ByteSliceToString(utsname.Release[:]), nil +} + +func getFirmwareSearchPaths(logger logger.Interface) ([]string, error) { + utsRelease, err := getUTSRelease(logger) + if err != nil { + return nil, fmt.Errorf("failed to get UTS_RELEASE: %v", err) + } + + firmwarePaths := []string{ + filepath.Join("/lib/firmware/updates/", utsRelease), + filepath.Join("/lib/firmware/updates/"), + filepath.Join("/lib/firmware/", utsRelease), + filepath.Join("/lib/firmware/"), + } + + customFirmwareClassPath, err := os.ReadFile("/sys/module/firmware_class/parameters/path") + if err != nil { + return nil, fmt.Errorf("failed to get custom firmware class path for driver version: %v", err) + } + if !(len(customFirmwareClassPath) == 1 && customFirmwareClassPath[0] == byte(10)) { + firmwarePaths = append(firmwarePaths, string(customFirmwareClassPath)) + } + return firmwarePaths, nil +} + // NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version. -func NewDriverFirmwareDiscoverer(logger logger.Interface, driverRoot string, version string) discover.Discover { - gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp*.bin") +func NewDriverFirmwareDiscoverer(logger logger.Interface, driverRoot string, version string) (discover.Discover, error) { + gspFirmwareSearchPaths, err := getFirmwareSearchPaths(logger) + if err != nil { + return nil, fmt.Errorf("failed to get libraries for driver version: %v", err) + } + gspFirmwarePaths := filepath.Join("nvidia", version, "gsp*.bin") return discover.NewMounts( logger, lookup.NewFileLocator( lookup.WithLogger(logger), lookup.WithRoot(driverRoot), + lookup.WithSearchPaths(gspFirmwareSearchPaths...), ), driverRoot, - []string{gspFirmwarePath}, - ) + []string{gspFirmwarePaths}, + ), nil } // NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver. From d3d41a3e1d323ee991da1e70020fda1bde543128 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 7 Jul 2023 15:19:23 +0200 Subject: [PATCH 2/3] Simplify handling of custom firmware path Signed-off-by: Evan Lezar --- pkg/nvcdi/driver-nvml.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/pkg/nvcdi/driver-nvml.go b/pkg/nvcdi/driver-nvml.go index 0343abc6..8b63beaa 100644 --- a/pkg/nvcdi/driver-nvml.go +++ b/pkg/nvcdi/driver-nvml.go @@ -101,7 +101,7 @@ func NewDriverLibraryDiscoverer(logger logger.Interface, driverRoot string, nvid return d, nil } -func getUTSRelease(logger logger.Interface) (string, error) { +func getUTSRelease() (string, error) { utsname := &unix.Utsname{} if err := unix.Uname(utsname); err != nil { return "", err @@ -110,7 +110,7 @@ func getUTSRelease(logger logger.Interface) (string, error) { } func getFirmwareSearchPaths(logger logger.Interface) ([]string, error) { - utsRelease, err := getUTSRelease(logger) + utsRelease, err := getUTSRelease() if err != nil { return nil, fmt.Errorf("failed to get UTS_RELEASE: %v", err) } @@ -122,21 +122,29 @@ func getFirmwareSearchPaths(logger logger.Interface) ([]string, error) { filepath.Join("/lib/firmware/"), } - customFirmwareClassPath, err := os.ReadFile("/sys/module/firmware_class/parameters/path") - if err != nil { - return nil, fmt.Errorf("failed to get custom firmware class path for driver version: %v", err) - } - if !(len(customFirmwareClassPath) == 1 && customFirmwareClassPath[0] == byte(10)) { - firmwarePaths = append(firmwarePaths, string(customFirmwareClassPath)) + if p := getCustomFirmwareClassPath(logger); p != "" { + logger.Debugf("using custom firmware class path: %s", p) + firmwarePaths = append(firmwarePaths, p) } return firmwarePaths, nil } +// getCustomFirmwareClassPath returns the custom firmware class path if it exists. +func getCustomFirmwareClassPath(logger logger.Interface) string { + customFirmwareClassPath, err := os.ReadFile("/sys/module/firmware_class/parameters/path") + if err != nil { + logger.Warningf("failed to get custom firmware class path: %v", err) + return "" + } + + return strings.TrimSpace(string(customFirmwareClassPath)) +} + // NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version. func NewDriverFirmwareDiscoverer(logger logger.Interface, driverRoot string, version string) (discover.Discover, error) { gspFirmwareSearchPaths, err := getFirmwareSearchPaths(logger) if err != nil { - return nil, fmt.Errorf("failed to get libraries for driver version: %v", err) + return nil, fmt.Errorf("failed to get firmware search paths: %v", err) } gspFirmwarePaths := filepath.Join("nvidia", version, "gsp*.bin") return discover.NewMounts( From 81908c8cc95e035c67f17bd6bad9c2e90c48a660 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 7 Jul 2023 15:26:26 +0200 Subject: [PATCH 3/3] Search custom firmware paths first Signed-off-by: Evan Lezar --- CHANGELOG.md | 1 + pkg/nvcdi/driver-nvml.go | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5fd23f..751986b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Add option to create device nodes when creating `/dev/char` symlinks * Create ouput folders if required when running `nvidia-ctk runtime configure` * Generate default config as post-install step. +* Added support for detecting GSP firmware at custom paths when generating CDI specifications. * [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library diff --git a/pkg/nvcdi/driver-nvml.go b/pkg/nvcdi/driver-nvml.go index 8b63beaa..bc8171a3 100644 --- a/pkg/nvcdi/driver-nvml.go +++ b/pkg/nvcdi/driver-nvml.go @@ -110,23 +110,26 @@ func getUTSRelease() (string, error) { } func getFirmwareSearchPaths(logger logger.Interface) ([]string, error) { + + var firmwarePaths []string + if p := getCustomFirmwareClassPath(logger); p != "" { + logger.Debugf("using custom firmware class path: %s", p) + firmwarePaths = append(firmwarePaths, p) + } + utsRelease, err := getUTSRelease() if err != nil { return nil, fmt.Errorf("failed to get UTS_RELEASE: %v", err) } - firmwarePaths := []string{ + standardPaths := []string{ filepath.Join("/lib/firmware/updates/", utsRelease), filepath.Join("/lib/firmware/updates/"), filepath.Join("/lib/firmware/", utsRelease), filepath.Join("/lib/firmware/"), } - if p := getCustomFirmwareClassPath(logger); p != "" { - logger.Debugf("using custom firmware class path: %s", p) - firmwarePaths = append(firmwarePaths, p) - } - return firmwarePaths, nil + return append(firmwarePaths, standardPaths...), nil } // getCustomFirmwareClassPath returns the custom firmware class path if it exists.