mirror of
https://github.com/NVIDIA/nvidia-container-toolkit
synced 2024-11-22 00:08:11 +00:00
Inject additional libraries for full X11 functionality
This change updates X11 library detection to ensure that this works for a wider range of driver installations. Signed-off-by: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Signed-off-by: tux-rampage <tuxrampage@gmail.com> Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Seungmin Kim
Evan Lezar
parent
6fd2fc0c24
commit
aae3da88c3
@ -50,16 +50,9 @@ func NewDRMNodesDiscoverer(logger logger.Interface, devices image.VisibleDevices
|
||||
|
||||
// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
|
||||
func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) (Discover, error) {
|
||||
libraries := NewMounts(
|
||||
logger,
|
||||
driver.Libraries(),
|
||||
driver.Root,
|
||||
[]string{
|
||||
"libnvidia-egl-gbm.so.*",
|
||||
},
|
||||
)
|
||||
libraries := newGraphicsLibrariesDiscoverer(logger, driver, nvidiaCDIHookPath)
|
||||
|
||||
jsonMounts := NewMounts(
|
||||
configs := NewMounts(
|
||||
logger,
|
||||
driver.Configs(),
|
||||
driver.Root,
|
||||
@ -68,26 +61,25 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, n
|
||||
"egl/egl_external_platform.d/15_nvidia_gbm.json",
|
||||
"egl/egl_external_platform.d/10_nvidia_wayland.json",
|
||||
"nvidia/nvoptix.bin",
|
||||
"X11/xorg.conf.d/10-nvidia.conf",
|
||||
"X11/xorg.conf.d/nvidia-drm-outputclass.conf",
|
||||
},
|
||||
)
|
||||
|
||||
xorg := optionalXorgDiscoverer(logger, driver, nvidiaCDIHookPath)
|
||||
|
||||
discover := Merge(
|
||||
libraries,
|
||||
jsonMounts,
|
||||
newVulkanMountsDiscoverer(logger, driver),
|
||||
xorg,
|
||||
configs,
|
||||
newVulkanConfigsDiscover(logger, driver),
|
||||
)
|
||||
|
||||
return discover, nil
|
||||
}
|
||||
|
||||
// newVulkanMountsDiscoverer creates a discoverer for vulkan ICD files.
|
||||
// newVulkanConfigsDiscover creates a discoverer for vulkan ICD files.
|
||||
// For these files we search the standard driver config paths as well as the
|
||||
// driver root itself. This allows us to support GKE installations where the
|
||||
// vulkan ICD files are at {{ .driverRoot }}/vulkan instead of in /etc/vulkan.
|
||||
func newVulkanMountsDiscoverer(logger logger.Interface, driver *root.Driver) Discover {
|
||||
func newVulkanConfigsDiscover(logger logger.Interface, driver *root.Driver) Discover {
|
||||
locator := lookup.First(driver.Configs(), driver.Files())
|
||||
return &mountsToContainerPath{
|
||||
logger: logger,
|
||||
@ -101,6 +93,165 @@ func newVulkanMountsDiscoverer(logger logger.Interface, driver *root.Driver) Dis
|
||||
}
|
||||
}
|
||||
|
||||
type graphicsDriverLibraries struct {
|
||||
Discover
|
||||
logger logger.Interface
|
||||
nvidiaCDIHookPath string
|
||||
}
|
||||
|
||||
var _ Discover = (*graphicsDriverLibraries)(nil)
|
||||
|
||||
func newGraphicsLibrariesDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) Discover {
|
||||
cudaLibRoot, cudaVersionPattern := getCUDALibRootAndVersionPattern(logger, driver)
|
||||
|
||||
libraries := NewMounts(
|
||||
logger,
|
||||
driver.Libraries(),
|
||||
driver.Root,
|
||||
[]string{
|
||||
// The libnvidia-egl-gbm and libnvidia-egl-wayland libraries do not
|
||||
// have the RM version. Use the *.* pattern to match X.Y.Z versions.
|
||||
"libnvidia-egl-gbm.so.*.*",
|
||||
"libnvidia-egl-wayland.so.*.*",
|
||||
// We include the following libraries to have them available for
|
||||
// symlink creation below:
|
||||
// If CDI injection is used, these should already be detected as:
|
||||
// * libnvidia-allocator.so.RM_VERSION
|
||||
// * libnvidia-vulkan-producer.so.RM_VERSION
|
||||
// but need to be handled for the legacy case too.
|
||||
"libnvidia-allocator.so." + cudaVersionPattern,
|
||||
"libnvidia-vulkan-producer.so." + cudaVersionPattern,
|
||||
},
|
||||
)
|
||||
|
||||
xorgLibraries := NewMounts(
|
||||
logger,
|
||||
lookup.NewFileLocator(
|
||||
lookup.WithLogger(logger),
|
||||
lookup.WithRoot(driver.Root),
|
||||
lookup.WithSearchPaths(buildXOrgSearchPaths(cudaLibRoot)...),
|
||||
lookup.WithCount(1),
|
||||
),
|
||||
driver.Root,
|
||||
[]string{
|
||||
"nvidia_drv.so",
|
||||
"libglxserver_nvidia.so." + cudaVersionPattern,
|
||||
},
|
||||
)
|
||||
|
||||
return &graphicsDriverLibraries{
|
||||
Discover: Merge(libraries, xorgLibraries),
|
||||
logger: logger,
|
||||
nvidiaCDIHookPath: nvidiaCDIHookPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Create necessary library symlinks for graphics drivers
|
||||
func (d graphicsDriverLibraries) Hooks() ([]Hook, error) {
|
||||
mounts, err := d.Discover.Mounts()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get library mounts: %v", err)
|
||||
}
|
||||
|
||||
var links []string
|
||||
for _, mount := range mounts {
|
||||
dir, filename := filepath.Split(mount.Path)
|
||||
switch {
|
||||
case d.isDriverLibrary(filename, "libnvidia-allocator.so"):
|
||||
// gbm/nvidia-drm_gbm.so is a symlink to ../libnvidia-allocator.so.1 which
|
||||
// in turn symlinks to libnvidia-allocator.so.RM_VERSION and is created
|
||||
// when ldconfig is run in the container.
|
||||
// create libnvidia-allocate.so.1 -> libnvidia-allocate.so.RM_VERSION symlink
|
||||
links = append(links, fmt.Sprintf("%s::%s", filename, filepath.Join(dir, "libnvidia-allocator.so.1")))
|
||||
// create gbm/nvidia-drm_gbm.so -> ../libnvidia-allocate.so.1 symlink
|
||||
linkPath := filepath.Join(dir, "gbm", "nvidia-drm_gbm.so")
|
||||
links = append(links, fmt.Sprintf("%s::%s", "../libnvidia-allocator.so.1", linkPath))
|
||||
case d.isDriverLibrary(filename, "libnvidia-vulkan-producer.so"):
|
||||
// libnvidia-vulkan-producer.so is a drirect symlink to libnvidia-vulkan-producer.so.RM_VERSION
|
||||
// create libnvidia-vulkan-producer.so -> libnvidia-vulkan-producer.so.RM_VERSION symlink
|
||||
linkPath := filepath.Join(dir, "libnvidia-vulkan-producer.so")
|
||||
links = append(links, fmt.Sprintf("%s::%s", filename, linkPath))
|
||||
case d.isDriverLibrary(filename, "libglxserver_nvidia.so"):
|
||||
// libglxserver_nvidia.so is a directl symlink to libglxserver_nvidia.so.RM_VERSION
|
||||
// create libglxserver_nvidia.so -> libglxserver_nvidia.so.RM_VERSION symlink
|
||||
linkPath := filepath.Join(dir, "libglxserver_nvidia.so")
|
||||
links = append(links, fmt.Sprintf("%s::%s", filename, linkPath))
|
||||
}
|
||||
}
|
||||
if len(links) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
hooks := CreateCreateSymlinkHook(d.nvidiaCDIHookPath, links)
|
||||
|
||||
return hooks.Hooks()
|
||||
}
|
||||
|
||||
// isDriverLibrary checks whether the specified filename is a specific driver library.
|
||||
func (d graphicsDriverLibraries) isDriverLibrary(filename string, libraryName string) bool {
|
||||
// TODO: Instead of `.*.*` we could use the driver version.
|
||||
pattern := strings.TrimSuffix(libraryName, ".") + ".*.*"
|
||||
match, _ := filepath.Match(pattern, filename)
|
||||
return match
|
||||
}
|
||||
|
||||
// getCUDALibRootAndVersionPattern returns the parent directory and the version
|
||||
// suffix of the libcuda.so.*.* library at the driver root.
|
||||
// If the library cannot be located an empty root is returned.
|
||||
// If the version string cannot be extracted, the generic *.* pattern is returned.
|
||||
func getCUDALibRootAndVersionPattern(logger logger.Interface, driver *root.Driver) (string, string) {
|
||||
libCudaPaths, err := cuda.New(
|
||||
driver.Libraries(),
|
||||
).Locate(".*.*")
|
||||
if err != nil {
|
||||
logger.Warningf("failed to locate libcuda.so: %v; using *.*", err)
|
||||
return "", "*.*"
|
||||
}
|
||||
libcudaPath := libCudaPaths[0]
|
||||
|
||||
libRoot := filepath.Dir(libcudaPath)
|
||||
version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
|
||||
if version == "" {
|
||||
logger.Warningf("failed to extract version from %v; using *.*", libcudaPath)
|
||||
version = "*.*"
|
||||
}
|
||||
|
||||
return libRoot, version
|
||||
}
|
||||
|
||||
// buildXOrgSearchPaths returns the ordered list of search paths for XOrg files.
|
||||
func buildXOrgSearchPaths(libRoot string) []string {
|
||||
var paths []string
|
||||
if libRoot != "" {
|
||||
paths = append(paths,
|
||||
filepath.Join(libRoot, "nvidia/xorg"),
|
||||
filepath.Join(libRoot, "xorg", "modules", "drivers"),
|
||||
filepath.Join(libRoot, "xorg", "modules", "extensions"),
|
||||
filepath.Join(libRoot, "xorg", "modules/updates", "drivers"),
|
||||
filepath.Join(libRoot, "xorg", "modules/updates", "extensions"),
|
||||
)
|
||||
}
|
||||
|
||||
return append(paths,
|
||||
filepath.Join("/usr/lib/xorg", "modules", "drivers"),
|
||||
filepath.Join("/usr/lib/xorg", "modules", "extensions"),
|
||||
filepath.Join("/usr/lib/xorg", "modules/updates", "drivers"),
|
||||
filepath.Join("/usr/lib/xorg", "modules/updates", "extensions"),
|
||||
filepath.Join("/usr/lib64/xorg", "modules", "drivers"),
|
||||
filepath.Join("/usr/lib64/xorg", "modules", "extensions"),
|
||||
filepath.Join("/usr/lib64/xorg", "modules/updates", "drivers"),
|
||||
filepath.Join("/usr/lib64/xorg", "modules/updates", "extensions"),
|
||||
filepath.Join("/usr/X11R6/lib", "modules", "drivers"),
|
||||
filepath.Join("/usr/X11R6/lib", "modules", "extensions"),
|
||||
filepath.Join("/usr/X11R6/lib", "modules/updates", "drivers"),
|
||||
filepath.Join("/usr/X11R6/lib", "modules/updates", "extensions"),
|
||||
filepath.Join("/usr/X11R6/lib64", "modules", "drivers"),
|
||||
filepath.Join("/usr/X11R6/lib64", "modules", "extensions"),
|
||||
filepath.Join("/usr/X11R6/lib64", "modules/updates", "drivers"),
|
||||
filepath.Join("/usr/X11R6/lib64", "modules/updates", "extensions"),
|
||||
)
|
||||
}
|
||||
|
||||
type drmDevicesByPath struct {
|
||||
None
|
||||
logger logger.Interface
|
||||
@ -248,118 +399,6 @@ func newDRMDeviceFilter(devices image.VisibleDevices, devRoot string) (Filter, e
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
type xorgHooks struct {
|
||||
libraries Discover
|
||||
driverVersion string
|
||||
nvidiaCDIHookPath string
|
||||
}
|
||||
|
||||
var _ Discover = (*xorgHooks)(nil)
|
||||
|
||||
// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
|
||||
// If the creation of the discoverer fails, a None discoverer is returned.
|
||||
func optionalXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) Discover {
|
||||
xorg, err := newXorgDiscoverer(logger, driver, nvidiaCDIHookPath)
|
||||
if err != nil {
|
||||
logger.Warningf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
|
||||
return None{}
|
||||
}
|
||||
return xorg
|
||||
}
|
||||
|
||||
func newXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) (Discover, error) {
|
||||
libCudaPaths, err := cuda.New(
|
||||
driver.Libraries(),
|
||||
).Locate(".*.*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
|
||||
}
|
||||
libcudaPath := libCudaPaths[0]
|
||||
|
||||
version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
|
||||
if version == "" {
|
||||
return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
|
||||
}
|
||||
|
||||
libRoot := filepath.Dir(libcudaPath)
|
||||
xorgLibs := NewMounts(
|
||||
logger,
|
||||
lookup.NewFileLocator(
|
||||
lookup.WithLogger(logger),
|
||||
lookup.WithRoot(driver.Root),
|
||||
lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
|
||||
lookup.WithCount(1),
|
||||
),
|
||||
driver.Root,
|
||||
[]string{
|
||||
"nvidia/xorg/nvidia_drv.so",
|
||||
fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
|
||||
},
|
||||
)
|
||||
xorgHooks := xorgHooks{
|
||||
libraries: xorgLibs,
|
||||
driverVersion: version,
|
||||
nvidiaCDIHookPath: nvidiaCDIHookPath,
|
||||
}
|
||||
|
||||
xorgConfig := NewMounts(
|
||||
logger,
|
||||
driver.Configs(),
|
||||
driver.Root,
|
||||
[]string{"X11/xorg.conf.d/10-nvidia.conf"},
|
||||
)
|
||||
|
||||
d := Merge(
|
||||
xorgLibs,
|
||||
xorgConfig,
|
||||
xorgHooks,
|
||||
)
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Devices returns no devices for Xorg
|
||||
func (m xorgHooks) Devices() ([]Device, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Hooks returns a hook to create symlinks for Xorg libraries
|
||||
func (m xorgHooks) Hooks() ([]Hook, error) {
|
||||
mounts, err := m.libraries.Mounts()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get mounts: %v", err)
|
||||
}
|
||||
if len(mounts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var target string
|
||||
for _, mount := range mounts {
|
||||
filename := filepath.Base(mount.HostPath)
|
||||
if filename == "libglxserver_nvidia.so."+m.driverVersion {
|
||||
target = mount.Path
|
||||
}
|
||||
}
|
||||
|
||||
if target == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
link := strings.TrimSuffix(target, "."+m.driverVersion)
|
||||
links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
|
||||
symlinkHook := CreateCreateSymlinkHook(
|
||||
m.nvidiaCDIHookPath,
|
||||
links,
|
||||
)
|
||||
|
||||
return symlinkHook.Hooks()
|
||||
}
|
||||
|
||||
// Mounts returns the libraries required for Xorg
|
||||
func (m xorgHooks) Mounts() ([]Mount, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// selectDeviceByPath is a filter that allows devices to be selected by the path
|
||||
type selectDeviceByPath map[string]bool
|
||||
|
||||
|
||||
170
internal/discover/graphics_test.go
Normal file
170
internal/discover/graphics_test.go
Normal file
@ -0,0 +1,170 @@
|
||||
/**
|
||||
# Copyright 2024 NVIDIA CORPORATION
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
**/
|
||||
|
||||
package discover
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
testlog "github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGraphicsLibrariesDiscoverer(t *testing.T) {
|
||||
logger, _ := testlog.NewNullLogger()
|
||||
|
||||
testCases := []struct {
|
||||
description string
|
||||
libraries *DiscoverMock
|
||||
expectedMounts []Mount
|
||||
expectedHooks []Hook
|
||||
}{
|
||||
{
|
||||
description: "none discovered",
|
||||
libraries: &DiscoverMock{
|
||||
MountsFunc: func() ([]Mount, error) {
|
||||
mounts := []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-egl-gbm.so.123.45.67",
|
||||
},
|
||||
}
|
||||
return mounts, nil
|
||||
},
|
||||
},
|
||||
expectedMounts: []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-egl-gbm.so.123.45.67",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "libnvidia-allocator discovered",
|
||||
libraries: &DiscoverMock{
|
||||
MountsFunc: func() ([]Mount, error) {
|
||||
mounts := []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-allocator.so.123.45.67",
|
||||
},
|
||||
}
|
||||
return mounts, nil
|
||||
},
|
||||
},
|
||||
expectedMounts: []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-allocator.so.123.45.67",
|
||||
},
|
||||
},
|
||||
expectedHooks: []Hook{
|
||||
{
|
||||
Lifecycle: "createContainer",
|
||||
Path: "/usr/bin/nvidia-cdi-hook",
|
||||
Args: []string{"nvidia-cdi-hook", "create-symlinks",
|
||||
"--link", "libnvidia-allocator.so.123.45.67::/usr/lib64/libnvidia-allocator.so.1",
|
||||
"--link", "../libnvidia-allocator.so.1::/usr/lib64/gbm/nvidia-drm_gbm.so",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "libnvidia-vulkan-producer discovered",
|
||||
libraries: &DiscoverMock{
|
||||
MountsFunc: func() ([]Mount, error) {
|
||||
mounts := []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-vulkan-producer.so.123.45.67",
|
||||
},
|
||||
}
|
||||
return mounts, nil
|
||||
},
|
||||
},
|
||||
expectedMounts: []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-vulkan-producer.so.123.45.67",
|
||||
},
|
||||
},
|
||||
expectedHooks: []Hook{
|
||||
{
|
||||
Lifecycle: "createContainer",
|
||||
Path: "/usr/bin/nvidia-cdi-hook",
|
||||
Args: []string{"nvidia-cdi-hook", "create-symlinks",
|
||||
"--link", "libnvidia-vulkan-producer.so.123.45.67::/usr/lib64/libnvidia-vulkan-producer.so",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "libnvidia-allocator and libnvidia-vulkan-producer discovered",
|
||||
libraries: &DiscoverMock{
|
||||
MountsFunc: func() ([]Mount, error) {
|
||||
mounts := []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-allocator.so.123.45.67",
|
||||
},
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-vulkan-producer.so.123.45.67",
|
||||
},
|
||||
}
|
||||
return mounts, nil
|
||||
},
|
||||
},
|
||||
expectedMounts: []Mount{
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-allocator.so.123.45.67",
|
||||
},
|
||||
{
|
||||
Path: "/usr/lib64/libnvidia-vulkan-producer.so.123.45.67",
|
||||
},
|
||||
},
|
||||
expectedHooks: []Hook{
|
||||
{
|
||||
Lifecycle: "createContainer",
|
||||
Path: "/usr/bin/nvidia-cdi-hook",
|
||||
Args: []string{"nvidia-cdi-hook", "create-symlinks",
|
||||
"--link", "libnvidia-allocator.so.123.45.67::/usr/lib64/libnvidia-allocator.so.1",
|
||||
"--link", "../libnvidia-allocator.so.1::/usr/lib64/gbm/nvidia-drm_gbm.so",
|
||||
"--link", "libnvidia-vulkan-producer.so.123.45.67::/usr/lib64/libnvidia-vulkan-producer.so",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.description, func(t *testing.T) {
|
||||
d := &graphicsDriverLibraries{
|
||||
Discover: tc.libraries,
|
||||
logger: logger,
|
||||
nvidiaCDIHookPath: "/usr/bin/nvidia-cdi-hook",
|
||||
}
|
||||
|
||||
devices, err := d.Devices()
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, devices)
|
||||
require.Len(t, tc.libraries.calls.Devices, 1)
|
||||
|
||||
mounts, err := d.Mounts()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, tc.expectedMounts, mounts)
|
||||
require.Len(t, tc.libraries.calls.Mounts, 1)
|
||||
|
||||
hooks, err := d.Hooks()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, tc.expectedHooks, hooks)
|
||||
require.Len(t, tc.libraries.calls.Mounts, 2)
|
||||
require.Len(t, tc.libraries.calls.Hooks, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user