From 5f2be723359e7e5e9adaea0631ef2c87a5e760bc Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 6 Jun 2024 13:42:47 +0200 Subject: [PATCH] Support vulkan ICD files in a driver root Signed-off-by: Evan Lezar --- internal/discover/graphics.go | 22 ++- internal/discover/mounts-to-container-path.go | 81 ++++++++++ .../discover/mounts-to-container-path_test.go | 148 ++++++++++++++++++ internal/lookup/root/root.go | 10 ++ 4 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 internal/discover/mounts-to-container-path.go create mode 100644 internal/discover/mounts-to-container-path_test.go diff --git a/internal/discover/graphics.go b/internal/discover/graphics.go index b7efec8b..21a5678b 100644 --- a/internal/discover/graphics.go +++ b/internal/discover/graphics.go @@ -65,9 +65,6 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, n driver.Root, []string{ "glvnd/egl_vendor.d/10_nvidia.json", - "vulkan/icd.d/nvidia_icd.json", - "vulkan/icd.d/nvidia_layers.json", - "vulkan/implicit_layer.d/nvidia_layers.json", "egl/egl_external_platform.d/15_nvidia_gbm.json", "egl/egl_external_platform.d/10_nvidia_wayland.json", "nvidia/nvoptix.bin", @@ -79,12 +76,31 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, n discover := Merge( libraries, jsonMounts, + newVulkanMountsDiscoverer(logger, driver), xorg, ) return discover, nil } +// newVulkanMountsDiscoverer creates a discoverer for vulkan ICD files. +// For these files we search the standard driver config paths as well as the +// driver root itself. This allows us to support GKE installations where the +// vulkan ICD files are at {{ .driverRoot }}/vulkan instead of in /etc/vulkan. +func newVulkanMountsDiscoverer(logger logger.Interface, driver *root.Driver) Discover { + locator := lookup.First(driver.Configs(), driver.Files()) + return &mountsToContainerPath{ + logger: logger, + locator: locator, + required: []string{ + "vulkan/icd.d/nvidia_icd.json", + "vulkan/icd.d/nvidia_layers.json", + "vulkan/implicit_layer.d/nvidia_layers.json", + }, + containerRoot: "/etc", + } +} + type drmDevicesByPath struct { None logger logger.Interface diff --git a/internal/discover/mounts-to-container-path.go b/internal/discover/mounts-to-container-path.go new file mode 100644 index 00000000..602f153d --- /dev/null +++ b/internal/discover/mounts-to-container-path.go @@ -0,0 +1,81 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" +) + +// mountsToContainerPath defines a Discoverer for a required set of mounts. +// When these are discovered by a locator the specified container root is used +// to construct the container path for the mount returned. +type mountsToContainerPath struct { + None + logger logger.Interface + locator lookup.Locator + required []string + containerRoot string +} + +func (d *mountsToContainerPath) Mounts() ([]Mount, error) { + seen := make(map[string]bool) + var mounts []Mount + for _, target := range d.required { + if strings.Contains(target, "*") { + // TODO: We could relax this condition. + return nil, fmt.Errorf("wildcard patterns are not supported: %s", target) + } + candidates, err := d.locator.Locate(target) + if err != nil { + d.logger.Warningf("Could not locate %v: %v", target, err) + continue + } + if len(candidates) == 0 { + d.logger.Warningf("Missing %v", target) + continue + } + hostPath := candidates[0] + if seen[hostPath] { + d.logger.Debugf("Skipping duplicate mount %v", hostPath) + continue + } + seen[hostPath] = true + d.logger.Debugf("Located %v as %v", target, hostPath) + + containerPath := filepath.Join(d.containerRoot, target) + d.logger.Infof("Selecting %v as %v", hostPath, containerPath) + + mount := Mount{ + HostPath: hostPath, + Path: containerPath, + Options: []string{ + "ro", + "nosuid", + "nodev", + "bind", + }, + } + mounts = append(mounts, mount) + } + + return mounts, nil +} diff --git a/internal/discover/mounts-to-container-path_test.go b/internal/discover/mounts-to-container-path_test.go new file mode 100644 index 00000000..e0a88801 --- /dev/null +++ b/internal/discover/mounts-to-container-path_test.go @@ -0,0 +1,148 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "errors" + "testing" + + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" +) + +func TestMountsToContainerPath(t *testing.T) { + logger, _ := testlog.NewNullLogger() + mountOptions := []string{ + "ro", + "nosuid", + "nodev", + "bind", + } + + testCases := []struct { + description string + required []string + locator lookup.Locator + containerRoot string + expectedMounts []Mount + expectedError error + }{ + { + description: "containerRoot is prepended", + required: []string{"a/path/exists.txt", "another/path/exists.txt"}, + locator: &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + return []string{"/located/root/" + s}, nil + }, + }, + containerRoot: "/container", + expectedMounts: []Mount{ + { + HostPath: "/located/root/a/path/exists.txt", + Path: "/container/a/path/exists.txt", + Options: mountOptions, + }, + { + HostPath: "/located/root/another/path/exists.txt", + Path: "/container/another/path/exists.txt", + Options: mountOptions, + }, + }, + }, + { + description: "duplicate mounts are skipped", + required: []string{"a/path/exists.txt", "another/path/exists.txt"}, + locator: &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + return []string{"/located/root/single.txt"}, nil + }, + }, + containerRoot: "/container", + expectedMounts: []Mount{ + { + HostPath: "/located/root/single.txt", + Path: "/container/a/path/exists.txt", + Options: mountOptions, + }, + }, + }, + { + description: "locator errors are ignored", + required: []string{"a/path/exists.txt"}, + locator: &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + return nil, errors.New("not found") + }, + }, + containerRoot: "/container", + expectedMounts: []Mount{}, + }, + { + description: "not located are ignored", + required: []string{"a/path/exists.txt"}, + locator: &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + return nil, nil + }, + }, + containerRoot: "/container", + expectedMounts: []Mount{}, + }, + { + description: "second candidate is ignored", + required: []string{"a/path/exists.txt"}, + locator: &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + return []string{"/located/root/" + s, "/located2/root/" + s}, nil + }, + }, + containerRoot: "/container", + expectedMounts: []Mount{ + { + HostPath: "/located/root/a/path/exists.txt", + Path: "/container/a/path/exists.txt", + Options: mountOptions, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + d := mountsToContainerPath{ + logger: logger, + locator: tc.locator, + required: tc.required, + containerRoot: tc.containerRoot, + } + + devices, err := d.Devices() + require.NoError(t, err) + require.Empty(t, devices) + + hooks, err := d.Hooks() + require.NoError(t, err) + require.Empty(t, hooks) + + mounts, err := d.Mounts() + require.ErrorIs(t, err, tc.expectedError) + require.ElementsMatch(t, tc.expectedMounts, mounts) + }) + } +} diff --git a/internal/lookup/root/root.go b/internal/lookup/root/root.go index 4a475ccd..1d5e6bd7 100644 --- a/internal/lookup/root/root.go +++ b/internal/lookup/root/root.go @@ -47,6 +47,16 @@ func New(opts ...Option) *Driver { return d } +// Files returns a Locator for arbitrary driver files. +func (r *Driver) Files(opts ...lookup.Option) lookup.Locator { + return lookup.NewFileLocator( + append(opts, + lookup.WithLogger(r.logger), + lookup.WithRoot(r.Root), + )..., + ) +} + // Libraries returns a Locator for driver libraries. func (r *Driver) Libraries() lookup.Locator { return lookup.NewLibraryLocator(