From 563db0e0beaa1da8017617da88403c4dfcfb3bb1 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:05:14 -0700 Subject: [PATCH] nvsandboxutils: Add usage of GetGpuResource and GetFileContent APIs This change adds a new discoverer for Sandboxutils to report the file system paths and associated symbolic links using GetGpuResource and GetFileContent APIs. Both GPU and MIG devices are supported. If the Sandboxutils discoverer fails, the NVML discoverer is used to report the file system information. Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- .golangci.yml | 5 + internal/discover/cache.go | 80 ++++++++ internal/discover/first-valid.go | 72 ++++++++ internal/platform-support/dgpu/dgpu.go | 64 ++++++- .../platform-support/dgpu/nvsandboxutils.go | 131 +++++++++++++ .../dgpu/nvsandboxutils_test.go | 174 ++++++++++++++++++ internal/platform-support/dgpu/options.go | 11 ++ pkg/nvcdi/full-gpu-nvml.go | 1 + pkg/nvcdi/mig-device-nvml.go | 1 + 9 files changed, 536 insertions(+), 3 deletions(-) create mode 100644 internal/discover/cache.go create mode 100644 internal/discover/first-valid.go create mode 100644 internal/platform-support/dgpu/nvsandboxutils.go create mode 100644 internal/platform-support/dgpu/nvsandboxutils_test.go diff --git a/.golangci.yml b/.golangci.yml index b4c2f1c2..825d94c3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -36,3 +36,8 @@ issues: linters: - errcheck text: config.Delete + # RENDERD refers to the Render Device and not the past tense of render. + - path: .*.go + linters: + - misspell + text: "`RENDERD` is a misspelling of `RENDERED`" diff --git a/internal/discover/cache.go b/internal/discover/cache.go new file mode 100644 index 00000000..e9e4b615 --- /dev/null +++ b/internal/discover/cache.go @@ -0,0 +1,80 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import "sync" + +type cache struct { + d Discover + + sync.Mutex + devices []Device + hooks []Hook + mounts []Mount +} + +var _ Discover = (*cache)(nil) + +// WithCache decorates the specified disoverer with a cache. +func WithCache(d Discover) Discover { + if d == nil { + return None{} + } + return &cache{d: d} +} + +func (c *cache) Devices() ([]Device, error) { + c.Lock() + defer c.Unlock() + + if c.devices == nil { + devices, err := c.d.Devices() + if err != nil { + return nil, err + } + c.devices = devices + } + return c.devices, nil +} + +func (c *cache) Hooks() ([]Hook, error) { + c.Lock() + defer c.Unlock() + + if c.hooks == nil { + hooks, err := c.d.Hooks() + if err != nil { + return nil, err + } + c.hooks = hooks + } + return c.hooks, nil +} + +func (c *cache) Mounts() ([]Mount, error) { + c.Lock() + defer c.Unlock() + + if c.mounts == nil { + mounts, err := c.d.Mounts() + if err != nil { + return nil, err + } + c.mounts = mounts + } + return c.mounts, nil +} diff --git a/internal/discover/first-valid.go b/internal/discover/first-valid.go new file mode 100644 index 00000000..36de9204 --- /dev/null +++ b/internal/discover/first-valid.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import "errors" + +type firstOf []Discover + +// FirstValid returns a discoverer that returns the first non-error result from a list of discoverers. +func FirstValid(discoverers ...Discover) Discover { + var f firstOf + for _, d := range discoverers { + if d == nil { + continue + } + f = append(f, d) + } + return f +} + +func (f firstOf) Devices() ([]Device, error) { + var errs error + for _, d := range f { + devices, err := d.Devices() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return devices, nil + } + return nil, errs +} + +func (f firstOf) Hooks() ([]Hook, error) { + var errs error + for _, d := range f { + hooks, err := d.Hooks() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return hooks, nil + } + return nil, errs +} + +func (f firstOf) Mounts() ([]Mount, error) { + var errs error + for _, d := range f { + mounts, err := d.Mounts() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return mounts, nil + } + return nil, nil +} diff --git a/internal/platform-support/dgpu/dgpu.go b/internal/platform-support/dgpu/dgpu.go index b79f6bd4..de411bd6 100644 --- a/internal/platform-support/dgpu/dgpu.go +++ b/internal/platform-support/dgpu/dgpu.go @@ -17,6 +17,8 @@ package dgpu import ( + "errors" + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" @@ -25,22 +27,78 @@ import ( ) // NewForDevice creates a discoverer for the specified Device. +// nvsandboxutils is used for discovery if specified, otherwise NVML is used. func NewForDevice(d device.Device, opts ...Option) (discover.Discover, error) { o := new(opts...) - return o.newNvmlDGPUDiscoverer(&toRequiredInfo{d}) + var discoverers []discover.Discover + var errs error + nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(d) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvsandboxutilsDiscoverer != nil { + discoverers = append(discoverers, nvsandboxutilsDiscoverer) + } + + nvmlDiscoverer, err := o.newNvmlDGPUDiscoverer(&toRequiredInfo{d}) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvmlDiscoverer != nil { + discoverers = append(discoverers, nvmlDiscoverer) + } + + if len(discoverers) == 0 { + return nil, errs + } + + return discover.WithCache( + discover.FirstValid( + discoverers..., + ), + ), nil } -// NewForDevice creates a discoverer for the specified device and its associated MIG device. +// NewForMigDevice creates a discoverer for the specified device and its associated MIG device. +// nvsandboxutils is used for discovery if specified, otherwise NVML is used. func NewForMigDevice(d device.Device, mig device.MigDevice, opts ...Option) (discover.Discover, error) { o := new(opts...) + o.isMigDevice = true - return o.newNvmlMigDiscoverer( + var discoverers []discover.Discover + var errs error + nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(mig) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvsandboxutilsDiscoverer != nil { + discoverers = append(discoverers, nvsandboxutilsDiscoverer) + } + + nvmlDiscoverer, err := o.newNvmlMigDiscoverer( &toRequiredMigInfo{ MigDevice: mig, parent: &toRequiredInfo{d}, }, ) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvmlDiscoverer != nil { + discoverers = append(discoverers, nvmlDiscoverer) + } + + if len(discoverers) == 0 { + return nil, errs + } + + return discover.WithCache( + discover.FirstValid( + discoverers..., + ), + ), nil + } func new(opts ...Option) *options { diff --git a/internal/platform-support/dgpu/nvsandboxutils.go b/internal/platform-support/dgpu/nvsandboxutils.go new file mode 100644 index 00000000..7022deab --- /dev/null +++ b/internal/platform-support/dgpu/nvsandboxutils.go @@ -0,0 +1,131 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package dgpu + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" +) + +type nvsandboxutilsDGPU struct { + lib nvsandboxutils.Interface + uuid string + devRoot string + isMig bool + nvidiaCDIHookPath string + deviceLinks []string +} + +var _ discover.Discover = (*nvsandboxutilsDGPU)(nil) + +type UUIDer interface { + GetUUID() (string, nvml.Return) +} + +func (o *options) newNvsandboxutilsDGPUDiscoverer(d UUIDer) (discover.Discover, error) { + if o.nvsandboxutilslib == nil { + return nil, nil + } + + uuid, nvmlRet := d.GetUUID() + if nvmlRet != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device UUID: %w", nvmlRet) + } + + nvd := nvsandboxutilsDGPU{ + lib: o.nvsandboxutilslib, + uuid: uuid, + devRoot: strings.TrimSuffix(filepath.Clean(o.devRoot), "/dev"), + isMig: o.isMigDevice, + nvidiaCDIHookPath: o.nvidiaCDIHookPath, + } + + return &nvd, nil +} + +func (d *nvsandboxutilsDGPU) Devices() ([]discover.Device, error) { + gpuFileInfos, ret := d.lib.GetGpuResource(d.uuid) + if ret != nvsandboxutils.SUCCESS { + return nil, fmt.Errorf("failed to get GPU resource: %w", ret) + } + + var devices []discover.Device + for _, info := range gpuFileInfos { + switch { + case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD: + if d.isMig { + continue + } + fallthrough + case info.SubType == nvsandboxutils.NV_DEV_NVIDIA, info.SubType == nvsandboxutils.NV_DEV_NVIDIA_CAPS_NVIDIA_CAP: + containerPath := info.Path + if d.devRoot != "/" { + containerPath = strings.TrimPrefix(containerPath, d.devRoot) + } + + // TODO: Extend discover.Device with additional information. + device := discover.Device{ + HostPath: info.Path, + Path: containerPath, + } + devices = append(devices, device) + case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD_SYMLINK, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD_SYMLINK: + if d.isMig { + continue + } + if info.Flags == nvsandboxutils.NV_FILE_FLAG_CONTENT { + targetPath, ret := d.lib.GetFileContent(info.Path) + if ret != nvsandboxutils.SUCCESS { + return nil, fmt.Errorf("failed to get symlink: %w", ret) + } + d.deviceLinks = append(d.deviceLinks, fmt.Sprintf("%v::%v", targetPath, info.Path)) + } + } + } + + return devices, nil +} + +// Hooks returns a hook to create the by-path symlinks for the discovered devices. +func (d *nvsandboxutilsDGPU) Hooks() ([]discover.Hook, error) { + if len(d.deviceLinks) == 0 { + return nil, nil + } + + var args []string + for _, l := range d.deviceLinks { + args = append(args, "--link", l) + } + + hook := discover.CreateNvidiaCDIHook( + d.nvidiaCDIHookPath, + "create-symlinks", + args..., + ) + + return []discover.Hook{hook}, nil +} + +func (d *nvsandboxutilsDGPU) Mounts() ([]discover.Mount, error) { + return nil, nil +} diff --git a/internal/platform-support/dgpu/nvsandboxutils_test.go b/internal/platform-support/dgpu/nvsandboxutils_test.go new file mode 100644 index 00000000..fb9df187 --- /dev/null +++ b/internal/platform-support/dgpu/nvsandboxutils_test.go @@ -0,0 +1,174 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package dgpu + +import ( + "testing" + + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" + "github.com/NVIDIA/go-nvml/pkg/nvml" + mocknvml "github.com/NVIDIA/go-nvml/pkg/nvml/mock" + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" + mocknvsandboxutils "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils/mock" +) + +func TestNewNvsandboxutilsDGPUDiscoverer(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + nvmllib := &mocknvml.Interface{} + devicelib := device.New( + nvmllib, + ) + + testCases := []struct { + description string + devRoot string + device nvml.Device + nvsandboxutils nvsandboxutils.Interface + expectedError error + expectedDevices []discover.Device + expectedHooks []discover.Hook + expectedMounts []discover.Mount + }{ + { + description: "detects host devices", + device: &mocknvml.Device{ + GetUUIDFunc: func() (string, nvml.Return) { + return "GPU-1234", nvml.SUCCESS + }, + }, + nvsandboxutils: &mocknvsandboxutils.Interface{ + GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + infos := []nvsandboxutils.GpuFileInfo{ + { + Path: "/dev/nvidia0", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidiactl", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidia-uvm", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidia-uvm-tools", + Type: nvsandboxutils.NV_DEV, + }, + } + return infos, nvsandboxutils.SUCCESS + }, + }, + expectedDevices: []discover.Device{ + { + Path: "/dev/nvidia0", + HostPath: "/dev/nvidia0", + }, + { + Path: "/dev/nvidiactl", + HostPath: "/dev/nvidiactl", + }, + { + Path: "/dev/nvidia-uvm", + HostPath: "/dev/nvidia-uvm", + }, + { + Path: "/dev/nvidia-uvm-tools", + HostPath: "/dev/nvidia-uvm-tools", + }, + }, + }, + { + description: "detects container devices", + devRoot: "/some/root", + device: &mocknvml.Device{ + GetUUIDFunc: func() (string, nvml.Return) { + return "GPU-1234", nvml.SUCCESS + }, + }, + nvsandboxutils: &mocknvsandboxutils.Interface{ + GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + infos := []nvsandboxutils.GpuFileInfo{ + { + Path: "/some/root/dev/nvidia0", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidiactl", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidia-uvm", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidia-uvm-tools", + Type: nvsandboxutils.NV_DEV, + }, + } + return infos, nvsandboxutils.SUCCESS + }, + }, + expectedDevices: []discover.Device{ + { + Path: "/dev/nvidia0", + HostPath: "/some/root/dev/nvidia0", + }, + { + Path: "/dev/nvidiactl", + HostPath: "/some/root/dev/nvidiactl", + }, + { + Path: "/dev/nvidia-uvm", + HostPath: "/some/root/dev/nvidia-uvm", + }, + { + Path: "/dev/nvidia-uvm-tools", + HostPath: "/some/root/dev/nvidia-uvm-tools", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + o := &options{ + logger: logger, + devRoot: tc.devRoot, + nvsandboxutilslib: tc.nvsandboxutils, + } + + device, err := devicelib.NewDevice(tc.device) + require.NoError(t, err) + + d, err := o.newNvsandboxutilsDGPUDiscoverer(device) + require.ErrorIs(t, err, tc.expectedError) + + devices, _ := d.Devices() + require.EqualValues(t, tc.expectedDevices, devices) + hooks, _ := d.Hooks() + require.EqualValues(t, tc.expectedHooks, hooks) + mounts, _ := d.Mounts() + require.EqualValues(t, tc.expectedMounts, mounts) + }) + } +} diff --git a/internal/platform-support/dgpu/options.go b/internal/platform-support/dgpu/options.go index 41e4d7a9..2fd1c01b 100644 --- a/internal/platform-support/dgpu/options.go +++ b/internal/platform-support/dgpu/options.go @@ -19,6 +19,7 @@ package dgpu import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" ) type options struct { @@ -26,10 +27,13 @@ type options struct { devRoot string nvidiaCDIHookPath string + isMigDevice bool // migCaps stores the MIG capabilities for the system. // If MIG is not available, this is nil. migCaps nvcaps.MigCaps migCapsError error + + nvsandboxutilslib nvsandboxutils.Interface } type Option func(*options) @@ -61,3 +65,10 @@ func WithMIGCaps(migCaps nvcaps.MigCaps) Option { l.migCaps = migCaps } } + +// WithNvsandboxuitilsLib sets the nvsandboxutils library implementation. +func WithNvsandboxuitilsLib(nvsandboxutilslib nvsandboxutils.Interface) Option { + return func(l *options) { + l.nvsandboxutilslib = nvsandboxutilslib + } +} diff --git a/pkg/nvcdi/full-gpu-nvml.go b/pkg/nvcdi/full-gpu-nvml.go index 881d102d..003515ca 100644 --- a/pkg/nvcdi/full-gpu-nvml.go +++ b/pkg/nvcdi/full-gpu-nvml.go @@ -72,6 +72,7 @@ func (l *nvmllib) newFullGPUDiscoverer(d device.Device) (discover.Discover, erro dgpu.WithDevRoot(l.devRoot), dgpu.WithLogger(l.logger), dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath), + dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib), ) if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err) diff --git a/pkg/nvcdi/mig-device-nvml.go b/pkg/nvcdi/mig-device-nvml.go index 91fe879c..5c1a504c 100644 --- a/pkg/nvcdi/mig-device-nvml.go +++ b/pkg/nvcdi/mig-device-nvml.go @@ -55,6 +55,7 @@ func (l *nvmllib) GetMIGDeviceEdits(parent device.Device, mig device.MigDevice) dgpu.WithDevRoot(l.devRoot), dgpu.WithLogger(l.logger), dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath), + dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib), ) if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err)