From e78a4f5eacbf03723b45cab2c9c88b2e95f9569f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 10 May 2023 14:23:05 +0200 Subject: [PATCH] Add csv mode to nvcdi api Signed-off-by: Evan Lezar --- pkg/nvcdi/api.go | 3 ++ pkg/nvcdi/lib-csv.go | 87 ++++++++++++++++++++++++++++++++++++++++++++ pkg/nvcdi/lib.go | 11 ++++++ pkg/nvcdi/options.go | 7 ++++ 4 files changed, 108 insertions(+) create mode 100644 pkg/nvcdi/lib-csv.go diff --git a/pkg/nvcdi/api.go b/pkg/nvcdi/api.go index 4f145638..68bfd845 100644 --- a/pkg/nvcdi/api.go +++ b/pkg/nvcdi/api.go @@ -36,6 +36,9 @@ const ( ModeGds = "gds" // ModeMofed configures the CDI spec generator to generate a MOFED spec. ModeMofed = "mofed" + // ModeCSV configures the CDI spec generator to generate a spec based on the contents of CSV + // mountspec files. + ModeCSV = "csv" ) // Interface defines the API for the nvcdi package diff --git a/pkg/nvcdi/lib-csv.go b/pkg/nvcdi/lib-csv.go new file mode 100644 index 00000000..127c4beb --- /dev/null +++ b/pkg/nvcdi/lib-csv.go @@ -0,0 +1,87 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvcdi + +import ( + "fmt" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover/tegra" + "github.com/NVIDIA/nvidia-container-toolkit/internal/edits" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec" + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/container-orchestrated-devices/container-device-interface/specs-go" + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device" +) + +type csvlib nvcdilib + +var _ Interface = (*csvlib)(nil) + +// GetSpec should not be called for wsllib +func (l *csvlib) GetSpec() (spec.Interface, error) { + return nil, fmt.Errorf("Unexpected call to csvlib.GetSpec()") +} + +// GetAllDeviceSpecs returns the device specs for all available devices. +func (l *csvlib) GetAllDeviceSpecs() ([]specs.Device, error) { + d, err := tegra.New( + tegra.WithLogger(l.logger), + tegra.WithDriverRoot(l.driverRoot), + tegra.WithNVIDIACTKPath(l.nvidiaCTKPath), + tegra.WithCSVFiles(l.csvFiles), + ) + if err != nil { + return nil, fmt.Errorf("failed to create discoverer for CSV files: %v", err) + } + e, err := edits.FromDiscoverer(d) + if err != nil { + return nil, fmt.Errorf("failed to create container edits for CSV files: %v", err) + } + + deviceSpec := specs.Device{ + Name: "all", + ContainerEdits: *e.ContainerEdits, + } + return []specs.Device{deviceSpec}, nil +} + +// GetCommonEdits generates a CDI specification that can be used for ANY devices +func (l *csvlib) GetCommonEdits() (*cdi.ContainerEdits, error) { + d := discover.None{} + return edits.FromDiscoverer(d) +} + +// GetGPUDeviceEdits generates a CDI specification that can be used for GPU devices +func (l *csvlib) GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error) { + return nil, fmt.Errorf("GetGPUDeviceEdits is not supported for CSV files") +} + +// GetGPUDeviceSpecs returns the CDI device specs for the full GPU represented by 'device'. +func (l *csvlib) GetGPUDeviceSpecs(i int, d device.Device) (*specs.Device, error) { + return nil, fmt.Errorf("GetGPUDeviceSpecs is not supported for CSV files") +} + +// GetMIGDeviceEdits generates a CDI specification that can be used for MIG devices +func (l *csvlib) GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error) { + return nil, fmt.Errorf("GetMIGDeviceEdits is not supported for CSV files") +} + +// GetMIGDeviceSpecs returns the CDI device specs for the full MIG represented by 'device'. +func (l *csvlib) GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error) { + return nil, fmt.Errorf("GetMIGDeviceSpecs is not supported for CSV files") +} diff --git a/pkg/nvcdi/lib.go b/pkg/nvcdi/lib.go index fd5e2e54..87f0299e 100644 --- a/pkg/nvcdi/lib.go +++ b/pkg/nvcdi/lib.go @@ -45,6 +45,8 @@ type nvcdilib struct { driverRoot string nvidiaCTKPath string + csvFiles []string + vendor string class string @@ -80,6 +82,15 @@ func New(opts ...Option) (Interface, error) { var lib Interface switch l.resolveMode() { + case ModeCSV: + if len(l.csvFiles) == 0 { + l.csvFiles = []string{ + "/etc/nvidia-container-runtime/host-files-for-container.d/l4t.csv", + "/etc/nvidia-container-runtime/host-files-for-container.d/drivers.csv", + "/etc/nvidia-container-runtime/host-files-for-container.d/devices.csv", + } + } + lib = (*csvlib)(l) case ModeManagement: if l.vendor == "" { l.vendor = "management.nvidia.com" diff --git a/pkg/nvcdi/options.go b/pkg/nvcdi/options.go index 254c6e0e..6baa1b52 100644 --- a/pkg/nvcdi/options.go +++ b/pkg/nvcdi/options.go @@ -96,3 +96,10 @@ func WithMergedDeviceOptions(opts ...transform.MergedDeviceOption) Option { o.mergedDeviceOptions = opts } } + +// WithCSVFiles sets the CSV files for the library +func WithCSVFiles(csvFiles []string) Option { + return func(o *nvcdilib) { + o.csvFiles = csvFiles + } +}