diff --git a/.golangci.yml b/.golangci.yml index b4c2f1c2..825d94c3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -36,3 +36,8 @@ issues: linters: - errcheck text: config.Delete + # RENDERD refers to the Render Device and not the past tense of render. + - path: .*.go + linters: + - misspell + text: "`RENDERD` is a misspelling of `RENDERED`" diff --git a/internal/discover/cache.go b/internal/discover/cache.go new file mode 100644 index 00000000..e9e4b615 --- /dev/null +++ b/internal/discover/cache.go @@ -0,0 +1,80 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import "sync" + +type cache struct { + d Discover + + sync.Mutex + devices []Device + hooks []Hook + mounts []Mount +} + +var _ Discover = (*cache)(nil) + +// WithCache decorates the specified disoverer with a cache. +func WithCache(d Discover) Discover { + if d == nil { + return None{} + } + return &cache{d: d} +} + +func (c *cache) Devices() ([]Device, error) { + c.Lock() + defer c.Unlock() + + if c.devices == nil { + devices, err := c.d.Devices() + if err != nil { + return nil, err + } + c.devices = devices + } + return c.devices, nil +} + +func (c *cache) Hooks() ([]Hook, error) { + c.Lock() + defer c.Unlock() + + if c.hooks == nil { + hooks, err := c.d.Hooks() + if err != nil { + return nil, err + } + c.hooks = hooks + } + return c.hooks, nil +} + +func (c *cache) Mounts() ([]Mount, error) { + c.Lock() + defer c.Unlock() + + if c.mounts == nil { + mounts, err := c.d.Mounts() + if err != nil { + return nil, err + } + c.mounts = mounts + } + return c.mounts, nil +} diff --git a/internal/discover/first-valid.go b/internal/discover/first-valid.go new file mode 100644 index 00000000..36de9204 --- /dev/null +++ b/internal/discover/first-valid.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import "errors" + +type firstOf []Discover + +// FirstValid returns a discoverer that returns the first non-error result from a list of discoverers. +func FirstValid(discoverers ...Discover) Discover { + var f firstOf + for _, d := range discoverers { + if d == nil { + continue + } + f = append(f, d) + } + return f +} + +func (f firstOf) Devices() ([]Device, error) { + var errs error + for _, d := range f { + devices, err := d.Devices() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return devices, nil + } + return nil, errs +} + +func (f firstOf) Hooks() ([]Hook, error) { + var errs error + for _, d := range f { + hooks, err := d.Hooks() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return hooks, nil + } + return nil, errs +} + +func (f firstOf) Mounts() ([]Mount, error) { + var errs error + for _, d := range f { + mounts, err := d.Mounts() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return mounts, nil + } + return nil, nil +} diff --git a/internal/nvsandboxutils/api.go b/internal/nvsandboxutils/api.go new file mode 100644 index 00000000..6275a5c2 --- /dev/null +++ b/internal/nvsandboxutils/api.go @@ -0,0 +1,45 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +// libraryOptions hold the parameters than can be set by a LibraryOption +type libraryOptions struct { + path string + flags int +} + +// LibraryOption represents a functional option to configure the underlying nvsandboxutils library +type LibraryOption func(*libraryOptions) + +// WithLibraryPath provides an option to set the library name to be used by the nvsandboxutils library. +func WithLibraryPath(path string) LibraryOption { + return func(o *libraryOptions) { + o.path = path + } +} + +// SetLibraryOptions applies the specified options to the nvsandboxutils library. +// If this is called when a library is already loaded, an error is raised. +func SetLibraryOptions(opts ...LibraryOption) error { + libnvsandboxutils.Lock() + defer libnvsandboxutils.Unlock() + if libnvsandboxutils.refcount != 0 { + return errLibraryAlreadyLoaded + } + libnvsandboxutils.init(opts...) + return nil +} diff --git a/internal/nvsandboxutils/cgo_helpers.h b/internal/nvsandboxutils/cgo_helpers.h new file mode 100644 index 00000000..23b3c256 --- /dev/null +++ b/internal/nvsandboxutils/cgo_helpers.h @@ -0,0 +1,25 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +#include "nvsandboxutils.h" +#include +#pragma once + +#define __CGOGEN 1 + diff --git a/internal/nvsandboxutils/cgo_helpers_static.go b/internal/nvsandboxutils/cgo_helpers_static.go new file mode 100644 index 00000000..5924d622 --- /dev/null +++ b/internal/nvsandboxutils/cgo_helpers_static.go @@ -0,0 +1,38 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +var cgoAllocsUnknown = new(struct{}) + +func clen(n []byte) int { + for i := 0; i < len(n); i++ { + if n[i] == 0 { + return i + } + } + return len(n) +} + +// Creates an int8 array of fixed input length to store the Go string. +// TODO: Add error check if input string has a length greater than INPUT_LENGTH +func convertStringToFixedArray(str string) [INPUT_LENGTH]int8 { + var output [INPUT_LENGTH]int8 + for i, s := range str { + output[i] = int8(s) + } + return output +} diff --git a/internal/nvsandboxutils/const.go b/internal/nvsandboxutils/const.go new file mode 100644 index 00000000..9e8cdf3f --- /dev/null +++ b/internal/nvsandboxutils/const.go @@ -0,0 +1,156 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package nvsandboxutils + +/* +#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup +#include "nvsandboxutils.h" +#include +#include "cgo_helpers.h" +*/ +import "C" + +const ( + // INPUT_LENGTH as defined in nvsandboxutils/nvsandboxutils.h + INPUT_LENGTH = 256 + // MAX_FILE_PATH as defined in nvsandboxutils/nvsandboxutils.h + MAX_FILE_PATH = 256 + // MAX_NAME_LENGTH as defined in nvsandboxutils/nvsandboxutils.h + MAX_NAME_LENGTH = 256 +) + +// Ret as declared in nvsandboxutils/nvsandboxutils.h +type Ret int32 + +// Ret enumeration from nvsandboxutils/nvsandboxutils.h +const ( + SUCCESS Ret = iota + ERROR_UNINITIALIZED Ret = 1 + ERROR_NOT_SUPPORTED Ret = 2 + ERROR_INVALID_ARG Ret = 3 + ERROR_INSUFFICIENT_SIZE Ret = 4 + ERROR_VERSION_NOT_SUPPORTED Ret = 5 + ERROR_LIBRARY_LOAD Ret = 6 + ERROR_FUNCTION_NOT_FOUND Ret = 7 + ERROR_DEVICE_NOT_FOUND Ret = 8 + ERROR_NVML_LIB_CALL Ret = 9 + ERROR_OUT_OF_MEMORY Ret = 10 + ERROR_FILEPATH_NOT_FOUND Ret = 11 + ERROR_UNKNOWN Ret = 65535 +) + +// LogLevel as declared in nvsandboxutils/nvsandboxutils.h +type LogLevel int32 + +// LogLevel enumeration from nvsandboxutils/nvsandboxutils.h +const ( + LOG_LEVEL_FATAL LogLevel = iota + LOG_LEVEL_ERROR LogLevel = 1 + LOG_LEVEL_WARN LogLevel = 2 + LOG_LEVEL_DEBUG LogLevel = 3 + LOG_LEVEL_INFO LogLevel = 4 + LOG_LEVEL_NONE LogLevel = 65535 +) + +// RootfsInputType as declared in nvsandboxutils/nvsandboxutils.h +type RootfsInputType int32 + +// RootfsInputType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_ROOTFS_DEFAULT RootfsInputType = iota + NV_ROOTFS_PATH RootfsInputType = 1 + NV_ROOTFS_PID RootfsInputType = 2 +) + +// FileType as declared in nvsandboxutils/nvsandboxutils.h +type FileType int32 + +// FileType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_DEV FileType = iota + NV_PROC FileType = 1 + NV_SYS FileType = 2 +) + +// FileSystemSubType as declared in nvsandboxutils/nvsandboxutils.h +type FileSystemSubType int32 + +// FileSystemSubType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_DEV_NVIDIA FileSystemSubType = iota + NV_DEV_DRI_CARD FileSystemSubType = 1 + NV_DEV_DRI_RENDERD FileSystemSubType = 2 + NV_DEV_DRI_CARD_SYMLINK FileSystemSubType = 3 + NV_DEV_DRI_RENDERD_SYMLINK FileSystemSubType = 4 + NV_DEV_NVIDIA_UVM FileSystemSubType = 5 + NV_DEV_NVIDIA_UVM_TOOLS FileSystemSubType = 6 + NV_DEV_NVIDIA_MODESET FileSystemSubType = 7 + NV_DEV_NVIDIA_CTL FileSystemSubType = 8 + NV_DEV_GDRDRV FileSystemSubType = 9 + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP FileSystemSubType = 10 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID FileSystemSubType = 11 + NV_PROC_DRIVER_NVIDIA_GPUS FileSystemSubType = 12 + NV_PROC_NVIDIA_PARAMS FileSystemSubType = 13 + NV_PROC_NVIDIA_CAPS_MIG_MINORS FileSystemSubType = 14 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU FileSystemSubType = 15 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES FileSystemSubType = 16 + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS FileSystemSubType = 17 + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID FileSystemSubType = 18 + NV_SYS_MODULE_NVIDIA_DRIVER FileSystemSubType = 19 + NV_NUM_SUBTYPE FileSystemSubType = 20 +) + +// FileModule as declared in nvsandboxutils/nvsandboxutils.h +type FileModule int32 + +// FileModule enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_GPU FileModule = iota + NV_MIG FileModule = 1 + NV_DRIVER_NVIDIA FileModule = 2 + NV_DRIVER_NVIDIA_UVM FileModule = 3 + NV_DRIVER_NVIDIA_MODESET FileModule = 4 + NV_DRIVER_GDRDRV FileModule = 5 + NV_SYSTEM FileModule = 6 +) + +// FileFlag as declared in nvsandboxutils/nvsandboxutils.h +type FileFlag int32 + +// FileFlag enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_FILE_FLAG_HINT FileFlag = 1 + NV_FILE_FLAG_MASKOUT FileFlag = 2 + NV_FILE_FLAG_CONTENT FileFlag = 4 + NV_FILE_FLAG_DEPRECTATED FileFlag = 8 + NV_FILE_FLAG_CANDIDATES FileFlag = 16 +) + +// GpuInputType as declared in nvsandboxutils/nvsandboxutils.h +type GpuInputType int32 + +// GpuInputType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_GPU_INPUT_GPU_UUID GpuInputType = iota + NV_GPU_INPUT_MIG_UUID GpuInputType = 1 + NV_GPU_INPUT_PCI_ID GpuInputType = 2 + NV_GPU_INPUT_PCI_INDEX GpuInputType = 3 +) diff --git a/internal/nvsandboxutils/doc.go b/internal/nvsandboxutils/doc.go new file mode 100644 index 00000000..231c68c2 --- /dev/null +++ b/internal/nvsandboxutils/doc.go @@ -0,0 +1,23 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +/* +Package NVSANDBOXUTILS bindings +*/ +package nvsandboxutils diff --git a/internal/nvsandboxutils/dynamicLibrary_mock.go b/internal/nvsandboxutils/dynamicLibrary_mock.go new file mode 100644 index 00000000..a22e5669 --- /dev/null +++ b/internal/nvsandboxutils/dynamicLibrary_mock.go @@ -0,0 +1,157 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package nvsandboxutils + +import ( + "sync" +) + +// Ensure, that dynamicLibraryMock does implement dynamicLibrary. +// If this is not the case, regenerate this file with moq. +var _ dynamicLibrary = &dynamicLibraryMock{} + +// dynamicLibraryMock is a mock implementation of dynamicLibrary. +// +// func TestSomethingThatUsesdynamicLibrary(t *testing.T) { +// +// // make and configure a mocked dynamicLibrary +// mockeddynamicLibrary := &dynamicLibraryMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// LookupFunc: func(s string) error { +// panic("mock out the Lookup method") +// }, +// OpenFunc: func() error { +// panic("mock out the Open method") +// }, +// } +// +// // use mockeddynamicLibrary in code that requires dynamicLibrary +// // and then make assertions. +// +// } +type dynamicLibraryMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // LookupFunc mocks the Lookup method. + LookupFunc func(s string) error + + // OpenFunc mocks the Open method. + OpenFunc func() error + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // Lookup holds details about calls to the Lookup method. + Lookup []struct { + // S is the s argument value. + S string + } + // Open holds details about calls to the Open method. + Open []struct { + } + } + lockClose sync.RWMutex + lockLookup sync.RWMutex + lockOpen sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *dynamicLibraryMock) Close() error { + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + if mock.CloseFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockeddynamicLibrary.CloseCalls()) +func (mock *dynamicLibraryMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// Lookup calls LookupFunc. +func (mock *dynamicLibraryMock) Lookup(s string) error { + callInfo := struct { + S string + }{ + S: s, + } + mock.lockLookup.Lock() + mock.calls.Lookup = append(mock.calls.Lookup, callInfo) + mock.lockLookup.Unlock() + if mock.LookupFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.LookupFunc(s) +} + +// LookupCalls gets all the calls that were made to Lookup. +// Check the length with: +// +// len(mockeddynamicLibrary.LookupCalls()) +func (mock *dynamicLibraryMock) LookupCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockLookup.RLock() + calls = mock.calls.Lookup + mock.lockLookup.RUnlock() + return calls +} + +// Open calls OpenFunc. +func (mock *dynamicLibraryMock) Open() error { + callInfo := struct { + }{} + mock.lockOpen.Lock() + mock.calls.Open = append(mock.calls.Open, callInfo) + mock.lockOpen.Unlock() + if mock.OpenFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.OpenFunc() +} + +// OpenCalls gets all the calls that were made to Open. +// Check the length with: +// +// len(mockeddynamicLibrary.OpenCalls()) +func (mock *dynamicLibraryMock) OpenCalls() []struct { +} { + var calls []struct { + } + mock.lockOpen.RLock() + calls = mock.calls.Open + mock.lockOpen.RUnlock() + return calls +} diff --git a/internal/nvsandboxutils/gen/generate-bindings.sh b/internal/nvsandboxutils/gen/generate-bindings.sh new file mode 100755 index 00000000..e32641a3 --- /dev/null +++ b/internal/nvsandboxutils/gen/generate-bindings.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file generates bindings for nvsandboxutils by calling c-for-go. + +set -x -e + +PWD=$(pwd) +GEN_DIR="$PWD/gen" +PKG_DIR="$PWD" +GEN_BINDINGS_DIR="$GEN_DIR/nvsandboxutils" +PKG_BINDINGS_DIR="$PKG_DIR" + +SOURCES=$(find "$GEN_BINDINGS_DIR" -type f) + +mkdir -p "$PKG_BINDINGS_DIR" + +cp "$GEN_BINDINGS_DIR/nvsandboxutils.h" "$PKG_BINDINGS_DIR/nvsandboxutils.h" +spatch --in-place --very-quiet --sp-file "$GEN_BINDINGS_DIR/anonymous_structs.cocci" "$PKG_BINDINGS_DIR/nvsandboxutils.h" > /dev/null + +echo "Generating the bindings..." +c-for-go -out "$PKG_DIR/.." "$GEN_BINDINGS_DIR/nvsandboxutils.yml" +cd "$PKG_BINDINGS_DIR" +go tool cgo -godefs types.go > types_gen.go +go fmt types_gen.go +cd - > /dev/null +rm -rf "$PKG_BINDINGS_DIR/cgo_helpers.go" "$PKG_BINDINGS_DIR/types.go" "$PKG_BINDINGS_DIR/_obj" +go run "$GEN_BINDINGS_DIR/generateapi.go" --sourceDir "$PKG_BINDINGS_DIR" --output "$PKG_BINDINGS_DIR/zz_generated.api.go" +# go fmt "$PKG_BINDINGS_DIR" + +SED_SEARCH_STRING='// WARNING: This file has automatically been generated on' +SED_REPLACE_STRING='// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.' +grep -l -R "$SED_SEARCH_STRING" "$PKG_DIR" | grep -v "/gen/" | xargs sed -i -E "s#$SED_SEARCH_STRING.*\$#$SED_REPLACE_STRING#g" + +SED_SEARCH_STRING='// (.*) nvsandboxutils/nvsandboxutils.h:[0-9]+' +SED_REPLACE_STRING='// \1 nvsandboxutils/nvsandboxutils.h' +grep -l -RE "$SED_SEARCH_STRING" "$PKG_DIR" | grep -v "/gen/" | xargs sed -i -E "s#$SED_SEARCH_STRING\$#$SED_REPLACE_STRING#g" + diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci b/internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci new file mode 100644 index 00000000..c8028fb5 --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci @@ -0,0 +1,100 @@ +@patch@ +type WRAPPER_TYPE; +field list FIELDS; +identifier V; +expression E; +fresh identifier ST = "nvSandboxUtilsGenerated_struct___"; +fresh identifier TEMP_VAR = "nvSandboxUtilsGenerated_variable___" ## V; +@@ + +++ struct ST { +++ WRAPPER_TYPE TEMP_VAR; +++ FIELDS +++ }; ++ + +WRAPPER_TYPE +{ + ... +( +- struct { +- FIELDS +- } V[E]; ++ struct ST V[E]; + +| + +- struct { +- FIELDS +- } V; ++ struct ST V; +) + ... +}; + +@capture@ +type WRAPPER_TYPE; +identifier TEMP_VAR; +identifier ST =~ "^nvSandboxUtilsGenerated_struct___"; +@@ + +struct ST { + WRAPPER_TYPE TEMP_VAR; + ... +}; + +@script:python concat@ +WRAPPER_TYPE << capture.WRAPPER_TYPE; +TEMP_VAR << capture.TEMP_VAR; +ST << capture.ST; +T; +@@ + +def removePrefix(string, prefix): + if string.startswith(prefix): + return string[len(prefix):] + return string + +def removeSuffix(string, suffix): + if string.endswith(suffix): + return string[:-len(suffix)] + return string + +WRAPPER_TYPE = removeSuffix(WRAPPER_TYPE, "_t") +TEMP_VAR = removePrefix(TEMP_VAR, "nvSandboxUtilsGenerated_variable___") +coccinelle.T = cocci.make_type(WRAPPER_TYPE + TEMP_VAR[0].upper() + TEMP_VAR[1:] + "_t") + +@add_typedef@ +identifier capture.ST; +type concat.T; +type WRAPPER_TYPE; +identifier TEMP_VAR; +@@ + +- struct ST { ++ typedef struct { +- WRAPPER_TYPE TEMP_VAR; + ... +- }; ++ } T; + +@update@ +identifier capture.ST; +type concat.T; +identifier V; +expression E; +type WRAPPER_TYPE; +@@ + +WRAPPER_TYPE +{ + ... +( +- struct ST V[E]; ++ T V[E]; +| +- struct ST V; ++ T V; +) + ... +}; diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go b/internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go new file mode 100644 index 00000000..104b824b --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go @@ -0,0 +1,389 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "unicode" +) + +type GeneratableInterfacePoperties struct { + Type string + Interface string + Exclude []string + PackageMethodsAliasedFrom string +} + +var GeneratableInterfaces = []GeneratableInterfacePoperties{ + { + Type: "library", + Interface: "Interface", + PackageMethodsAliasedFrom: "libnvsandboxutils", + }, +} + +func main() { + sourceDir := flag.String("sourceDir", "", "Path to the source directory for all go files") + output := flag.String("output", "", "Path to the output file (default: stdout)") + flag.Parse() + + // Check if required flags are provided + if *sourceDir == "" { + flag.Usage() + return + } + + writer, closer, err := getWriter(*output) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + defer func() { + _ = closer() + }() + + header, err := generateHeader() + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, header) + + for i, p := range GeneratableInterfaces { + if p.PackageMethodsAliasedFrom != "" { + comment, err := generatePackageMethodsComment(p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, comment) + + output, err := generatePackageMethods(*sourceDir, p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprintf(writer, "%s\n", output) + } + + comment, err := generateInterfaceComment(p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, comment) + + output, err := generateInterface(*sourceDir, p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, output) + + if i < (len(GeneratableInterfaces) - 1) { + fmt.Fprint(writer, "\n") + } + } +} + +func getWriter(outputFile string) (io.Writer, func() error, error) { + if outputFile == "" { + return os.Stdout, func() error { return nil }, nil + } + + file, err := os.Create(outputFile) + if err != nil { + return nil, nil, err + } + + return file, file.Close, nil +} + +func generateHeader() (string, error) { + lines := []string{ + "/**", + "# Copyright 2024 NVIDIA CORPORATION", + "#", + "# Licensed under the Apache License, Version 2.0 (the \"License\");", + "# you may not use this file except in compliance with the License.", + "# You may obtain a copy of the License at", + "#", + "# http://www.apache.org/licenses/LICENSE-2.0", + "#", + "# Unless required by applicable law or agreed to in writing, software", + "# distributed under the License is distributed on an \"AS IS\" BASIS,", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "# See the License for the specific language governing permissions and", + "# limitations under the License.", + "**/", + "", + "// Generated Code; DO NOT EDIT.", + "", + "package nvsandboxutils", + "", + "", + } + return strings.Join(lines, "\n"), nil +} + +func generatePackageMethodsComment(input GeneratableInterfacePoperties) (string, error) { + commentFmt := []string{ + "// The variables below represent package level methods from the %s type.", + } + + var signature strings.Builder + comment := strings.Join(commentFmt, "\n") + comment = fmt.Sprintf(comment, input.Type) + signature.WriteString(fmt.Sprintf("%s\n", comment)) + return signature.String(), nil +} + +func generateInterfaceComment(input GeneratableInterfacePoperties) (string, error) { + commentFmt := []string{ + "// %s represents the interface for the %s type.", + "//", + "//go:generate moq -out mock/%s.go -pkg mock . %s:%s", + } + + var signature strings.Builder + comment := strings.Join(commentFmt, "\n") + comment = fmt.Sprintf(comment, input.Interface, input.Type, strings.ToLower(input.Interface), input.Interface, input.Interface) + signature.WriteString(fmt.Sprintf("%s\n", comment)) + return signature.String(), nil +} + +func generatePackageMethods(sourceDir string, input GeneratableInterfacePoperties) (string, error) { + var signature strings.Builder + + signature.WriteString("var (\n") + + methods, err := extractMethodsFromPackage(sourceDir, input) + if err != nil { + return "", err + } + + for _, method := range methods { + name := method.Name.Name + formatted := fmt.Sprintf("\t%s = %s.%s\n", name, input.PackageMethodsAliasedFrom, name) + signature.WriteString(formatted) + } + + signature.WriteString(")\n") + + return signature.String(), nil +} + +func generateInterface(sourceDir string, input GeneratableInterfacePoperties) (string, error) { + var signature strings.Builder + + signature.WriteString(fmt.Sprintf("type %s interface {\n", input.Interface)) + + methods, err := extractMethodsFromPackage(sourceDir, input) + if err != nil { + return "", err + } + + for _, method := range methods { + formatted := fmt.Sprintf("\t%s\n", formatMethodSignature(method)) + signature.WriteString(formatted) + } + + signature.WriteString("}\n") + + return signature.String(), nil +} + +func getGoFiles(sourceDir string) (map[string][]byte, error) { + gofiles := make(map[string][]byte) + + err := filepath.WalkDir(sourceDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".go" { + return nil + } + + content, err := os.ReadFile(path) + if err != nil { + return err + } + + gofiles[path] = content + + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking %s: %w", sourceDir, err) + } + + return gofiles, nil +} + +func extractMethodsFromPackage(sourceDir string, input GeneratableInterfacePoperties) ([]*ast.FuncDecl, error) { + gofiles, err := getGoFiles(sourceDir) + if err != nil { + return nil, err + } + + var methods []*ast.FuncDecl + for file, content := range gofiles { + m, err := extractMethods(file, content, input) + if err != nil { + return nil, err + } + methods = append(methods, m...) + } + + sort.Slice(methods, func(i, j int) bool { + return methods[i].Name.Name < methods[j].Name.Name + }) + + return methods, nil +} + +func extractMethods(sourceFile string, sourceContent []byte, input GeneratableInterfacePoperties) ([]*ast.FuncDecl, error) { + // Parse source file + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, sourceFile, sourceContent, parser.ParseComments) + if err != nil { + return nil, err + } + + // Traverse AST to find type declarations and associated methods + var methods []*ast.FuncDecl + for _, decl := range node.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + + // Check if the function is a method associated with the specified type + if receiverType := funcDecl.Recv; receiverType != nil { + var ident *ast.Ident + + for _, field := range receiverType.List { + switch fieldType := field.Type.(type) { + case *ast.Ident: + ident = fieldType + case *ast.StarExpr: + // Update ident if it's a *ast.StarExpr + if newIdent, ok := fieldType.X.(*ast.Ident); ok { + // If the inner type is an *ast.Ident, update ident + ident = newIdent + } + } + + // No identifier found + if ident == nil { + continue + } + + // Identifier is not the one we are looking for + if ident.Name != input.Type { + continue + } + + // Ignore non-public methods + if !isPublic(funcDecl.Name.Name) { + continue + } + + // Ignore method in the exclude list + if slices.Contains(input.Exclude, funcDecl.Name.Name) { + continue + } + + methods = append(methods, funcDecl) + } + } + } + + return methods, nil +} + +func formatMethodSignature(decl *ast.FuncDecl) string { + var signature strings.Builder + + // Write method name + signature.WriteString(decl.Name.Name) + signature.WriteString("(") + + // Write parameters + if decl.Type.Params != nil { + for i, param := range decl.Type.Params.List { + if i > 0 { + signature.WriteString(", ") + } + signature.WriteString(formatFieldList(param)) + } + } + + signature.WriteString(")") + + // Write return types + if decl.Type.Results != nil { + signature.WriteString(" ") + if len(decl.Type.Results.List) > 1 { + signature.WriteString("(") + } + for i, result := range decl.Type.Results.List { + if i > 0 { + signature.WriteString(", ") + } + signature.WriteString(formatFieldList(result)) + } + if len(decl.Type.Results.List) > 1 { + signature.WriteString(")") + } + } + + return signature.String() +} + +func formatFieldList(field *ast.Field) string { + var builder strings.Builder + switch fieldType := field.Type.(type) { + case *ast.Ident: + builder.WriteString(fieldType.Name) + case *ast.ArrayType: + builder.WriteString("[]") + builder.WriteString(formatFieldList(&ast.Field{Type: fieldType.Elt})) + case *ast.StarExpr: + builder.WriteString("*") + builder.WriteString(formatFieldList(&ast.Field{Type: fieldType.X})) + } + return builder.String() +} + +func isPublic(name string) bool { + if len(name) == 0 { + return false + } + return unicode.IsUpper([]rune(name)[0]) +} diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h new file mode 100644 index 00000000..eb087156 --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVSANDBOXUTILS_H__ +#define __NVSANDBOXUTILS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define INPUT_LENGTH 256 +#define MAX_FILE_PATH 256 +#define MAX_NAME_LENGTH 256 + +/***************************************************************************************************/ +/** @defgroup enums Enumerations + * @{ + */ +/***************************************************************************************************/ + +/** + * Return types + */ +typedef enum +{ + NVSANDBOXUTILS_SUCCESS = 0, //!< The operation was successful + NVSANDBOXUTILS_ERROR_UNINITIALIZED = 1, //!< The library wasn't successfully initialized + NVSANDBOXUTILS_ERROR_NOT_SUPPORTED = 2, //!< The requested operation is not supported on target device + NVSANDBOXUTILS_ERROR_INVALID_ARG = 3, //!< A supplied argument is invalid + NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE = 4, //!< A supplied argument is not large enough + NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED = 5, //!< Requested library version is not supported + NVSANDBOXUTILS_ERROR_LIBRARY_LOAD = 6, //!< The library load failed + NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND = 7, //!< Called function was not found + NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND = 8, //!< Target device was not found + NVSANDBOXUTILS_ERROR_NVML_LIB_CALL = 9, //!< NVML library call failed + NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY = 10, //!< There is insufficient memory + NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND = 11, //!< A supplied file path was not found + NVSANDBOXUTILS_ERROR_UNKNOWN = 0xFFFF, //!< Unknown error occurred +} nvSandboxUtilsRet_t; + +/** + * Return if there is an error + */ +#define RETURN_ON_SANDBOX_ERROR(result) \ + if ((result) != NVSANDBOXUTILS_SUCCESS) { \ + NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \ + return result; \ + } + +/** + * Log levels + */ +typedef enum +{ + NVSANDBOXUTILS_LOG_LEVEL_FATAL = 0, //!< Log fatal errors + NVSANDBOXUTILS_LOG_LEVEL_ERROR = 1, //!< Log all errors + NVSANDBOXUTILS_LOG_LEVEL_WARN = 2, //!< Log all warnings + NVSANDBOXUTILS_LOG_LEVEL_DEBUG = 3, //!< Log all debug messages + NVSANDBOXUTILS_LOG_LEVEL_INFO = 4, //!< Log all info messages + NVSANDBOXUTILS_LOG_LEVEL_NONE = 0xFFFF, //!< Log none +} nvSandboxUtilsLogLevel_t; + +/** + * Input rootfs to help access files inside the driver container + */ +typedef enum +{ + NV_ROOTFS_DEFAULT, //!< Default no rootfs + NV_ROOTFS_PATH, //!< /run/nvidia/driver + NV_ROOTFS_PID, //!< /proc/PID/mountinfo +} nvSandboxUtilsRootfsInputType_t; + +/** + * File type + */ +typedef enum +{ + NV_DEV, //!< /dev file system + NV_PROC, //!< /proc file system + NV_SYS, //!< /sys file system +} nvSandboxUtilsFileType_t; + +/** + * File subtype + */ +typedef enum +{ + NV_DEV_NVIDIA, //!< /dev/nvidia0 + NV_DEV_DRI_CARD, //!< /dev/dri/card1 + NV_DEV_DRI_RENDERD, //!< /dev/dri/renderD128 + NV_DEV_DRI_CARD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-card + NV_DEV_DRI_RENDERD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-render + NV_DEV_NVIDIA_UVM, //!< /dev/nvidia-uvm + NV_DEV_NVIDIA_UVM_TOOLS, //!< /dev/nvidia-uvm-tools + NV_DEV_NVIDIA_MODESET, //!< /dev/nvidia-uvm-modeset + NV_DEV_NVIDIA_CTL, //!< /dev/nvidiactl + NV_DEV_GDRDRV, //!< /dev/gdrdrv + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP, //!< /dev/nvidia-caps/nvidia-cap22 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID, //!< /proc/driver/nvidia/gpus/0000:2d:00.0 + NV_PROC_DRIVER_NVIDIA_GPUS, //!< /proc/driver/nvidia/gpus (for mask out) + NV_PROC_NVIDIA_PARAMS, //!< /proc/driver/nvidia/params + NV_PROC_NVIDIA_CAPS_MIG_MINORS, //!< /proc/driver/nvidia-caps/mig-minors + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU, //!< /proc/driver/nvidia/capabilities/gpu0 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES, //!< /proc/driver/nvidia/capabilities (for mask out) + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS, //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID, //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0 + NV_SYS_MODULE_NVIDIA_DRIVER, //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out) + NV_NUM_SUBTYPE, // always at the end. +} nvSandboxUtilsFileSystemSubType_t; + +/** + * File module + */ +typedef enum +{ + NV_GPU, //!< Target device + NV_MIG, //!< Target device- MIG + NV_DRIVER_NVIDIA, //!< NVIDIA kernel driver + NV_DRIVER_NVIDIA_UVM, //!< NVIDIA kernel driver-UVM + NV_DRIVER_NVIDIA_MODESET, //!< NVIDIA kernel driver-modeset + NV_DRIVER_GDRDRV, //!< GDRDRV driver + NV_SYSTEM, //!< System module +} nvSandboxUtilsFileModule_t; + +/** + * Flag to provide additional details about the file + */ +typedef enum +{ + NV_FILE_FLAG_HINT = (1 << 0), //!< Default no hint + NV_FILE_FLAG_MASKOUT = (1 << 1), //!< For /proc/driver/nvidia/gpus + NV_FILE_FLAG_CONTENT = (1 << 2), //!< For /proc/driver/nvidia/params + //!< For SYMLINK + //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file + NV_FILE_FLAG_DEPRECTATED = (1 << 3), //!< For all the FIRMWARE GSP file + NV_FILE_FLAG_CANDIDATES = (1 << 4), //!< For libcuda.so +} nvSandboxUtilsFileFlag_t; + +/** + * Input type of the target device + */ +typedef enum +{ + NV_GPU_INPUT_GPU_UUID, //!< GPU UUID + NV_GPU_INPUT_MIG_UUID, //!< MIG UUID + NV_GPU_INPUT_PCI_ID, //!< PCIe DBDF ID + NV_GPU_INPUT_PCI_INDEX, //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF) +} nvSandboxUtilsGpuInputType_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup dataTypes Structures and Unions + * @{ + */ +/***************************************************************************************************/ + +/** + * Initalization input v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsRootfsInputType_t type; //!< One of \p nvSandboxUtilsRootfsInputType_t + char value[INPUT_LENGTH]; //!< String representation of input +} nvSandboxUtilsInitInput_v1_t; + +typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t; + +/** + * File system information + */ +typedef struct nvSandboxUtilsGpuFileInfo_v1_t +{ + struct nvSandboxUtilsGpuFileInfo_v1_t *next; //!< Pointer to the next node in the linked list + nvSandboxUtilsFileType_t fileType; //!< One of \p nvSandboxUtilsFileType_t + nvSandboxUtilsFileSystemSubType_t fileSubType; //!< One of \p nvSandboxUtilsFileSystemSubType_t + nvSandboxUtilsFileModule_t module; //!< One of \p nvSandboxUtilsFileModule_t + nvSandboxUtilsFileFlag_t flags; //!< One of \p nvSandboxUtilsFileFlag_t + char *filePath; //!< Relative file path to rootfs +}nvSandboxUtilsGpuFileInfo_v1_t; + +/** + * GPU resource request v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsGpuInputType_t inputType; //!< One of \p nvSandboxUtilsGpuInputType_t + char input[INPUT_LENGTH]; //!< String representation of input + nvSandboxUtilsGpuFileInfo_v1_t *files; //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t +} nvSandboxUtilsGpuRes_v1_t; + +typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup funcs Functions + * @{ + */ +/***************************************************************************************************/ + +/* ************************************************* + * Initialize library + * ************************************************* + */ +/** + * Prepare library resources before library API can be used. + * This initialization will not fail if one of the initialization prerequisites fails. + * @param input Reference to the called-supplied input struct that has initialization fields + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p input->value isn't a valid rootfs path + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p input->version isn't supported by the library + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND if any of the required file paths are not found during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY if there is insufficient system memory during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD on any error during loading the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input); + +/* ************************************************* + * Shutdown library + * ************************************************* + */ +/** + * Clean up library resources created by init call + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + */ +nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void); + +/* ************************************************* + * Get NVIDIA RM driver version + * ************************************************* + */ +/** + * Get NVIDIA RM driver version + * @param version Reference to caller-supplied buffer to return driver version string + * @param length The maximum allowed length of the string returned in \p version + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p version is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL on any error during driver version query from NVML + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length); + +/* ************************************************* + * Get /dev, /proc, /sys file system information + * ************************************************* + */ +/** + * Get /dev, /proc, /sys file system information + * @param request Reference to caller-supplied request struct to return the file system information + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p request->input doesn't match any device + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p request->version isn't supported by the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request); + +/* ************************************************* + * Get content of given file path + * ************************************************* + */ +/** + * Get file content of input file path + * @param filePath Reference to the file path + * @param content Reference to the caller-supplied buffer to return the file content + * @param contentSize Reference to the maximum allowed size of content. It is updated to the actual size of the content on return + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p filePath or \p content is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE if \p contentSize is too small + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND on an error while obtaining the content for the file path + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize); + +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif // __NVSANDBOXUTILS_H__ diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml new file mode 100644 index 00000000..851db743 --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml @@ -0,0 +1,66 @@ +# Copyright (c) 2024, NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +GENERATOR: + PackageName: nvsandboxutils + PackageDescription: "Package NVSANDBOXUTILS bindings" + PackageLicense: |- + Copyright (c) 2024, NVIDIA CORPORATION + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Includes: ["nvsandboxutils.h"] + FlagGroups: + - {name: "LDFLAGS", traits: ["linux"], flags: ["-Wl,--export-dynamic","-Wl,--unresolved-symbols=ignore-in-object-files"]} + - {name: "LDFLAGS", traits: ["darwin"], flags: ["-Wl,-undefined,dynamic_lookup"]} +PARSER: + SourcesPaths: ["nvsandboxutils.h"] +TRANSLATOR: + ConstRules: + defines: eval + enum: eval + PtrTips: + function: + - {target: "^nvSandboxUtils", default: "sref"} + MemTips: + - {target: "^nvSandboxUtils", default: "raw"} + Rules: + const: + - {action: accept, from: "^NVSANDBOXUTILS_"} + - {action: accept, from: "^nvSandboxUtils"} + - {action: replace, from: "^NVSANDBOXUTILS_"} + - {action: replace, from: "^nvSandboxUtils"} + - {action: accept, from: "^NV"} + - {action: accept, from: "^MAX"} + - {action: accept, from: "^INPUT"} + - {action: replace, from: "_t$"} + - {transform: export} + type: + - {action: accept, from: "^nvSandboxUtils"} + - {action: replace, from: "^nvSandboxUtils"} + - {action: replace, from: "_t$"} + - {transform: export} + function: + - {action: accept, from: "^nvSandboxUtils"} + - {transform: unexport} diff --git a/internal/nvsandboxutils/gen/update-bindings.sh b/internal/nvsandboxutils/gen/update-bindings.sh new file mode 100755 index 00000000..5ac472ea --- /dev/null +++ b/internal/nvsandboxutils/gen/update-bindings.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file allows for the nvsandboxutils bindings to be updated using the tooling +# implemented in https://github.com/NVIDIA/go-nvml. +# To run this: +# cd internal/nvsandboxutils +# ./update-bindings.sh + +set -e + +BUILDIMAGE=bindings + +docker build \ + --build-arg GOLANG_VERSION=1.22.1 \ + --build-arg C_FOR_GO_TAG=8eeee8c3b71f9c3c90c4a73db54ed08b0bba971d \ + -t ${BUILDIMAGE} \ + -f docker/Dockerfile.devel \ + https://github.com/NVIDIA/go-nvml.git + + +docker run --rm -ti \ + -e GOCACHE=/tmp/.cache/go \ + -e GOMODCACHE=/tmp/.cache/gomod \ + -v $(pwd):/nvsandboxutils \ + -w /nvsandboxutils \ + -u $(id -u):$(id -g) \ + ${BUILDIMAGE} \ + ./gen/generate-bindings.sh diff --git a/internal/nvsandboxutils/gpu-resources.go b/internal/nvsandboxutils/gpu-resources.go new file mode 100644 index 00000000..e47a9996 --- /dev/null +++ b/internal/nvsandboxutils/gpu-resources.go @@ -0,0 +1,66 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "strings" +) + +import "C" + +type GpuResource struct { + Version uint32 +} + +type GpuFileInfo struct { + Path string + Type FileType + SubType FileSystemSubType + Module FileModule + Flags FileFlag +} + +func (l *library) GetGpuResource(uuid string) ([]GpuFileInfo, Ret) { + deviceType := NV_GPU_INPUT_GPU_UUID + if strings.HasPrefix(uuid, "MIG-") { + deviceType = NV_GPU_INPUT_MIG_UUID + } + + request := GpuRes{ + Version: 1, + InputType: uint32(deviceType), + Input: convertStringToFixedArray(uuid), + } + + ret := nvSandboxUtilsGetGpuResource(&request) + if ret != SUCCESS { + return nil, ret + } + + var fileInfos []GpuFileInfo + for fileInfo := request.Files; fileInfo != nil; fileInfo = fileInfo.Next { + fi := GpuFileInfo{ + Path: C.GoString((*C.char)(fileInfo.FilePath)), + Type: FileType(fileInfo.FileType), + SubType: FileSystemSubType(fileInfo.FileSubType), + Module: FileModule(fileInfo.Module), + Flags: FileFlag(fileInfo.Flags), + } + fileInfos = append(fileInfos, fi) + } + return fileInfos, SUCCESS +} diff --git a/internal/nvsandboxutils/impl.go b/internal/nvsandboxutils/impl.go new file mode 100644 index 00000000..0f6948a2 --- /dev/null +++ b/internal/nvsandboxutils/impl.go @@ -0,0 +1,64 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import "C" + +func (l *library) Init(path string) Ret { + if err := l.load(); err != nil { + return ERROR_LIBRARY_LOAD + } + + input := InitInput{ + Version: 1, + Type: uint32(NV_ROOTFS_PATH), + Value: convertStringToFixedArray(path), + } + + return nvSandboxUtilsInit(&input) +} + +func (l *library) Shutdown() Ret { + ret := nvSandboxUtilsShutdown() + if ret != SUCCESS { + return ret + } + + err := l.close() + if err != nil { + return ERROR_UNKNOWN + } + + return ret +} + +// TODO: Is this length specified in the header file? +const VERSION_LENGTH = 100 + +func (l *library) GetDriverVersion() (string, Ret) { + Version := make([]byte, VERSION_LENGTH) + ret := nvSandboxUtilsGetDriverVersion(&Version[0], VERSION_LENGTH) + return string(Version[:clen(Version)]), ret +} + +func (l *library) GetFileContent(path string) (string, Ret) { + Content := make([]byte, MAX_FILE_PATH) + FilePath := []byte(path + string(byte(0))) + Size := uint32(MAX_FILE_PATH) + ret := nvSandboxUtilsGetFileContent(&FilePath[0], &Content[0], &Size) + return string(Content[:clen(Content)]), ret +} diff --git a/internal/nvsandboxutils/lib.go b/internal/nvsandboxutils/lib.go new file mode 100644 index 00000000..3a85ef89 --- /dev/null +++ b/internal/nvsandboxutils/lib.go @@ -0,0 +1,156 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "fmt" + "sync" + + "github.com/NVIDIA/go-nvml/pkg/dl" +) + +const ( + defaultNvSandboxUtilsLibraryName = "libnvidia-sandboxutils.so.1" + defaultNvSandboxUtilsLibraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL +) + +var errLibraryNotLoaded = errors.New("library not loaded") +var errLibraryAlreadyLoaded = errors.New("library already loaded") + +// dynamicLibrary is an interface for abstacting the underlying library. +// This also allows for mocking and testing. + +//go:generate moq -rm -stub -out dynamicLibrary_mock.go . dynamicLibrary +type dynamicLibrary interface { + Lookup(string) error + Open() error + Close() error +} + +// library represents an nvsandboxutils library. +// This includes a reference to the underlying DynamicLibrary +type library struct { + sync.Mutex + path string + refcount refcount + dl dynamicLibrary +} + +// libnvsandboxutils is a global instance of the nvsandboxutils library. +var libnvsandboxutils = newLibrary() + +func New(opts ...LibraryOption) Interface { + return newLibrary(opts...) +} + +func newLibrary(opts ...LibraryOption) *library { + l := &library{} + l.init(opts...) + return l +} + +func (l *library) init(opts ...LibraryOption) { + o := libraryOptions{} + for _, opt := range opts { + opt(&o) + } + + if o.path == "" { + o.path = defaultNvSandboxUtilsLibraryName + } + if o.flags == 0 { + o.flags = defaultNvSandboxUtilsLibraryLoadFlags + } + + l.path = o.path + l.dl = dl.New(o.path, o.flags) +} + +// LookupSymbol checks whether the specified library symbol exists in the library. +// Note that this requires that the library be loaded. +func (l *library) LookupSymbol(name string) error { + if l == nil || l.refcount == 0 { + return fmt.Errorf("error looking up %s: %w", name, errLibraryNotLoaded) + } + return l.dl.Lookup(name) +} + +// load initializes the library and updates the versioned symbols. +// Multiple calls to an already loaded library will return without error. +func (l *library) load() (rerr error) { + l.Lock() + defer l.Unlock() + + defer func() { l.refcount.IncOnNoError(rerr) }() + if l.refcount > 0 { + return nil + } + + if err := l.dl.Open(); err != nil { + return fmt.Errorf("error opening %s: %w", l.path, err) + } + + // Update the errorStringFunc to point to nvsandboxutils.ErrorString + errorStringFunc = nvsanboxutilsErrorString + + // Update all versioned symbols + l.updateVersionedSymbols() + + return nil +} + +// close the underlying library and ensure that the global pointer to the +// library is set to nil to ensure that subsequent calls to open will reinitialize it. +// Multiple calls to an already closed nvsandboxutils library will return without error. +func (l *library) close() (rerr error) { + l.Lock() + defer l.Unlock() + + defer func() { l.refcount.DecOnNoError(rerr) }() + if l.refcount != 1 { + return nil + } + + if err := l.dl.Close(); err != nil { + return fmt.Errorf("error closing %s: %w", l.path, err) + } + + // Update the errorStringFunc to point to defaultErrorStringFunc + errorStringFunc = defaultErrorStringFunc + + return nil +} + +// Default all versioned APIs to v1 (to infer the types) +var ( +// Insert default versions for APIs here. +// Example: +// nvsandboxUtilsFunction = nvsandboxUtilsFunction_v1 +) + +// updateVersionedSymbols checks for versioned symbols in the loaded dynamic library. +// If newer versioned symbols exist, these replace the default `v1` symbols initialized above. +// When new versioned symbols are added, these would have to be initialized above and have +// corresponding checks and subsequent assignments added below. +func (l *library) updateVersionedSymbols() { + // Example: + // err := l.dl.Lookup("nvsandboxUtilsFunction_v2") + // if err == nil { + // nvsandboxUtilsFunction = nvsandboxUtilsFunction_v2 + // } +} diff --git a/internal/nvsandboxutils/lib_test.go b/internal/nvsandboxutils/lib_test.go new file mode 100644 index 00000000..b87c7242 --- /dev/null +++ b/internal/nvsandboxutils/lib_test.go @@ -0,0 +1,245 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestLibrary(dl dynamicLibrary) *library { + return &library{dl: dl} +} + +func TestLookupFromDefault(t *testing.T) { + errClose := errors.New("close error") + errOpen := errors.New("open error") + errLookup := errors.New("lookup error") + + testCases := []struct { + description string + dl dynamicLibrary + skipLoadLibrary bool + expectedLoadError error + expectedLookupErrror error + expectedCloseError error + }{ + { + description: "library not loaded yields error", + dl: &dynamicLibraryMock{}, + skipLoadLibrary: true, + expectedLookupErrror: errLibraryNotLoaded, + }, + { + description: "open error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errOpen + }, + }, + + expectedLoadError: errOpen, + expectedLookupErrror: errLibraryNotLoaded, + }, + { + description: "lookup error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return fmt.Errorf("%w: %s", errLookup, s) + }, + CloseFunc: func() error { + return nil + }, + }, + + expectedLookupErrror: errLookup, + }, + { + description: "lookup succeeds", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + }, + { + description: "lookup succeeds", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + }, + { + description: "close error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return errClose + }, + }, + expectedCloseError: errClose, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l := newTestLibrary(tc.dl) + if !tc.skipLoadLibrary { + require.ErrorIs(t, l.load(), tc.expectedLoadError) + } + require.ErrorIs(t, l.LookupSymbol("symbol"), tc.expectedLookupErrror) + require.ErrorIs(t, l.close(), tc.expectedCloseError) + if tc.expectedCloseError == nil { + require.Equal(t, 0, int(l.refcount)) + } else { + require.Equal(t, 1, int(l.refcount)) + } + }) + } +} + +func TestLoadAndCloseNesting(t *testing.T) { + dl := &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return nil + }, + } + + l := newTestLibrary(dl) + + // When calling close before opening the library nothing happens. + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 0, len(dl.calls.Close)) + + // When calling load twice, the library was only opened once + require.Equal(t, 0, len(dl.calls.Open)) + require.Nil(t, l.load()) + require.Equal(t, 1, len(dl.calls.Open)) + require.Nil(t, l.load()) + require.Equal(t, 1, len(dl.calls.Open)) + + // Only after calling close twice, was the library closed + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 1, len(dl.calls.Close)) + + // Calling close again doesn't attempt to close the library again + require.Nil(t, l.close()) + require.Equal(t, 1, len(dl.calls.Close)) +} + +func TestLoadAndCloseWithErrors(t *testing.T) { + testCases := []struct { + description string + dl dynamicLibrary + expectedLoadRefcount refcount + expectedCloseRefcount refcount + }{ + { + description: "regular flow", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + expectedLoadRefcount: 1, + expectedCloseRefcount: 0, + }, + { + description: "open error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errors.New("") + }, + CloseFunc: func() error { + return nil + }, + }, + expectedLoadRefcount: 0, + expectedCloseRefcount: 0, + }, + { + description: "close error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return errors.New("") + }, + }, + expectedLoadRefcount: 1, + expectedCloseRefcount: 1, + }, + { + description: "open and close error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errors.New("") + }, + CloseFunc: func() error { + return errors.New("") + }, + }, + expectedLoadRefcount: 0, + expectedCloseRefcount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l := newTestLibrary(tc.dl) + _ = l.load() + require.Equal(t, tc.expectedLoadRefcount, l.refcount) + _ = l.close() + require.Equal(t, tc.expectedCloseRefcount, l.refcount) + }) + } +} diff --git a/internal/nvsandboxutils/mock/interface.go b/internal/nvsandboxutils/mock/interface.go new file mode 100644 index 00000000..d7b27ebf --- /dev/null +++ b/internal/nvsandboxutils/mock/interface.go @@ -0,0 +1,325 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" + "sync" +) + +// Ensure, that Interface does implement nvsandboxutils.Interface. +// If this is not the case, regenerate this file with moq. +var _ nvsandboxutils.Interface = &Interface{} + +// Interface is a mock implementation of nvsandboxutils.Interface. +// +// func TestSomethingThatUsesInterface(t *testing.T) { +// +// // make and configure a mocked nvsandboxutils.Interface +// mockedInterface := &Interface{ +// ErrorStringFunc: func(ret nvsandboxutils.Ret) string { +// panic("mock out the ErrorString method") +// }, +// GetDriverVersionFunc: func() (string, nvsandboxutils.Ret) { +// panic("mock out the GetDriverVersion method") +// }, +// GetFileContentFunc: func(s string) (string, nvsandboxutils.Ret) { +// panic("mock out the GetFileContent method") +// }, +// GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { +// panic("mock out the GetGpuResource method") +// }, +// InitFunc: func(s string) nvsandboxutils.Ret { +// panic("mock out the Init method") +// }, +// LookupSymbolFunc: func(s string) error { +// panic("mock out the LookupSymbol method") +// }, +// ShutdownFunc: func() nvsandboxutils.Ret { +// panic("mock out the Shutdown method") +// }, +// } +// +// // use mockedInterface in code that requires nvsandboxutils.Interface +// // and then make assertions. +// +// } +type Interface struct { + // ErrorStringFunc mocks the ErrorString method. + ErrorStringFunc func(ret nvsandboxutils.Ret) string + + // GetDriverVersionFunc mocks the GetDriverVersion method. + GetDriverVersionFunc func() (string, nvsandboxutils.Ret) + + // GetFileContentFunc mocks the GetFileContent method. + GetFileContentFunc func(s string) (string, nvsandboxutils.Ret) + + // GetGpuResourceFunc mocks the GetGpuResource method. + GetGpuResourceFunc func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) + + // InitFunc mocks the Init method. + InitFunc func(s string) nvsandboxutils.Ret + + // LookupSymbolFunc mocks the LookupSymbol method. + LookupSymbolFunc func(s string) error + + // ShutdownFunc mocks the Shutdown method. + ShutdownFunc func() nvsandboxutils.Ret + + // calls tracks calls to the methods. + calls struct { + // ErrorString holds details about calls to the ErrorString method. + ErrorString []struct { + // Ret is the ret argument value. + Ret nvsandboxutils.Ret + } + // GetDriverVersion holds details about calls to the GetDriverVersion method. + GetDriverVersion []struct { + } + // GetFileContent holds details about calls to the GetFileContent method. + GetFileContent []struct { + // S is the s argument value. + S string + } + // GetGpuResource holds details about calls to the GetGpuResource method. + GetGpuResource []struct { + // S is the s argument value. + S string + } + // Init holds details about calls to the Init method. + Init []struct { + // S is the s argument value. + S string + } + // LookupSymbol holds details about calls to the LookupSymbol method. + LookupSymbol []struct { + // S is the s argument value. + S string + } + // Shutdown holds details about calls to the Shutdown method. + Shutdown []struct { + } + } + lockErrorString sync.RWMutex + lockGetDriverVersion sync.RWMutex + lockGetFileContent sync.RWMutex + lockGetGpuResource sync.RWMutex + lockInit sync.RWMutex + lockLookupSymbol sync.RWMutex + lockShutdown sync.RWMutex +} + +// ErrorString calls ErrorStringFunc. +func (mock *Interface) ErrorString(ret nvsandboxutils.Ret) string { + if mock.ErrorStringFunc == nil { + panic("Interface.ErrorStringFunc: method is nil but Interface.ErrorString was just called") + } + callInfo := struct { + Ret nvsandboxutils.Ret + }{ + Ret: ret, + } + mock.lockErrorString.Lock() + mock.calls.ErrorString = append(mock.calls.ErrorString, callInfo) + mock.lockErrorString.Unlock() + return mock.ErrorStringFunc(ret) +} + +// ErrorStringCalls gets all the calls that were made to ErrorString. +// Check the length with: +// +// len(mockedInterface.ErrorStringCalls()) +func (mock *Interface) ErrorStringCalls() []struct { + Ret nvsandboxutils.Ret +} { + var calls []struct { + Ret nvsandboxutils.Ret + } + mock.lockErrorString.RLock() + calls = mock.calls.ErrorString + mock.lockErrorString.RUnlock() + return calls +} + +// GetDriverVersion calls GetDriverVersionFunc. +func (mock *Interface) GetDriverVersion() (string, nvsandboxutils.Ret) { + if mock.GetDriverVersionFunc == nil { + panic("Interface.GetDriverVersionFunc: method is nil but Interface.GetDriverVersion was just called") + } + callInfo := struct { + }{} + mock.lockGetDriverVersion.Lock() + mock.calls.GetDriverVersion = append(mock.calls.GetDriverVersion, callInfo) + mock.lockGetDriverVersion.Unlock() + return mock.GetDriverVersionFunc() +} + +// GetDriverVersionCalls gets all the calls that were made to GetDriverVersion. +// Check the length with: +// +// len(mockedInterface.GetDriverVersionCalls()) +func (mock *Interface) GetDriverVersionCalls() []struct { +} { + var calls []struct { + } + mock.lockGetDriverVersion.RLock() + calls = mock.calls.GetDriverVersion + mock.lockGetDriverVersion.RUnlock() + return calls +} + +// GetFileContent calls GetFileContentFunc. +func (mock *Interface) GetFileContent(s string) (string, nvsandboxutils.Ret) { + if mock.GetFileContentFunc == nil { + panic("Interface.GetFileContentFunc: method is nil but Interface.GetFileContent was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockGetFileContent.Lock() + mock.calls.GetFileContent = append(mock.calls.GetFileContent, callInfo) + mock.lockGetFileContent.Unlock() + return mock.GetFileContentFunc(s) +} + +// GetFileContentCalls gets all the calls that were made to GetFileContent. +// Check the length with: +// +// len(mockedInterface.GetFileContentCalls()) +func (mock *Interface) GetFileContentCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockGetFileContent.RLock() + calls = mock.calls.GetFileContent + mock.lockGetFileContent.RUnlock() + return calls +} + +// GetGpuResource calls GetGpuResourceFunc. +func (mock *Interface) GetGpuResource(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + if mock.GetGpuResourceFunc == nil { + panic("Interface.GetGpuResourceFunc: method is nil but Interface.GetGpuResource was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockGetGpuResource.Lock() + mock.calls.GetGpuResource = append(mock.calls.GetGpuResource, callInfo) + mock.lockGetGpuResource.Unlock() + return mock.GetGpuResourceFunc(s) +} + +// GetGpuResourceCalls gets all the calls that were made to GetGpuResource. +// Check the length with: +// +// len(mockedInterface.GetGpuResourceCalls()) +func (mock *Interface) GetGpuResourceCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockGetGpuResource.RLock() + calls = mock.calls.GetGpuResource + mock.lockGetGpuResource.RUnlock() + return calls +} + +// Init calls InitFunc. +func (mock *Interface) Init(s string) nvsandboxutils.Ret { + if mock.InitFunc == nil { + panic("Interface.InitFunc: method is nil but Interface.Init was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockInit.Lock() + mock.calls.Init = append(mock.calls.Init, callInfo) + mock.lockInit.Unlock() + return mock.InitFunc(s) +} + +// InitCalls gets all the calls that were made to Init. +// Check the length with: +// +// len(mockedInterface.InitCalls()) +func (mock *Interface) InitCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockInit.RLock() + calls = mock.calls.Init + mock.lockInit.RUnlock() + return calls +} + +// LookupSymbol calls LookupSymbolFunc. +func (mock *Interface) LookupSymbol(s string) error { + if mock.LookupSymbolFunc == nil { + panic("Interface.LookupSymbolFunc: method is nil but Interface.LookupSymbol was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockLookupSymbol.Lock() + mock.calls.LookupSymbol = append(mock.calls.LookupSymbol, callInfo) + mock.lockLookupSymbol.Unlock() + return mock.LookupSymbolFunc(s) +} + +// LookupSymbolCalls gets all the calls that were made to LookupSymbol. +// Check the length with: +// +// len(mockedInterface.LookupSymbolCalls()) +func (mock *Interface) LookupSymbolCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockLookupSymbol.RLock() + calls = mock.calls.LookupSymbol + mock.lockLookupSymbol.RUnlock() + return calls +} + +// Shutdown calls ShutdownFunc. +func (mock *Interface) Shutdown() nvsandboxutils.Ret { + if mock.ShutdownFunc == nil { + panic("Interface.ShutdownFunc: method is nil but Interface.Shutdown was just called") + } + callInfo := struct { + }{} + mock.lockShutdown.Lock() + mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo) + mock.lockShutdown.Unlock() + return mock.ShutdownFunc() +} + +// ShutdownCalls gets all the calls that were made to Shutdown. +// Check the length with: +// +// len(mockedInterface.ShutdownCalls()) +func (mock *Interface) ShutdownCalls() []struct { +} { + var calls []struct { + } + mock.lockShutdown.RLock() + calls = mock.calls.Shutdown + mock.lockShutdown.RUnlock() + return calls +} diff --git a/internal/nvsandboxutils/nvsandboxutils.go b/internal/nvsandboxutils/nvsandboxutils.go new file mode 100644 index 00000000..29544bc9 --- /dev/null +++ b/internal/nvsandboxutils/nvsandboxutils.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package nvsandboxutils + +/* +#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup +#include "nvsandboxutils.h" +#include +#include "cgo_helpers.h" +*/ +import "C" +import "unsafe" + +// nvSandboxUtilsInit function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsInit(Input *InitInput) Ret { + cInput, _ := (*C.nvSandboxUtilsInitInput_t)(unsafe.Pointer(Input)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsInit(cInput) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsShutdown function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsShutdown() Ret { + __ret := C.nvSandboxUtilsShutdown() + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetDriverVersion function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetDriverVersion(Version *byte, Length uint32) Ret { + cVersion, _ := (*C.char)(unsafe.Pointer(Version)), cgoAllocsUnknown + cLength, _ := (C.uint)(Length), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetDriverVersion(cVersion, cLength) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetGpuResource function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetGpuResource(Request *GpuRes) Ret { + cRequest, _ := (*C.nvSandboxUtilsGpuRes_t)(unsafe.Pointer(Request)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetGpuResource(cRequest) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetFileContent function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetFileContent(FilePath *byte, Content *byte, ContentSize *uint32) Ret { + cFilePath, _ := (*C.char)(unsafe.Pointer(FilePath)), cgoAllocsUnknown + cContent, _ := (*C.char)(unsafe.Pointer(Content)), cgoAllocsUnknown + cContentSize, _ := (*C.uint)(unsafe.Pointer(ContentSize)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetFileContent(cFilePath, cContent, cContentSize) + __v := (Ret)(__ret) + return __v +} diff --git a/internal/nvsandboxutils/nvsandboxutils.h b/internal/nvsandboxutils/nvsandboxutils.h new file mode 100644 index 00000000..3c66e159 --- /dev/null +++ b/internal/nvsandboxutils/nvsandboxutils.h @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVSANDBOXUTILS_H__ +#define __NVSANDBOXUTILS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define INPUT_LENGTH 256 +#define MAX_FILE_PATH 256 +#define MAX_NAME_LENGTH 256 + +/***************************************************************************************************/ +/** @defgroup enums Enumerations + * @{ + */ +/***************************************************************************************************/ + +/** + * Return types + */ +typedef enum +{ + NVSANDBOXUTILS_SUCCESS = 0, //!< The operation was successful + NVSANDBOXUTILS_ERROR_UNINITIALIZED = 1, //!< The library wasn't successfully initialized + NVSANDBOXUTILS_ERROR_NOT_SUPPORTED = 2, //!< The requested operation is not supported on target device + NVSANDBOXUTILS_ERROR_INVALID_ARG = 3, //!< A supplied argument is invalid + NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE = 4, //!< A supplied argument is not large enough + NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED = 5, //!< Requested library version is not supported + NVSANDBOXUTILS_ERROR_LIBRARY_LOAD = 6, //!< The library load failed + NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND = 7, //!< Called function was not found + NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND = 8, //!< Target device was not found + NVSANDBOXUTILS_ERROR_NVML_LIB_CALL = 9, //!< NVML library call failed + NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY = 10, //!< There is insufficient memory + NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND = 11, //!< A supplied file path was not found + NVSANDBOXUTILS_ERROR_UNKNOWN = 0xFFFF, //!< Unknown error occurred +} nvSandboxUtilsRet_t; + +/** + * Return if there is an error + */ +#define RETURN_ON_SANDBOX_ERROR(result) \ + if ((result) != NVSANDBOXUTILS_SUCCESS) { \ + NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \ + return result; \ + } + +/** + * Log levels + */ +typedef enum +{ + NVSANDBOXUTILS_LOG_LEVEL_FATAL = 0, //!< Log fatal errors + NVSANDBOXUTILS_LOG_LEVEL_ERROR = 1, //!< Log all errors + NVSANDBOXUTILS_LOG_LEVEL_WARN = 2, //!< Log all warnings + NVSANDBOXUTILS_LOG_LEVEL_DEBUG = 3, //!< Log all debug messages + NVSANDBOXUTILS_LOG_LEVEL_INFO = 4, //!< Log all info messages + NVSANDBOXUTILS_LOG_LEVEL_NONE = 0xFFFF, //!< Log none +} nvSandboxUtilsLogLevel_t; + +/** + * Input rootfs to help access files inside the driver container + */ +typedef enum +{ + NV_ROOTFS_DEFAULT, //!< Default no rootfs + NV_ROOTFS_PATH, //!< /run/nvidia/driver + NV_ROOTFS_PID, //!< /proc/PID/mountinfo +} nvSandboxUtilsRootfsInputType_t; + +/** + * File type + */ +typedef enum +{ + NV_DEV, //!< /dev file system + NV_PROC, //!< /proc file system + NV_SYS, //!< /sys file system +} nvSandboxUtilsFileType_t; + +/** + * File subtype + */ +typedef enum +{ + NV_DEV_NVIDIA, //!< /dev/nvidia0 + NV_DEV_DRI_CARD, //!< /dev/dri/card1 + NV_DEV_DRI_RENDERD, //!< /dev/dri/renderD128 + NV_DEV_DRI_CARD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-card + NV_DEV_DRI_RENDERD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-render + NV_DEV_NVIDIA_UVM, //!< /dev/nvidia-uvm + NV_DEV_NVIDIA_UVM_TOOLS, //!< /dev/nvidia-uvm-tools + NV_DEV_NVIDIA_MODESET, //!< /dev/nvidia-uvm-modeset + NV_DEV_NVIDIA_CTL, //!< /dev/nvidiactl + NV_DEV_GDRDRV, //!< /dev/gdrdrv + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP, //!< /dev/nvidia-caps/nvidia-cap22 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID, //!< /proc/driver/nvidia/gpus/0000:2d:00.0 + NV_PROC_DRIVER_NVIDIA_GPUS, //!< /proc/driver/nvidia/gpus (for mask out) + NV_PROC_NVIDIA_PARAMS, //!< /proc/driver/nvidia/params + NV_PROC_NVIDIA_CAPS_MIG_MINORS, //!< /proc/driver/nvidia-caps/mig-minors + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU, //!< /proc/driver/nvidia/capabilities/gpu0 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES, //!< /proc/driver/nvidia/capabilities (for mask out) + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS, //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID, //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0 + NV_SYS_MODULE_NVIDIA_DRIVER, //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out) + NV_NUM_SUBTYPE, // always at the end. +} nvSandboxUtilsFileSystemSubType_t; + +/** + * File module + */ +typedef enum +{ + NV_GPU, //!< Target device + NV_MIG, //!< Target device- MIG + NV_DRIVER_NVIDIA, //!< NVIDIA kernel driver + NV_DRIVER_NVIDIA_UVM, //!< NVIDIA kernel driver-UVM + NV_DRIVER_NVIDIA_MODESET, //!< NVIDIA kernel driver-modeset + NV_DRIVER_GDRDRV, //!< GDRDRV driver + NV_SYSTEM, //!< System module +} nvSandboxUtilsFileModule_t; + +/** + * Flag to provide additional details about the file + */ +typedef enum +{ + NV_FILE_FLAG_HINT = (1 << 0), //!< Default no hint + NV_FILE_FLAG_MASKOUT = (1 << 1), //!< For /proc/driver/nvidia/gpus + NV_FILE_FLAG_CONTENT = (1 << 2), //!< For /proc/driver/nvidia/params + //!< For SYMLINK + //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file + NV_FILE_FLAG_DEPRECTATED = (1 << 3), //!< For all the FIRMWARE GSP file + NV_FILE_FLAG_CANDIDATES = (1 << 4), //!< For libcuda.so +} nvSandboxUtilsFileFlag_t; + +/** + * Input type of the target device + */ +typedef enum +{ + NV_GPU_INPUT_GPU_UUID, //!< GPU UUID + NV_GPU_INPUT_MIG_UUID, //!< MIG UUID + NV_GPU_INPUT_PCI_ID, //!< PCIe DBDF ID + NV_GPU_INPUT_PCI_INDEX, //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF) +} nvSandboxUtilsGpuInputType_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup dataTypes Structures and Unions + * @{ + */ +/***************************************************************************************************/ + +/** + * Initalization input v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsRootfsInputType_t type; //!< One of \p nvSandboxUtilsRootfsInputType_t + char value[INPUT_LENGTH]; //!< String representation of input +} nvSandboxUtilsInitInput_v1_t; + +typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t; + +/** + * File system information + */ +typedef struct nvSandboxUtilsGpuFileInfo_v1_t +{ + struct nvSandboxUtilsGpuFileInfo_v1_t *next; //!< Pointer to the next node in the linked list + nvSandboxUtilsFileType_t fileType; //!< One of \p nvSandboxUtilsFileType_t + nvSandboxUtilsFileSystemSubType_t fileSubType; //!< One of \p nvSandboxUtilsFileSystemSubType_t + nvSandboxUtilsFileModule_t module; //!< One of \p nvSandboxUtilsFileModule_t + nvSandboxUtilsFileFlag_t flags; //!< One of \p nvSandboxUtilsFileFlag_t + char *filePath; //!< Relative file path to rootfs +}nvSandboxUtilsGpuFileInfo_v1_t; + +/** + * GPU resource request v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsGpuInputType_t inputType; //!< One of \p nvSandboxUtilsGpuInputType_t + char input[INPUT_LENGTH]; //!< String representation of input + nvSandboxUtilsGpuFileInfo_v1_t *files; //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t +} nvSandboxUtilsGpuRes_v1_t; + +typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup funcs Functions + * @{ + */ +/***************************************************************************************************/ + +/* ************************************************* + * Initialize library + * ************************************************* + */ +/** + * Prepare library resources before library API can be used. + * This initialization will not fail if one of the initialization prerequisites fails. + * @param input Reference to the called-supplied input struct that has initialization fields + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p input->value isn't a valid rootfs path + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p input->version isn't supported by the library + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND if any of the required file paths are not found during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY if there is insufficient system memory during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD on any error during loading the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input); + +/* ************************************************* + * Shutdown library + * ************************************************* + */ +/** + * Clean up library resources created by init call + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + */ +nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void); + +/* ************************************************* + * Get NVIDIA RM driver version + * ************************************************* + */ +/** + * Get NVIDIA RM driver version + * @param version Reference to caller-supplied buffer to return driver version string + * @param length The maximum allowed length of the string returned in \p version + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p version is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL on any error during driver version query from NVML + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length); + +/* ************************************************* + * Get /dev, /proc, /sys file system information + * ************************************************* + */ +/** + * Get /dev, /proc, /sys file system information + * @param request Reference to caller-supplied request struct to return the file system information + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p request->input doesn't match any device + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p request->version isn't supported by the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request); + +/* ************************************************* + * Get content of given file path + * ************************************************* + */ +/** + * Get file content of input file path + * @param filePath Reference to the file path + * @param content Reference to the caller-supplied buffer to return the file content + * @param contentSize Reference to the maximum allowed size of content. It is updated to the actual size of the content on return + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p filePath or \p content is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE if \p contentSize is too small + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND on an error while obtaining the content for the file path + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize); + +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif // __NVSANDBOXUTILS_H__ diff --git a/internal/nvsandboxutils/refcount.go b/internal/nvsandboxutils/refcount.go new file mode 100644 index 00000000..f93107b0 --- /dev/null +++ b/internal/nvsandboxutils/refcount.go @@ -0,0 +1,31 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +type refcount int + +func (r *refcount) IncOnNoError(err error) { + if err == nil { + (*r)++ + } +} + +func (r *refcount) DecOnNoError(err error) { + if err == nil && (*r) > 0 { + (*r)-- + } +} diff --git a/internal/nvsandboxutils/refcount_test.go b/internal/nvsandboxutils/refcount_test.go new file mode 100644 index 00000000..bd311e10 --- /dev/null +++ b/internal/nvsandboxutils/refcount_test.go @@ -0,0 +1,139 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefcount(t *testing.T) { + testCases := []struct { + description string + workload func(r *refcount) + expectedRefcount refcount + }{ + { + description: "No inc or dec", + workload: func(r *refcount) {}, + expectedRefcount: refcount(0), + }, + { + description: "Single inc, no error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + }, + expectedRefcount: refcount(1), + }, + { + description: "Single inc, with error", + workload: func(r *refcount) { + r.IncOnNoError(errors.New("")) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, no error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + }, + expectedRefcount: refcount(2), + }, + { + description: "Double inc, one with error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(errors.New("")) + }, + expectedRefcount: refcount(1), + }, + { + description: "Single dec, no error", + workload: func(r *refcount) { + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Single dec, with error", + workload: func(r *refcount) { + r.DecOnNoError(errors.New("")) + }, + expectedRefcount: refcount(0), + }, + { + description: "Single inc, single dec, no errors", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, no errors", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, one inc error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(errors.New("")) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, one dec error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(nil) + r.DecOnNoError(errors.New("")) + }, + expectedRefcount: refcount(1), + }, + { + description: "Double inc, Tripple dec, one dec error early on", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(errors.New("")) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + var r refcount + tc.workload(&r) + require.Equal(t, tc.expectedRefcount, r) + }) + } +} diff --git a/internal/nvsandboxutils/return.go b/internal/nvsandboxutils/return.go new file mode 100644 index 00000000..90d4ed84 --- /dev/null +++ b/internal/nvsandboxutils/return.go @@ -0,0 +1,74 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "fmt" +) + +// nvsandboxutils.ErrorString() +func (l *library) ErrorString(r Ret) string { + return r.Error() +} + +// String returns the string representation of a Ret. +func (r Ret) String() string { + return r.Error() +} + +// Error returns the string representation of a Ret. +func (r Ret) Error() string { + return errorStringFunc(r) +} + +// Assigned to nvsandboxutils.ErrorString if the system nvsandboxutils library is in use. +var errorStringFunc = defaultErrorStringFunc + +// nvsanboxutilsErrorString is an alias for the default error string function. +var nvsanboxutilsErrorString = defaultErrorStringFunc + +// defaultErrorStringFunc provides a basic nvsandboxutils.ErrorString implementation. +// This allows the nvsandboxutils.ErrorString function to be used even if the nvsandboxutils library +// is not loaded. +var defaultErrorStringFunc = func(r Ret) string { + switch r { + case SUCCESS: + return "SUCCESS" + case ERROR_UNINITIALIZED: + return "ERROR_UNINITIALIZED" + case ERROR_NOT_SUPPORTED: + return "ERROR_NOT_SUPPORTED" + case ERROR_INVALID_ARG: + return "ERROR_INVALID_ARG" + case ERROR_INSUFFICIENT_SIZE: + return "ERROR_INSUFFICIENT_SIZE" + case ERROR_VERSION_NOT_SUPPORTED: + return "ERROR_VERSION_NOT_SUPPORTED" + case ERROR_LIBRARY_LOAD: + return "ERROR_LIBRARY_LOAD" + case ERROR_FUNCTION_NOT_FOUND: + return "ERROR_FUNCTION_NOT_FOUND" + case ERROR_DEVICE_NOT_FOUND: + return "ERROR_DEVICE_NOT_FOUND" + case ERROR_NVML_LIB_CALL: + return "ERROR_NVML_LIB_CALL" + case ERROR_UNKNOWN: + return "ERROR_UNKNOWN" + default: + return fmt.Sprintf("unknown return value: %d", r) + } +} diff --git a/internal/nvsandboxutils/types_gen.go b/internal/nvsandboxutils/types_gen.go new file mode 100644 index 00000000..90a00ed6 --- /dev/null +++ b/internal/nvsandboxutils/types_gen.go @@ -0,0 +1,39 @@ +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types.go + +package nvsandboxutils + +type InitInput_v1 struct { + Version uint32 + Type uint32 + Value [256]int8 +} + +type InitInput struct { + Version uint32 + Type uint32 + Value [256]int8 +} + +type GpuFileInfo_v1 struct { + Next *GpuFileInfo_v1 + FileType uint32 + FileSubType uint32 + Module uint32 + Flags uint32 + FilePath *int8 +} + +type GpuRes_v1 struct { + Version uint32 + InputType uint32 + Input [256]int8 + Files *GpuFileInfo_v1 +} + +type GpuRes struct { + Version uint32 + InputType uint32 + Input [256]int8 + Files *GpuFileInfo_v1 +} diff --git a/internal/nvsandboxutils/zz_generated.api.go b/internal/nvsandboxutils/zz_generated.api.go new file mode 100644 index 00000000..1cc2b01a --- /dev/null +++ b/internal/nvsandboxutils/zz_generated.api.go @@ -0,0 +1,43 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// Generated Code; DO NOT EDIT. + +package nvsandboxutils + +// The variables below represent package level methods from the library type. +var ( + ErrorString = libnvsandboxutils.ErrorString + GetDriverVersion = libnvsandboxutils.GetDriverVersion + GetFileContent = libnvsandboxutils.GetFileContent + GetGpuResource = libnvsandboxutils.GetGpuResource + Init = libnvsandboxutils.Init + LookupSymbol = libnvsandboxutils.LookupSymbol + Shutdown = libnvsandboxutils.Shutdown +) + +// Interface represents the interface for the library type. +// +//go:generate moq -out mock/interface.go -pkg mock . Interface:Interface +type Interface interface { + ErrorString(Ret) string + GetDriverVersion() (string, Ret) + GetFileContent(string) (string, Ret) + GetGpuResource(string) ([]GpuFileInfo, Ret) + Init(string) Ret + LookupSymbol(string) error + Shutdown() Ret +} diff --git a/internal/platform-support/dgpu/dgpu.go b/internal/platform-support/dgpu/dgpu.go index b79f6bd4..de411bd6 100644 --- a/internal/platform-support/dgpu/dgpu.go +++ b/internal/platform-support/dgpu/dgpu.go @@ -17,6 +17,8 @@ package dgpu import ( + "errors" + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" @@ -25,22 +27,78 @@ import ( ) // NewForDevice creates a discoverer for the specified Device. +// nvsandboxutils is used for discovery if specified, otherwise NVML is used. func NewForDevice(d device.Device, opts ...Option) (discover.Discover, error) { o := new(opts...) - return o.newNvmlDGPUDiscoverer(&toRequiredInfo{d}) + var discoverers []discover.Discover + var errs error + nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(d) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvsandboxutilsDiscoverer != nil { + discoverers = append(discoverers, nvsandboxutilsDiscoverer) + } + + nvmlDiscoverer, err := o.newNvmlDGPUDiscoverer(&toRequiredInfo{d}) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvmlDiscoverer != nil { + discoverers = append(discoverers, nvmlDiscoverer) + } + + if len(discoverers) == 0 { + return nil, errs + } + + return discover.WithCache( + discover.FirstValid( + discoverers..., + ), + ), nil } -// NewForDevice creates a discoverer for the specified device and its associated MIG device. +// NewForMigDevice creates a discoverer for the specified device and its associated MIG device. +// nvsandboxutils is used for discovery if specified, otherwise NVML is used. func NewForMigDevice(d device.Device, mig device.MigDevice, opts ...Option) (discover.Discover, error) { o := new(opts...) + o.isMigDevice = true - return o.newNvmlMigDiscoverer( + var discoverers []discover.Discover + var errs error + nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(mig) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvsandboxutilsDiscoverer != nil { + discoverers = append(discoverers, nvsandboxutilsDiscoverer) + } + + nvmlDiscoverer, err := o.newNvmlMigDiscoverer( &toRequiredMigInfo{ MigDevice: mig, parent: &toRequiredInfo{d}, }, ) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvmlDiscoverer != nil { + discoverers = append(discoverers, nvmlDiscoverer) + } + + if len(discoverers) == 0 { + return nil, errs + } + + return discover.WithCache( + discover.FirstValid( + discoverers..., + ), + ), nil + } func new(opts ...Option) *options { diff --git a/internal/platform-support/dgpu/nvsandboxutils.go b/internal/platform-support/dgpu/nvsandboxutils.go new file mode 100644 index 00000000..7022deab --- /dev/null +++ b/internal/platform-support/dgpu/nvsandboxutils.go @@ -0,0 +1,131 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package dgpu + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" +) + +type nvsandboxutilsDGPU struct { + lib nvsandboxutils.Interface + uuid string + devRoot string + isMig bool + nvidiaCDIHookPath string + deviceLinks []string +} + +var _ discover.Discover = (*nvsandboxutilsDGPU)(nil) + +type UUIDer interface { + GetUUID() (string, nvml.Return) +} + +func (o *options) newNvsandboxutilsDGPUDiscoverer(d UUIDer) (discover.Discover, error) { + if o.nvsandboxutilslib == nil { + return nil, nil + } + + uuid, nvmlRet := d.GetUUID() + if nvmlRet != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device UUID: %w", nvmlRet) + } + + nvd := nvsandboxutilsDGPU{ + lib: o.nvsandboxutilslib, + uuid: uuid, + devRoot: strings.TrimSuffix(filepath.Clean(o.devRoot), "/dev"), + isMig: o.isMigDevice, + nvidiaCDIHookPath: o.nvidiaCDIHookPath, + } + + return &nvd, nil +} + +func (d *nvsandboxutilsDGPU) Devices() ([]discover.Device, error) { + gpuFileInfos, ret := d.lib.GetGpuResource(d.uuid) + if ret != nvsandboxutils.SUCCESS { + return nil, fmt.Errorf("failed to get GPU resource: %w", ret) + } + + var devices []discover.Device + for _, info := range gpuFileInfos { + switch { + case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD: + if d.isMig { + continue + } + fallthrough + case info.SubType == nvsandboxutils.NV_DEV_NVIDIA, info.SubType == nvsandboxutils.NV_DEV_NVIDIA_CAPS_NVIDIA_CAP: + containerPath := info.Path + if d.devRoot != "/" { + containerPath = strings.TrimPrefix(containerPath, d.devRoot) + } + + // TODO: Extend discover.Device with additional information. + device := discover.Device{ + HostPath: info.Path, + Path: containerPath, + } + devices = append(devices, device) + case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD_SYMLINK, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD_SYMLINK: + if d.isMig { + continue + } + if info.Flags == nvsandboxutils.NV_FILE_FLAG_CONTENT { + targetPath, ret := d.lib.GetFileContent(info.Path) + if ret != nvsandboxutils.SUCCESS { + return nil, fmt.Errorf("failed to get symlink: %w", ret) + } + d.deviceLinks = append(d.deviceLinks, fmt.Sprintf("%v::%v", targetPath, info.Path)) + } + } + } + + return devices, nil +} + +// Hooks returns a hook to create the by-path symlinks for the discovered devices. +func (d *nvsandboxutilsDGPU) Hooks() ([]discover.Hook, error) { + if len(d.deviceLinks) == 0 { + return nil, nil + } + + var args []string + for _, l := range d.deviceLinks { + args = append(args, "--link", l) + } + + hook := discover.CreateNvidiaCDIHook( + d.nvidiaCDIHookPath, + "create-symlinks", + args..., + ) + + return []discover.Hook{hook}, nil +} + +func (d *nvsandboxutilsDGPU) Mounts() ([]discover.Mount, error) { + return nil, nil +} diff --git a/internal/platform-support/dgpu/nvsandboxutils_test.go b/internal/platform-support/dgpu/nvsandboxutils_test.go new file mode 100644 index 00000000..fb9df187 --- /dev/null +++ b/internal/platform-support/dgpu/nvsandboxutils_test.go @@ -0,0 +1,174 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package dgpu + +import ( + "testing" + + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" + "github.com/NVIDIA/go-nvml/pkg/nvml" + mocknvml "github.com/NVIDIA/go-nvml/pkg/nvml/mock" + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" + mocknvsandboxutils "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils/mock" +) + +func TestNewNvsandboxutilsDGPUDiscoverer(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + nvmllib := &mocknvml.Interface{} + devicelib := device.New( + nvmllib, + ) + + testCases := []struct { + description string + devRoot string + device nvml.Device + nvsandboxutils nvsandboxutils.Interface + expectedError error + expectedDevices []discover.Device + expectedHooks []discover.Hook + expectedMounts []discover.Mount + }{ + { + description: "detects host devices", + device: &mocknvml.Device{ + GetUUIDFunc: func() (string, nvml.Return) { + return "GPU-1234", nvml.SUCCESS + }, + }, + nvsandboxutils: &mocknvsandboxutils.Interface{ + GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + infos := []nvsandboxutils.GpuFileInfo{ + { + Path: "/dev/nvidia0", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidiactl", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidia-uvm", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidia-uvm-tools", + Type: nvsandboxutils.NV_DEV, + }, + } + return infos, nvsandboxutils.SUCCESS + }, + }, + expectedDevices: []discover.Device{ + { + Path: "/dev/nvidia0", + HostPath: "/dev/nvidia0", + }, + { + Path: "/dev/nvidiactl", + HostPath: "/dev/nvidiactl", + }, + { + Path: "/dev/nvidia-uvm", + HostPath: "/dev/nvidia-uvm", + }, + { + Path: "/dev/nvidia-uvm-tools", + HostPath: "/dev/nvidia-uvm-tools", + }, + }, + }, + { + description: "detects container devices", + devRoot: "/some/root", + device: &mocknvml.Device{ + GetUUIDFunc: func() (string, nvml.Return) { + return "GPU-1234", nvml.SUCCESS + }, + }, + nvsandboxutils: &mocknvsandboxutils.Interface{ + GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + infos := []nvsandboxutils.GpuFileInfo{ + { + Path: "/some/root/dev/nvidia0", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidiactl", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidia-uvm", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidia-uvm-tools", + Type: nvsandboxutils.NV_DEV, + }, + } + return infos, nvsandboxutils.SUCCESS + }, + }, + expectedDevices: []discover.Device{ + { + Path: "/dev/nvidia0", + HostPath: "/some/root/dev/nvidia0", + }, + { + Path: "/dev/nvidiactl", + HostPath: "/some/root/dev/nvidiactl", + }, + { + Path: "/dev/nvidia-uvm", + HostPath: "/some/root/dev/nvidia-uvm", + }, + { + Path: "/dev/nvidia-uvm-tools", + HostPath: "/some/root/dev/nvidia-uvm-tools", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + o := &options{ + logger: logger, + devRoot: tc.devRoot, + nvsandboxutilslib: tc.nvsandboxutils, + } + + device, err := devicelib.NewDevice(tc.device) + require.NoError(t, err) + + d, err := o.newNvsandboxutilsDGPUDiscoverer(device) + require.ErrorIs(t, err, tc.expectedError) + + devices, _ := d.Devices() + require.EqualValues(t, tc.expectedDevices, devices) + hooks, _ := d.Hooks() + require.EqualValues(t, tc.expectedHooks, hooks) + mounts, _ := d.Mounts() + require.EqualValues(t, tc.expectedMounts, mounts) + }) + } +} diff --git a/internal/platform-support/dgpu/options.go b/internal/platform-support/dgpu/options.go index 41e4d7a9..2fd1c01b 100644 --- a/internal/platform-support/dgpu/options.go +++ b/internal/platform-support/dgpu/options.go @@ -19,6 +19,7 @@ package dgpu import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" ) type options struct { @@ -26,10 +27,13 @@ type options struct { devRoot string nvidiaCDIHookPath string + isMigDevice bool // migCaps stores the MIG capabilities for the system. // If MIG is not available, this is nil. migCaps nvcaps.MigCaps migCapsError error + + nvsandboxutilslib nvsandboxutils.Interface } type Option func(*options) @@ -61,3 +65,10 @@ func WithMIGCaps(migCaps nvcaps.MigCaps) Option { l.migCaps = migCaps } } + +// WithNvsandboxuitilsLib sets the nvsandboxutils library implementation. +func WithNvsandboxuitilsLib(nvsandboxutilslib nvsandboxutils.Interface) Option { + return func(l *options) { + l.nvsandboxutilslib = nvsandboxutilslib + } +} diff --git a/pkg/nvcdi/full-gpu-nvml.go b/pkg/nvcdi/full-gpu-nvml.go index 881d102d..003515ca 100644 --- a/pkg/nvcdi/full-gpu-nvml.go +++ b/pkg/nvcdi/full-gpu-nvml.go @@ -72,6 +72,7 @@ func (l *nvmllib) newFullGPUDiscoverer(d device.Device) (discover.Discover, erro dgpu.WithDevRoot(l.devRoot), dgpu.WithLogger(l.logger), dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath), + dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib), ) if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err) diff --git a/pkg/nvcdi/lib-nvml.go b/pkg/nvcdi/lib-nvml.go index ab7cb8ba..d461aa51 100644 --- a/pkg/nvcdi/lib-nvml.go +++ b/pkg/nvcdi/lib-nvml.go @@ -27,6 +27,7 @@ import ( "tags.cncf.io/container-device-interface/specs-go" "github.com/NVIDIA/nvidia-container-toolkit/internal/edits" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec" ) @@ -52,6 +53,19 @@ func (l *nvmllib) GetAllDeviceSpecs() ([]specs.Device, error) { } }() + if l.nvsandboxutilslib != nil { + if r := l.nvsandboxutilslib.Init(l.driverRoot); r != nvsandboxutils.SUCCESS { + l.logger.Warningf("Failed to init nvsandboxutils: %v; ignoring", r) + l.nvsandboxutilslib = nil + } + defer func() { + if l.nvsandboxutilslib == nil { + return + } + _ = l.nvsandboxutilslib.Shutdown() + }() + } + gpuDeviceSpecs, err := l.getGPUDeviceSpecs() if err != nil { return nil, err diff --git a/pkg/nvcdi/lib.go b/pkg/nvcdi/lib.go index d2db3b6c..9e1c035b 100644 --- a/pkg/nvcdi/lib.go +++ b/pkg/nvcdi/lib.go @@ -26,6 +26,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform" @@ -43,6 +44,7 @@ type wrapper struct { type nvcdilib struct { logger logger.Interface nvmllib nvml.Interface + nvsandboxutilslib nvsandboxutils.Interface mode string devicelib device.Interface deviceNamers DeviceNamers @@ -107,6 +109,19 @@ func New(opts ...Option) (Interface, error) { } l.nvmllib = nvml.New(nvmlOpts...) } + if l.nvsandboxutilslib == nil { + var nvsandboxutilsOpts []nvsandboxutils.LibraryOption + // Set the library path for libnvidia-sandboxutils + candidates, err := l.driver.Libraries().Locate("libnvidia-sandboxutils.so.1") + if err != nil { + l.logger.Warningf("Ignoring error in locating libnvidia-sandboxutils.so.1: %v", err) + } else { + libNvidiaSandboxutilsPath := candidates[0] + l.logger.Infof("Using %v", libNvidiaSandboxutilsPath) + nvsandboxutilsOpts = append(nvsandboxutilsOpts, nvsandboxutils.WithLibraryPath(libNvidiaSandboxutilsPath)) + } + l.nvsandboxutilslib = nvsandboxutils.New(nvsandboxutilsOpts...) + } if l.devicelib == nil { l.devicelib = device.New(l.nvmllib) } @@ -214,6 +229,16 @@ func (l *nvcdilib) resolveMode() (rmode string) { // getCudaVersion returns the CUDA version of the current system. func (l *nvcdilib) getCudaVersion() (string, error) { + version, err := l.getCudaVersionNvsandboxutils() + if err == nil { + return version, err + } + + // Fallback to NVML + return l.getCudaVersionNvml() +} + +func (l *nvcdilib) getCudaVersionNvml() (string, error) { if hasNVML, reason := l.infolib.HasNvml(); !hasNVML { return "", fmt.Errorf("nvml not detected: %v", reason) } @@ -236,3 +261,12 @@ func (l *nvcdilib) getCudaVersion() (string, error) { } return version, nil } + +func (l *nvcdilib) getCudaVersionNvsandboxutils() (string, error) { + // Sandboxutils initialization should happen before this function is called + version, ret := l.nvsandboxutilslib.GetDriverVersion() + if ret != nvsandboxutils.SUCCESS { + return "", fmt.Errorf("%v", ret) + } + return version, nil +} diff --git a/pkg/nvcdi/mig-device-nvml.go b/pkg/nvcdi/mig-device-nvml.go index 91fe879c..5c1a504c 100644 --- a/pkg/nvcdi/mig-device-nvml.go +++ b/pkg/nvcdi/mig-device-nvml.go @@ -55,6 +55,7 @@ func (l *nvmllib) GetMIGDeviceEdits(parent device.Device, mig device.MigDevice) dgpu.WithDevRoot(l.devRoot), dgpu.WithLogger(l.logger), dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath), + dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib), ) if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err)