From 978d439cf8fa67d8a0cb78bb8ac3665027596fc8 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:05:05 -0700 Subject: [PATCH] nvsandboxutils: Add internal bindings This change adds the internal bindings for Sandboxutils, some of which have been automatically generated with the help of c-for-go. The format followed is similar to what is used in go-nvml. These would need to be regenerated when the header file is modified and new APIs are added. Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- internal/nvsandboxutils/api.go | 45 +++ internal/nvsandboxutils/cgo_helpers.h | 25 ++ internal/nvsandboxutils/const.go | 156 +++++++++ internal/nvsandboxutils/doc.go | 23 ++ .../nvsandboxutils/dynamicLibrary_mock.go | 157 +++++++++ internal/nvsandboxutils/lib.go | 156 +++++++++ internal/nvsandboxutils/lib_test.go | 245 +++++++++++++ internal/nvsandboxutils/mock/interface.go | 325 ++++++++++++++++++ internal/nvsandboxutils/nvsandboxutils.go | 72 ++++ internal/nvsandboxutils/nvsandboxutils.h | 298 ++++++++++++++++ internal/nvsandboxutils/refcount.go | 31 ++ internal/nvsandboxutils/refcount_test.go | 139 ++++++++ internal/nvsandboxutils/return.go | 74 ++++ internal/nvsandboxutils/types_gen.go | 39 +++ internal/nvsandboxutils/zz_generated.api.go | 43 +++ 15 files changed, 1828 insertions(+) create mode 100644 internal/nvsandboxutils/api.go create mode 100644 internal/nvsandboxutils/cgo_helpers.h create mode 100644 internal/nvsandboxutils/const.go create mode 100644 internal/nvsandboxutils/doc.go create mode 100644 internal/nvsandboxutils/dynamicLibrary_mock.go create mode 100644 internal/nvsandboxutils/lib.go create mode 100644 internal/nvsandboxutils/lib_test.go create mode 100644 internal/nvsandboxutils/mock/interface.go create mode 100644 internal/nvsandboxutils/nvsandboxutils.go create mode 100644 internal/nvsandboxutils/nvsandboxutils.h create mode 100644 internal/nvsandboxutils/refcount.go create mode 100644 internal/nvsandboxutils/refcount_test.go create mode 100644 internal/nvsandboxutils/return.go create mode 100644 internal/nvsandboxutils/types_gen.go create mode 100644 internal/nvsandboxutils/zz_generated.api.go diff --git a/internal/nvsandboxutils/api.go b/internal/nvsandboxutils/api.go new file mode 100644 index 00000000..6275a5c2 --- /dev/null +++ b/internal/nvsandboxutils/api.go @@ -0,0 +1,45 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +// libraryOptions hold the parameters than can be set by a LibraryOption +type libraryOptions struct { + path string + flags int +} + +// LibraryOption represents a functional option to configure the underlying nvsandboxutils library +type LibraryOption func(*libraryOptions) + +// WithLibraryPath provides an option to set the library name to be used by the nvsandboxutils library. +func WithLibraryPath(path string) LibraryOption { + return func(o *libraryOptions) { + o.path = path + } +} + +// SetLibraryOptions applies the specified options to the nvsandboxutils library. +// If this is called when a library is already loaded, an error is raised. +func SetLibraryOptions(opts ...LibraryOption) error { + libnvsandboxutils.Lock() + defer libnvsandboxutils.Unlock() + if libnvsandboxutils.refcount != 0 { + return errLibraryAlreadyLoaded + } + libnvsandboxutils.init(opts...) + return nil +} diff --git a/internal/nvsandboxutils/cgo_helpers.h b/internal/nvsandboxutils/cgo_helpers.h new file mode 100644 index 00000000..23b3c256 --- /dev/null +++ b/internal/nvsandboxutils/cgo_helpers.h @@ -0,0 +1,25 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +#include "nvsandboxutils.h" +#include +#pragma once + +#define __CGOGEN 1 + diff --git a/internal/nvsandboxutils/const.go b/internal/nvsandboxutils/const.go new file mode 100644 index 00000000..9e8cdf3f --- /dev/null +++ b/internal/nvsandboxutils/const.go @@ -0,0 +1,156 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package nvsandboxutils + +/* +#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup +#include "nvsandboxutils.h" +#include +#include "cgo_helpers.h" +*/ +import "C" + +const ( + // INPUT_LENGTH as defined in nvsandboxutils/nvsandboxutils.h + INPUT_LENGTH = 256 + // MAX_FILE_PATH as defined in nvsandboxutils/nvsandboxutils.h + MAX_FILE_PATH = 256 + // MAX_NAME_LENGTH as defined in nvsandboxutils/nvsandboxutils.h + MAX_NAME_LENGTH = 256 +) + +// Ret as declared in nvsandboxutils/nvsandboxutils.h +type Ret int32 + +// Ret enumeration from nvsandboxutils/nvsandboxutils.h +const ( + SUCCESS Ret = iota + ERROR_UNINITIALIZED Ret = 1 + ERROR_NOT_SUPPORTED Ret = 2 + ERROR_INVALID_ARG Ret = 3 + ERROR_INSUFFICIENT_SIZE Ret = 4 + ERROR_VERSION_NOT_SUPPORTED Ret = 5 + ERROR_LIBRARY_LOAD Ret = 6 + ERROR_FUNCTION_NOT_FOUND Ret = 7 + ERROR_DEVICE_NOT_FOUND Ret = 8 + ERROR_NVML_LIB_CALL Ret = 9 + ERROR_OUT_OF_MEMORY Ret = 10 + ERROR_FILEPATH_NOT_FOUND Ret = 11 + ERROR_UNKNOWN Ret = 65535 +) + +// LogLevel as declared in nvsandboxutils/nvsandboxutils.h +type LogLevel int32 + +// LogLevel enumeration from nvsandboxutils/nvsandboxutils.h +const ( + LOG_LEVEL_FATAL LogLevel = iota + LOG_LEVEL_ERROR LogLevel = 1 + LOG_LEVEL_WARN LogLevel = 2 + LOG_LEVEL_DEBUG LogLevel = 3 + LOG_LEVEL_INFO LogLevel = 4 + LOG_LEVEL_NONE LogLevel = 65535 +) + +// RootfsInputType as declared in nvsandboxutils/nvsandboxutils.h +type RootfsInputType int32 + +// RootfsInputType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_ROOTFS_DEFAULT RootfsInputType = iota + NV_ROOTFS_PATH RootfsInputType = 1 + NV_ROOTFS_PID RootfsInputType = 2 +) + +// FileType as declared in nvsandboxutils/nvsandboxutils.h +type FileType int32 + +// FileType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_DEV FileType = iota + NV_PROC FileType = 1 + NV_SYS FileType = 2 +) + +// FileSystemSubType as declared in nvsandboxutils/nvsandboxutils.h +type FileSystemSubType int32 + +// FileSystemSubType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_DEV_NVIDIA FileSystemSubType = iota + NV_DEV_DRI_CARD FileSystemSubType = 1 + NV_DEV_DRI_RENDERD FileSystemSubType = 2 + NV_DEV_DRI_CARD_SYMLINK FileSystemSubType = 3 + NV_DEV_DRI_RENDERD_SYMLINK FileSystemSubType = 4 + NV_DEV_NVIDIA_UVM FileSystemSubType = 5 + NV_DEV_NVIDIA_UVM_TOOLS FileSystemSubType = 6 + NV_DEV_NVIDIA_MODESET FileSystemSubType = 7 + NV_DEV_NVIDIA_CTL FileSystemSubType = 8 + NV_DEV_GDRDRV FileSystemSubType = 9 + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP FileSystemSubType = 10 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID FileSystemSubType = 11 + NV_PROC_DRIVER_NVIDIA_GPUS FileSystemSubType = 12 + NV_PROC_NVIDIA_PARAMS FileSystemSubType = 13 + NV_PROC_NVIDIA_CAPS_MIG_MINORS FileSystemSubType = 14 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU FileSystemSubType = 15 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES FileSystemSubType = 16 + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS FileSystemSubType = 17 + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID FileSystemSubType = 18 + NV_SYS_MODULE_NVIDIA_DRIVER FileSystemSubType = 19 + NV_NUM_SUBTYPE FileSystemSubType = 20 +) + +// FileModule as declared in nvsandboxutils/nvsandboxutils.h +type FileModule int32 + +// FileModule enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_GPU FileModule = iota + NV_MIG FileModule = 1 + NV_DRIVER_NVIDIA FileModule = 2 + NV_DRIVER_NVIDIA_UVM FileModule = 3 + NV_DRIVER_NVIDIA_MODESET FileModule = 4 + NV_DRIVER_GDRDRV FileModule = 5 + NV_SYSTEM FileModule = 6 +) + +// FileFlag as declared in nvsandboxutils/nvsandboxutils.h +type FileFlag int32 + +// FileFlag enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_FILE_FLAG_HINT FileFlag = 1 + NV_FILE_FLAG_MASKOUT FileFlag = 2 + NV_FILE_FLAG_CONTENT FileFlag = 4 + NV_FILE_FLAG_DEPRECTATED FileFlag = 8 + NV_FILE_FLAG_CANDIDATES FileFlag = 16 +) + +// GpuInputType as declared in nvsandboxutils/nvsandboxutils.h +type GpuInputType int32 + +// GpuInputType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_GPU_INPUT_GPU_UUID GpuInputType = iota + NV_GPU_INPUT_MIG_UUID GpuInputType = 1 + NV_GPU_INPUT_PCI_ID GpuInputType = 2 + NV_GPU_INPUT_PCI_INDEX GpuInputType = 3 +) diff --git a/internal/nvsandboxutils/doc.go b/internal/nvsandboxutils/doc.go new file mode 100644 index 00000000..231c68c2 --- /dev/null +++ b/internal/nvsandboxutils/doc.go @@ -0,0 +1,23 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +/* +Package NVSANDBOXUTILS bindings +*/ +package nvsandboxutils diff --git a/internal/nvsandboxutils/dynamicLibrary_mock.go b/internal/nvsandboxutils/dynamicLibrary_mock.go new file mode 100644 index 00000000..a22e5669 --- /dev/null +++ b/internal/nvsandboxutils/dynamicLibrary_mock.go @@ -0,0 +1,157 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package nvsandboxutils + +import ( + "sync" +) + +// Ensure, that dynamicLibraryMock does implement dynamicLibrary. +// If this is not the case, regenerate this file with moq. +var _ dynamicLibrary = &dynamicLibraryMock{} + +// dynamicLibraryMock is a mock implementation of dynamicLibrary. +// +// func TestSomethingThatUsesdynamicLibrary(t *testing.T) { +// +// // make and configure a mocked dynamicLibrary +// mockeddynamicLibrary := &dynamicLibraryMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// LookupFunc: func(s string) error { +// panic("mock out the Lookup method") +// }, +// OpenFunc: func() error { +// panic("mock out the Open method") +// }, +// } +// +// // use mockeddynamicLibrary in code that requires dynamicLibrary +// // and then make assertions. +// +// } +type dynamicLibraryMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // LookupFunc mocks the Lookup method. + LookupFunc func(s string) error + + // OpenFunc mocks the Open method. + OpenFunc func() error + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // Lookup holds details about calls to the Lookup method. + Lookup []struct { + // S is the s argument value. + S string + } + // Open holds details about calls to the Open method. + Open []struct { + } + } + lockClose sync.RWMutex + lockLookup sync.RWMutex + lockOpen sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *dynamicLibraryMock) Close() error { + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + if mock.CloseFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockeddynamicLibrary.CloseCalls()) +func (mock *dynamicLibraryMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// Lookup calls LookupFunc. +func (mock *dynamicLibraryMock) Lookup(s string) error { + callInfo := struct { + S string + }{ + S: s, + } + mock.lockLookup.Lock() + mock.calls.Lookup = append(mock.calls.Lookup, callInfo) + mock.lockLookup.Unlock() + if mock.LookupFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.LookupFunc(s) +} + +// LookupCalls gets all the calls that were made to Lookup. +// Check the length with: +// +// len(mockeddynamicLibrary.LookupCalls()) +func (mock *dynamicLibraryMock) LookupCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockLookup.RLock() + calls = mock.calls.Lookup + mock.lockLookup.RUnlock() + return calls +} + +// Open calls OpenFunc. +func (mock *dynamicLibraryMock) Open() error { + callInfo := struct { + }{} + mock.lockOpen.Lock() + mock.calls.Open = append(mock.calls.Open, callInfo) + mock.lockOpen.Unlock() + if mock.OpenFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.OpenFunc() +} + +// OpenCalls gets all the calls that were made to Open. +// Check the length with: +// +// len(mockeddynamicLibrary.OpenCalls()) +func (mock *dynamicLibraryMock) OpenCalls() []struct { +} { + var calls []struct { + } + mock.lockOpen.RLock() + calls = mock.calls.Open + mock.lockOpen.RUnlock() + return calls +} diff --git a/internal/nvsandboxutils/lib.go b/internal/nvsandboxutils/lib.go new file mode 100644 index 00000000..3a85ef89 --- /dev/null +++ b/internal/nvsandboxutils/lib.go @@ -0,0 +1,156 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "fmt" + "sync" + + "github.com/NVIDIA/go-nvml/pkg/dl" +) + +const ( + defaultNvSandboxUtilsLibraryName = "libnvidia-sandboxutils.so.1" + defaultNvSandboxUtilsLibraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL +) + +var errLibraryNotLoaded = errors.New("library not loaded") +var errLibraryAlreadyLoaded = errors.New("library already loaded") + +// dynamicLibrary is an interface for abstacting the underlying library. +// This also allows for mocking and testing. + +//go:generate moq -rm -stub -out dynamicLibrary_mock.go . dynamicLibrary +type dynamicLibrary interface { + Lookup(string) error + Open() error + Close() error +} + +// library represents an nvsandboxutils library. +// This includes a reference to the underlying DynamicLibrary +type library struct { + sync.Mutex + path string + refcount refcount + dl dynamicLibrary +} + +// libnvsandboxutils is a global instance of the nvsandboxutils library. +var libnvsandboxutils = newLibrary() + +func New(opts ...LibraryOption) Interface { + return newLibrary(opts...) +} + +func newLibrary(opts ...LibraryOption) *library { + l := &library{} + l.init(opts...) + return l +} + +func (l *library) init(opts ...LibraryOption) { + o := libraryOptions{} + for _, opt := range opts { + opt(&o) + } + + if o.path == "" { + o.path = defaultNvSandboxUtilsLibraryName + } + if o.flags == 0 { + o.flags = defaultNvSandboxUtilsLibraryLoadFlags + } + + l.path = o.path + l.dl = dl.New(o.path, o.flags) +} + +// LookupSymbol checks whether the specified library symbol exists in the library. +// Note that this requires that the library be loaded. +func (l *library) LookupSymbol(name string) error { + if l == nil || l.refcount == 0 { + return fmt.Errorf("error looking up %s: %w", name, errLibraryNotLoaded) + } + return l.dl.Lookup(name) +} + +// load initializes the library and updates the versioned symbols. +// Multiple calls to an already loaded library will return without error. +func (l *library) load() (rerr error) { + l.Lock() + defer l.Unlock() + + defer func() { l.refcount.IncOnNoError(rerr) }() + if l.refcount > 0 { + return nil + } + + if err := l.dl.Open(); err != nil { + return fmt.Errorf("error opening %s: %w", l.path, err) + } + + // Update the errorStringFunc to point to nvsandboxutils.ErrorString + errorStringFunc = nvsanboxutilsErrorString + + // Update all versioned symbols + l.updateVersionedSymbols() + + return nil +} + +// close the underlying library and ensure that the global pointer to the +// library is set to nil to ensure that subsequent calls to open will reinitialize it. +// Multiple calls to an already closed nvsandboxutils library will return without error. +func (l *library) close() (rerr error) { + l.Lock() + defer l.Unlock() + + defer func() { l.refcount.DecOnNoError(rerr) }() + if l.refcount != 1 { + return nil + } + + if err := l.dl.Close(); err != nil { + return fmt.Errorf("error closing %s: %w", l.path, err) + } + + // Update the errorStringFunc to point to defaultErrorStringFunc + errorStringFunc = defaultErrorStringFunc + + return nil +} + +// Default all versioned APIs to v1 (to infer the types) +var ( +// Insert default versions for APIs here. +// Example: +// nvsandboxUtilsFunction = nvsandboxUtilsFunction_v1 +) + +// updateVersionedSymbols checks for versioned symbols in the loaded dynamic library. +// If newer versioned symbols exist, these replace the default `v1` symbols initialized above. +// When new versioned symbols are added, these would have to be initialized above and have +// corresponding checks and subsequent assignments added below. +func (l *library) updateVersionedSymbols() { + // Example: + // err := l.dl.Lookup("nvsandboxUtilsFunction_v2") + // if err == nil { + // nvsandboxUtilsFunction = nvsandboxUtilsFunction_v2 + // } +} diff --git a/internal/nvsandboxutils/lib_test.go b/internal/nvsandboxutils/lib_test.go new file mode 100644 index 00000000..b87c7242 --- /dev/null +++ b/internal/nvsandboxutils/lib_test.go @@ -0,0 +1,245 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestLibrary(dl dynamicLibrary) *library { + return &library{dl: dl} +} + +func TestLookupFromDefault(t *testing.T) { + errClose := errors.New("close error") + errOpen := errors.New("open error") + errLookup := errors.New("lookup error") + + testCases := []struct { + description string + dl dynamicLibrary + skipLoadLibrary bool + expectedLoadError error + expectedLookupErrror error + expectedCloseError error + }{ + { + description: "library not loaded yields error", + dl: &dynamicLibraryMock{}, + skipLoadLibrary: true, + expectedLookupErrror: errLibraryNotLoaded, + }, + { + description: "open error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errOpen + }, + }, + + expectedLoadError: errOpen, + expectedLookupErrror: errLibraryNotLoaded, + }, + { + description: "lookup error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return fmt.Errorf("%w: %s", errLookup, s) + }, + CloseFunc: func() error { + return nil + }, + }, + + expectedLookupErrror: errLookup, + }, + { + description: "lookup succeeds", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + }, + { + description: "lookup succeeds", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + }, + { + description: "close error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return errClose + }, + }, + expectedCloseError: errClose, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l := newTestLibrary(tc.dl) + if !tc.skipLoadLibrary { + require.ErrorIs(t, l.load(), tc.expectedLoadError) + } + require.ErrorIs(t, l.LookupSymbol("symbol"), tc.expectedLookupErrror) + require.ErrorIs(t, l.close(), tc.expectedCloseError) + if tc.expectedCloseError == nil { + require.Equal(t, 0, int(l.refcount)) + } else { + require.Equal(t, 1, int(l.refcount)) + } + }) + } +} + +func TestLoadAndCloseNesting(t *testing.T) { + dl := &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return nil + }, + } + + l := newTestLibrary(dl) + + // When calling close before opening the library nothing happens. + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 0, len(dl.calls.Close)) + + // When calling load twice, the library was only opened once + require.Equal(t, 0, len(dl.calls.Open)) + require.Nil(t, l.load()) + require.Equal(t, 1, len(dl.calls.Open)) + require.Nil(t, l.load()) + require.Equal(t, 1, len(dl.calls.Open)) + + // Only after calling close twice, was the library closed + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 1, len(dl.calls.Close)) + + // Calling close again doesn't attempt to close the library again + require.Nil(t, l.close()) + require.Equal(t, 1, len(dl.calls.Close)) +} + +func TestLoadAndCloseWithErrors(t *testing.T) { + testCases := []struct { + description string + dl dynamicLibrary + expectedLoadRefcount refcount + expectedCloseRefcount refcount + }{ + { + description: "regular flow", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + expectedLoadRefcount: 1, + expectedCloseRefcount: 0, + }, + { + description: "open error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errors.New("") + }, + CloseFunc: func() error { + return nil + }, + }, + expectedLoadRefcount: 0, + expectedCloseRefcount: 0, + }, + { + description: "close error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return errors.New("") + }, + }, + expectedLoadRefcount: 1, + expectedCloseRefcount: 1, + }, + { + description: "open and close error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errors.New("") + }, + CloseFunc: func() error { + return errors.New("") + }, + }, + expectedLoadRefcount: 0, + expectedCloseRefcount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l := newTestLibrary(tc.dl) + _ = l.load() + require.Equal(t, tc.expectedLoadRefcount, l.refcount) + _ = l.close() + require.Equal(t, tc.expectedCloseRefcount, l.refcount) + }) + } +} diff --git a/internal/nvsandboxutils/mock/interface.go b/internal/nvsandboxutils/mock/interface.go new file mode 100644 index 00000000..d7b27ebf --- /dev/null +++ b/internal/nvsandboxutils/mock/interface.go @@ -0,0 +1,325 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" + "sync" +) + +// Ensure, that Interface does implement nvsandboxutils.Interface. +// If this is not the case, regenerate this file with moq. +var _ nvsandboxutils.Interface = &Interface{} + +// Interface is a mock implementation of nvsandboxutils.Interface. +// +// func TestSomethingThatUsesInterface(t *testing.T) { +// +// // make and configure a mocked nvsandboxutils.Interface +// mockedInterface := &Interface{ +// ErrorStringFunc: func(ret nvsandboxutils.Ret) string { +// panic("mock out the ErrorString method") +// }, +// GetDriverVersionFunc: func() (string, nvsandboxutils.Ret) { +// panic("mock out the GetDriverVersion method") +// }, +// GetFileContentFunc: func(s string) (string, nvsandboxutils.Ret) { +// panic("mock out the GetFileContent method") +// }, +// GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { +// panic("mock out the GetGpuResource method") +// }, +// InitFunc: func(s string) nvsandboxutils.Ret { +// panic("mock out the Init method") +// }, +// LookupSymbolFunc: func(s string) error { +// panic("mock out the LookupSymbol method") +// }, +// ShutdownFunc: func() nvsandboxutils.Ret { +// panic("mock out the Shutdown method") +// }, +// } +// +// // use mockedInterface in code that requires nvsandboxutils.Interface +// // and then make assertions. +// +// } +type Interface struct { + // ErrorStringFunc mocks the ErrorString method. + ErrorStringFunc func(ret nvsandboxutils.Ret) string + + // GetDriverVersionFunc mocks the GetDriverVersion method. + GetDriverVersionFunc func() (string, nvsandboxutils.Ret) + + // GetFileContentFunc mocks the GetFileContent method. + GetFileContentFunc func(s string) (string, nvsandboxutils.Ret) + + // GetGpuResourceFunc mocks the GetGpuResource method. + GetGpuResourceFunc func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) + + // InitFunc mocks the Init method. + InitFunc func(s string) nvsandboxutils.Ret + + // LookupSymbolFunc mocks the LookupSymbol method. + LookupSymbolFunc func(s string) error + + // ShutdownFunc mocks the Shutdown method. + ShutdownFunc func() nvsandboxutils.Ret + + // calls tracks calls to the methods. + calls struct { + // ErrorString holds details about calls to the ErrorString method. + ErrorString []struct { + // Ret is the ret argument value. + Ret nvsandboxutils.Ret + } + // GetDriverVersion holds details about calls to the GetDriverVersion method. + GetDriverVersion []struct { + } + // GetFileContent holds details about calls to the GetFileContent method. + GetFileContent []struct { + // S is the s argument value. + S string + } + // GetGpuResource holds details about calls to the GetGpuResource method. + GetGpuResource []struct { + // S is the s argument value. + S string + } + // Init holds details about calls to the Init method. + Init []struct { + // S is the s argument value. + S string + } + // LookupSymbol holds details about calls to the LookupSymbol method. + LookupSymbol []struct { + // S is the s argument value. + S string + } + // Shutdown holds details about calls to the Shutdown method. + Shutdown []struct { + } + } + lockErrorString sync.RWMutex + lockGetDriverVersion sync.RWMutex + lockGetFileContent sync.RWMutex + lockGetGpuResource sync.RWMutex + lockInit sync.RWMutex + lockLookupSymbol sync.RWMutex + lockShutdown sync.RWMutex +} + +// ErrorString calls ErrorStringFunc. +func (mock *Interface) ErrorString(ret nvsandboxutils.Ret) string { + if mock.ErrorStringFunc == nil { + panic("Interface.ErrorStringFunc: method is nil but Interface.ErrorString was just called") + } + callInfo := struct { + Ret nvsandboxutils.Ret + }{ + Ret: ret, + } + mock.lockErrorString.Lock() + mock.calls.ErrorString = append(mock.calls.ErrorString, callInfo) + mock.lockErrorString.Unlock() + return mock.ErrorStringFunc(ret) +} + +// ErrorStringCalls gets all the calls that were made to ErrorString. +// Check the length with: +// +// len(mockedInterface.ErrorStringCalls()) +func (mock *Interface) ErrorStringCalls() []struct { + Ret nvsandboxutils.Ret +} { + var calls []struct { + Ret nvsandboxutils.Ret + } + mock.lockErrorString.RLock() + calls = mock.calls.ErrorString + mock.lockErrorString.RUnlock() + return calls +} + +// GetDriverVersion calls GetDriverVersionFunc. +func (mock *Interface) GetDriverVersion() (string, nvsandboxutils.Ret) { + if mock.GetDriverVersionFunc == nil { + panic("Interface.GetDriverVersionFunc: method is nil but Interface.GetDriverVersion was just called") + } + callInfo := struct { + }{} + mock.lockGetDriverVersion.Lock() + mock.calls.GetDriverVersion = append(mock.calls.GetDriverVersion, callInfo) + mock.lockGetDriverVersion.Unlock() + return mock.GetDriverVersionFunc() +} + +// GetDriverVersionCalls gets all the calls that were made to GetDriverVersion. +// Check the length with: +// +// len(mockedInterface.GetDriverVersionCalls()) +func (mock *Interface) GetDriverVersionCalls() []struct { +} { + var calls []struct { + } + mock.lockGetDriverVersion.RLock() + calls = mock.calls.GetDriverVersion + mock.lockGetDriverVersion.RUnlock() + return calls +} + +// GetFileContent calls GetFileContentFunc. +func (mock *Interface) GetFileContent(s string) (string, nvsandboxutils.Ret) { + if mock.GetFileContentFunc == nil { + panic("Interface.GetFileContentFunc: method is nil but Interface.GetFileContent was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockGetFileContent.Lock() + mock.calls.GetFileContent = append(mock.calls.GetFileContent, callInfo) + mock.lockGetFileContent.Unlock() + return mock.GetFileContentFunc(s) +} + +// GetFileContentCalls gets all the calls that were made to GetFileContent. +// Check the length with: +// +// len(mockedInterface.GetFileContentCalls()) +func (mock *Interface) GetFileContentCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockGetFileContent.RLock() + calls = mock.calls.GetFileContent + mock.lockGetFileContent.RUnlock() + return calls +} + +// GetGpuResource calls GetGpuResourceFunc. +func (mock *Interface) GetGpuResource(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + if mock.GetGpuResourceFunc == nil { + panic("Interface.GetGpuResourceFunc: method is nil but Interface.GetGpuResource was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockGetGpuResource.Lock() + mock.calls.GetGpuResource = append(mock.calls.GetGpuResource, callInfo) + mock.lockGetGpuResource.Unlock() + return mock.GetGpuResourceFunc(s) +} + +// GetGpuResourceCalls gets all the calls that were made to GetGpuResource. +// Check the length with: +// +// len(mockedInterface.GetGpuResourceCalls()) +func (mock *Interface) GetGpuResourceCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockGetGpuResource.RLock() + calls = mock.calls.GetGpuResource + mock.lockGetGpuResource.RUnlock() + return calls +} + +// Init calls InitFunc. +func (mock *Interface) Init(s string) nvsandboxutils.Ret { + if mock.InitFunc == nil { + panic("Interface.InitFunc: method is nil but Interface.Init was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockInit.Lock() + mock.calls.Init = append(mock.calls.Init, callInfo) + mock.lockInit.Unlock() + return mock.InitFunc(s) +} + +// InitCalls gets all the calls that were made to Init. +// Check the length with: +// +// len(mockedInterface.InitCalls()) +func (mock *Interface) InitCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockInit.RLock() + calls = mock.calls.Init + mock.lockInit.RUnlock() + return calls +} + +// LookupSymbol calls LookupSymbolFunc. +func (mock *Interface) LookupSymbol(s string) error { + if mock.LookupSymbolFunc == nil { + panic("Interface.LookupSymbolFunc: method is nil but Interface.LookupSymbol was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockLookupSymbol.Lock() + mock.calls.LookupSymbol = append(mock.calls.LookupSymbol, callInfo) + mock.lockLookupSymbol.Unlock() + return mock.LookupSymbolFunc(s) +} + +// LookupSymbolCalls gets all the calls that were made to LookupSymbol. +// Check the length with: +// +// len(mockedInterface.LookupSymbolCalls()) +func (mock *Interface) LookupSymbolCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockLookupSymbol.RLock() + calls = mock.calls.LookupSymbol + mock.lockLookupSymbol.RUnlock() + return calls +} + +// Shutdown calls ShutdownFunc. +func (mock *Interface) Shutdown() nvsandboxutils.Ret { + if mock.ShutdownFunc == nil { + panic("Interface.ShutdownFunc: method is nil but Interface.Shutdown was just called") + } + callInfo := struct { + }{} + mock.lockShutdown.Lock() + mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo) + mock.lockShutdown.Unlock() + return mock.ShutdownFunc() +} + +// ShutdownCalls gets all the calls that were made to Shutdown. +// Check the length with: +// +// len(mockedInterface.ShutdownCalls()) +func (mock *Interface) ShutdownCalls() []struct { +} { + var calls []struct { + } + mock.lockShutdown.RLock() + calls = mock.calls.Shutdown + mock.lockShutdown.RUnlock() + return calls +} diff --git a/internal/nvsandboxutils/nvsandboxutils.go b/internal/nvsandboxutils/nvsandboxutils.go new file mode 100644 index 00000000..29544bc9 --- /dev/null +++ b/internal/nvsandboxutils/nvsandboxutils.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package nvsandboxutils + +/* +#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup +#include "nvsandboxutils.h" +#include +#include "cgo_helpers.h" +*/ +import "C" +import "unsafe" + +// nvSandboxUtilsInit function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsInit(Input *InitInput) Ret { + cInput, _ := (*C.nvSandboxUtilsInitInput_t)(unsafe.Pointer(Input)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsInit(cInput) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsShutdown function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsShutdown() Ret { + __ret := C.nvSandboxUtilsShutdown() + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetDriverVersion function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetDriverVersion(Version *byte, Length uint32) Ret { + cVersion, _ := (*C.char)(unsafe.Pointer(Version)), cgoAllocsUnknown + cLength, _ := (C.uint)(Length), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetDriverVersion(cVersion, cLength) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetGpuResource function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetGpuResource(Request *GpuRes) Ret { + cRequest, _ := (*C.nvSandboxUtilsGpuRes_t)(unsafe.Pointer(Request)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetGpuResource(cRequest) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetFileContent function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetFileContent(FilePath *byte, Content *byte, ContentSize *uint32) Ret { + cFilePath, _ := (*C.char)(unsafe.Pointer(FilePath)), cgoAllocsUnknown + cContent, _ := (*C.char)(unsafe.Pointer(Content)), cgoAllocsUnknown + cContentSize, _ := (*C.uint)(unsafe.Pointer(ContentSize)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetFileContent(cFilePath, cContent, cContentSize) + __v := (Ret)(__ret) + return __v +} diff --git a/internal/nvsandboxutils/nvsandboxutils.h b/internal/nvsandboxutils/nvsandboxutils.h new file mode 100644 index 00000000..3c66e159 --- /dev/null +++ b/internal/nvsandboxutils/nvsandboxutils.h @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVSANDBOXUTILS_H__ +#define __NVSANDBOXUTILS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define INPUT_LENGTH 256 +#define MAX_FILE_PATH 256 +#define MAX_NAME_LENGTH 256 + +/***************************************************************************************************/ +/** @defgroup enums Enumerations + * @{ + */ +/***************************************************************************************************/ + +/** + * Return types + */ +typedef enum +{ + NVSANDBOXUTILS_SUCCESS = 0, //!< The operation was successful + NVSANDBOXUTILS_ERROR_UNINITIALIZED = 1, //!< The library wasn't successfully initialized + NVSANDBOXUTILS_ERROR_NOT_SUPPORTED = 2, //!< The requested operation is not supported on target device + NVSANDBOXUTILS_ERROR_INVALID_ARG = 3, //!< A supplied argument is invalid + NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE = 4, //!< A supplied argument is not large enough + NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED = 5, //!< Requested library version is not supported + NVSANDBOXUTILS_ERROR_LIBRARY_LOAD = 6, //!< The library load failed + NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND = 7, //!< Called function was not found + NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND = 8, //!< Target device was not found + NVSANDBOXUTILS_ERROR_NVML_LIB_CALL = 9, //!< NVML library call failed + NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY = 10, //!< There is insufficient memory + NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND = 11, //!< A supplied file path was not found + NVSANDBOXUTILS_ERROR_UNKNOWN = 0xFFFF, //!< Unknown error occurred +} nvSandboxUtilsRet_t; + +/** + * Return if there is an error + */ +#define RETURN_ON_SANDBOX_ERROR(result) \ + if ((result) != NVSANDBOXUTILS_SUCCESS) { \ + NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \ + return result; \ + } + +/** + * Log levels + */ +typedef enum +{ + NVSANDBOXUTILS_LOG_LEVEL_FATAL = 0, //!< Log fatal errors + NVSANDBOXUTILS_LOG_LEVEL_ERROR = 1, //!< Log all errors + NVSANDBOXUTILS_LOG_LEVEL_WARN = 2, //!< Log all warnings + NVSANDBOXUTILS_LOG_LEVEL_DEBUG = 3, //!< Log all debug messages + NVSANDBOXUTILS_LOG_LEVEL_INFO = 4, //!< Log all info messages + NVSANDBOXUTILS_LOG_LEVEL_NONE = 0xFFFF, //!< Log none +} nvSandboxUtilsLogLevel_t; + +/** + * Input rootfs to help access files inside the driver container + */ +typedef enum +{ + NV_ROOTFS_DEFAULT, //!< Default no rootfs + NV_ROOTFS_PATH, //!< /run/nvidia/driver + NV_ROOTFS_PID, //!< /proc/PID/mountinfo +} nvSandboxUtilsRootfsInputType_t; + +/** + * File type + */ +typedef enum +{ + NV_DEV, //!< /dev file system + NV_PROC, //!< /proc file system + NV_SYS, //!< /sys file system +} nvSandboxUtilsFileType_t; + +/** + * File subtype + */ +typedef enum +{ + NV_DEV_NVIDIA, //!< /dev/nvidia0 + NV_DEV_DRI_CARD, //!< /dev/dri/card1 + NV_DEV_DRI_RENDERD, //!< /dev/dri/renderD128 + NV_DEV_DRI_CARD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-card + NV_DEV_DRI_RENDERD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-render + NV_DEV_NVIDIA_UVM, //!< /dev/nvidia-uvm + NV_DEV_NVIDIA_UVM_TOOLS, //!< /dev/nvidia-uvm-tools + NV_DEV_NVIDIA_MODESET, //!< /dev/nvidia-uvm-modeset + NV_DEV_NVIDIA_CTL, //!< /dev/nvidiactl + NV_DEV_GDRDRV, //!< /dev/gdrdrv + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP, //!< /dev/nvidia-caps/nvidia-cap22 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID, //!< /proc/driver/nvidia/gpus/0000:2d:00.0 + NV_PROC_DRIVER_NVIDIA_GPUS, //!< /proc/driver/nvidia/gpus (for mask out) + NV_PROC_NVIDIA_PARAMS, //!< /proc/driver/nvidia/params + NV_PROC_NVIDIA_CAPS_MIG_MINORS, //!< /proc/driver/nvidia-caps/mig-minors + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU, //!< /proc/driver/nvidia/capabilities/gpu0 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES, //!< /proc/driver/nvidia/capabilities (for mask out) + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS, //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID, //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0 + NV_SYS_MODULE_NVIDIA_DRIVER, //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out) + NV_NUM_SUBTYPE, // always at the end. +} nvSandboxUtilsFileSystemSubType_t; + +/** + * File module + */ +typedef enum +{ + NV_GPU, //!< Target device + NV_MIG, //!< Target device- MIG + NV_DRIVER_NVIDIA, //!< NVIDIA kernel driver + NV_DRIVER_NVIDIA_UVM, //!< NVIDIA kernel driver-UVM + NV_DRIVER_NVIDIA_MODESET, //!< NVIDIA kernel driver-modeset + NV_DRIVER_GDRDRV, //!< GDRDRV driver + NV_SYSTEM, //!< System module +} nvSandboxUtilsFileModule_t; + +/** + * Flag to provide additional details about the file + */ +typedef enum +{ + NV_FILE_FLAG_HINT = (1 << 0), //!< Default no hint + NV_FILE_FLAG_MASKOUT = (1 << 1), //!< For /proc/driver/nvidia/gpus + NV_FILE_FLAG_CONTENT = (1 << 2), //!< For /proc/driver/nvidia/params + //!< For SYMLINK + //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file + NV_FILE_FLAG_DEPRECTATED = (1 << 3), //!< For all the FIRMWARE GSP file + NV_FILE_FLAG_CANDIDATES = (1 << 4), //!< For libcuda.so +} nvSandboxUtilsFileFlag_t; + +/** + * Input type of the target device + */ +typedef enum +{ + NV_GPU_INPUT_GPU_UUID, //!< GPU UUID + NV_GPU_INPUT_MIG_UUID, //!< MIG UUID + NV_GPU_INPUT_PCI_ID, //!< PCIe DBDF ID + NV_GPU_INPUT_PCI_INDEX, //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF) +} nvSandboxUtilsGpuInputType_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup dataTypes Structures and Unions + * @{ + */ +/***************************************************************************************************/ + +/** + * Initalization input v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsRootfsInputType_t type; //!< One of \p nvSandboxUtilsRootfsInputType_t + char value[INPUT_LENGTH]; //!< String representation of input +} nvSandboxUtilsInitInput_v1_t; + +typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t; + +/** + * File system information + */ +typedef struct nvSandboxUtilsGpuFileInfo_v1_t +{ + struct nvSandboxUtilsGpuFileInfo_v1_t *next; //!< Pointer to the next node in the linked list + nvSandboxUtilsFileType_t fileType; //!< One of \p nvSandboxUtilsFileType_t + nvSandboxUtilsFileSystemSubType_t fileSubType; //!< One of \p nvSandboxUtilsFileSystemSubType_t + nvSandboxUtilsFileModule_t module; //!< One of \p nvSandboxUtilsFileModule_t + nvSandboxUtilsFileFlag_t flags; //!< One of \p nvSandboxUtilsFileFlag_t + char *filePath; //!< Relative file path to rootfs +}nvSandboxUtilsGpuFileInfo_v1_t; + +/** + * GPU resource request v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsGpuInputType_t inputType; //!< One of \p nvSandboxUtilsGpuInputType_t + char input[INPUT_LENGTH]; //!< String representation of input + nvSandboxUtilsGpuFileInfo_v1_t *files; //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t +} nvSandboxUtilsGpuRes_v1_t; + +typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup funcs Functions + * @{ + */ +/***************************************************************************************************/ + +/* ************************************************* + * Initialize library + * ************************************************* + */ +/** + * Prepare library resources before library API can be used. + * This initialization will not fail if one of the initialization prerequisites fails. + * @param input Reference to the called-supplied input struct that has initialization fields + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p input->value isn't a valid rootfs path + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p input->version isn't supported by the library + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND if any of the required file paths are not found during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY if there is insufficient system memory during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD on any error during loading the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input); + +/* ************************************************* + * Shutdown library + * ************************************************* + */ +/** + * Clean up library resources created by init call + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + */ +nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void); + +/* ************************************************* + * Get NVIDIA RM driver version + * ************************************************* + */ +/** + * Get NVIDIA RM driver version + * @param version Reference to caller-supplied buffer to return driver version string + * @param length The maximum allowed length of the string returned in \p version + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p version is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL on any error during driver version query from NVML + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length); + +/* ************************************************* + * Get /dev, /proc, /sys file system information + * ************************************************* + */ +/** + * Get /dev, /proc, /sys file system information + * @param request Reference to caller-supplied request struct to return the file system information + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p request->input doesn't match any device + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p request->version isn't supported by the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request); + +/* ************************************************* + * Get content of given file path + * ************************************************* + */ +/** + * Get file content of input file path + * @param filePath Reference to the file path + * @param content Reference to the caller-supplied buffer to return the file content + * @param contentSize Reference to the maximum allowed size of content. It is updated to the actual size of the content on return + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p filePath or \p content is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE if \p contentSize is too small + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND on an error while obtaining the content for the file path + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize); + +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif // __NVSANDBOXUTILS_H__ diff --git a/internal/nvsandboxutils/refcount.go b/internal/nvsandboxutils/refcount.go new file mode 100644 index 00000000..f93107b0 --- /dev/null +++ b/internal/nvsandboxutils/refcount.go @@ -0,0 +1,31 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +type refcount int + +func (r *refcount) IncOnNoError(err error) { + if err == nil { + (*r)++ + } +} + +func (r *refcount) DecOnNoError(err error) { + if err == nil && (*r) > 0 { + (*r)-- + } +} diff --git a/internal/nvsandboxutils/refcount_test.go b/internal/nvsandboxutils/refcount_test.go new file mode 100644 index 00000000..bd311e10 --- /dev/null +++ b/internal/nvsandboxutils/refcount_test.go @@ -0,0 +1,139 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefcount(t *testing.T) { + testCases := []struct { + description string + workload func(r *refcount) + expectedRefcount refcount + }{ + { + description: "No inc or dec", + workload: func(r *refcount) {}, + expectedRefcount: refcount(0), + }, + { + description: "Single inc, no error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + }, + expectedRefcount: refcount(1), + }, + { + description: "Single inc, with error", + workload: func(r *refcount) { + r.IncOnNoError(errors.New("")) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, no error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + }, + expectedRefcount: refcount(2), + }, + { + description: "Double inc, one with error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(errors.New("")) + }, + expectedRefcount: refcount(1), + }, + { + description: "Single dec, no error", + workload: func(r *refcount) { + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Single dec, with error", + workload: func(r *refcount) { + r.DecOnNoError(errors.New("")) + }, + expectedRefcount: refcount(0), + }, + { + description: "Single inc, single dec, no errors", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, no errors", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, one inc error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(errors.New("")) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, one dec error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(nil) + r.DecOnNoError(errors.New("")) + }, + expectedRefcount: refcount(1), + }, + { + description: "Double inc, Tripple dec, one dec error early on", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(errors.New("")) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + var r refcount + tc.workload(&r) + require.Equal(t, tc.expectedRefcount, r) + }) + } +} diff --git a/internal/nvsandboxutils/return.go b/internal/nvsandboxutils/return.go new file mode 100644 index 00000000..90d4ed84 --- /dev/null +++ b/internal/nvsandboxutils/return.go @@ -0,0 +1,74 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "fmt" +) + +// nvsandboxutils.ErrorString() +func (l *library) ErrorString(r Ret) string { + return r.Error() +} + +// String returns the string representation of a Ret. +func (r Ret) String() string { + return r.Error() +} + +// Error returns the string representation of a Ret. +func (r Ret) Error() string { + return errorStringFunc(r) +} + +// Assigned to nvsandboxutils.ErrorString if the system nvsandboxutils library is in use. +var errorStringFunc = defaultErrorStringFunc + +// nvsanboxutilsErrorString is an alias for the default error string function. +var nvsanboxutilsErrorString = defaultErrorStringFunc + +// defaultErrorStringFunc provides a basic nvsandboxutils.ErrorString implementation. +// This allows the nvsandboxutils.ErrorString function to be used even if the nvsandboxutils library +// is not loaded. +var defaultErrorStringFunc = func(r Ret) string { + switch r { + case SUCCESS: + return "SUCCESS" + case ERROR_UNINITIALIZED: + return "ERROR_UNINITIALIZED" + case ERROR_NOT_SUPPORTED: + return "ERROR_NOT_SUPPORTED" + case ERROR_INVALID_ARG: + return "ERROR_INVALID_ARG" + case ERROR_INSUFFICIENT_SIZE: + return "ERROR_INSUFFICIENT_SIZE" + case ERROR_VERSION_NOT_SUPPORTED: + return "ERROR_VERSION_NOT_SUPPORTED" + case ERROR_LIBRARY_LOAD: + return "ERROR_LIBRARY_LOAD" + case ERROR_FUNCTION_NOT_FOUND: + return "ERROR_FUNCTION_NOT_FOUND" + case ERROR_DEVICE_NOT_FOUND: + return "ERROR_DEVICE_NOT_FOUND" + case ERROR_NVML_LIB_CALL: + return "ERROR_NVML_LIB_CALL" + case ERROR_UNKNOWN: + return "ERROR_UNKNOWN" + default: + return fmt.Sprintf("unknown return value: %d", r) + } +} diff --git a/internal/nvsandboxutils/types_gen.go b/internal/nvsandboxutils/types_gen.go new file mode 100644 index 00000000..90a00ed6 --- /dev/null +++ b/internal/nvsandboxutils/types_gen.go @@ -0,0 +1,39 @@ +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types.go + +package nvsandboxutils + +type InitInput_v1 struct { + Version uint32 + Type uint32 + Value [256]int8 +} + +type InitInput struct { + Version uint32 + Type uint32 + Value [256]int8 +} + +type GpuFileInfo_v1 struct { + Next *GpuFileInfo_v1 + FileType uint32 + FileSubType uint32 + Module uint32 + Flags uint32 + FilePath *int8 +} + +type GpuRes_v1 struct { + Version uint32 + InputType uint32 + Input [256]int8 + Files *GpuFileInfo_v1 +} + +type GpuRes struct { + Version uint32 + InputType uint32 + Input [256]int8 + Files *GpuFileInfo_v1 +} diff --git a/internal/nvsandboxutils/zz_generated.api.go b/internal/nvsandboxutils/zz_generated.api.go new file mode 100644 index 00000000..1cc2b01a --- /dev/null +++ b/internal/nvsandboxutils/zz_generated.api.go @@ -0,0 +1,43 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// Generated Code; DO NOT EDIT. + +package nvsandboxutils + +// The variables below represent package level methods from the library type. +var ( + ErrorString = libnvsandboxutils.ErrorString + GetDriverVersion = libnvsandboxutils.GetDriverVersion + GetFileContent = libnvsandboxutils.GetFileContent + GetGpuResource = libnvsandboxutils.GetGpuResource + Init = libnvsandboxutils.Init + LookupSymbol = libnvsandboxutils.LookupSymbol + Shutdown = libnvsandboxutils.Shutdown +) + +// Interface represents the interface for the library type. +// +//go:generate moq -out mock/interface.go -pkg mock . Interface:Interface +type Interface interface { + ErrorString(Ret) string + GetDriverVersion() (string, Ret) + GetFileContent(string) (string, Ret) + GetGpuResource(string) ([]GpuFileInfo, Ret) + Init(string) Ret + LookupSymbol(string) error + Shutdown() Ret +}