From aa946f3f59c82c58f83d3f868eb11c8571405a31 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:04:48 -0700 Subject: [PATCH 1/5] nvsandboxutils: Add script to generate bindings This change adds a script and related files to generate the internal bindings for Sandboxutils library with the help of c-for-go. This can be used to update the bindings when the header file is modified with reference to how they are generated with the Makefile in go-nvml. Run: ./update-bindings.sh Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- .../nvsandboxutils/gen/generate-bindings.sh | 50 +++ .../nvsandboxutils/anonymous_structs.cocci | 100 +++++ .../gen/nvsandboxutils/generateapi.go | 389 ++++++++++++++++++ .../gen/nvsandboxutils/nvsandboxutils.h | 298 ++++++++++++++ .../gen/nvsandboxutils/nvsandboxutils.yml | 66 +++ .../nvsandboxutils/gen/update-bindings.sh | 41 ++ 6 files changed, 944 insertions(+) create mode 100755 internal/nvsandboxutils/gen/generate-bindings.sh create mode 100644 internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci create mode 100644 internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go create mode 100644 internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h create mode 100644 internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml create mode 100755 internal/nvsandboxutils/gen/update-bindings.sh diff --git a/internal/nvsandboxutils/gen/generate-bindings.sh b/internal/nvsandboxutils/gen/generate-bindings.sh new file mode 100755 index 00000000..e32641a3 --- /dev/null +++ b/internal/nvsandboxutils/gen/generate-bindings.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file generates bindings for nvsandboxutils by calling c-for-go. + +set -x -e + +PWD=$(pwd) +GEN_DIR="$PWD/gen" +PKG_DIR="$PWD" +GEN_BINDINGS_DIR="$GEN_DIR/nvsandboxutils" +PKG_BINDINGS_DIR="$PKG_DIR" + +SOURCES=$(find "$GEN_BINDINGS_DIR" -type f) + +mkdir -p "$PKG_BINDINGS_DIR" + +cp "$GEN_BINDINGS_DIR/nvsandboxutils.h" "$PKG_BINDINGS_DIR/nvsandboxutils.h" +spatch --in-place --very-quiet --sp-file "$GEN_BINDINGS_DIR/anonymous_structs.cocci" "$PKG_BINDINGS_DIR/nvsandboxutils.h" > /dev/null + +echo "Generating the bindings..." +c-for-go -out "$PKG_DIR/.." "$GEN_BINDINGS_DIR/nvsandboxutils.yml" +cd "$PKG_BINDINGS_DIR" +go tool cgo -godefs types.go > types_gen.go +go fmt types_gen.go +cd - > /dev/null +rm -rf "$PKG_BINDINGS_DIR/cgo_helpers.go" "$PKG_BINDINGS_DIR/types.go" "$PKG_BINDINGS_DIR/_obj" +go run "$GEN_BINDINGS_DIR/generateapi.go" --sourceDir "$PKG_BINDINGS_DIR" --output "$PKG_BINDINGS_DIR/zz_generated.api.go" +# go fmt "$PKG_BINDINGS_DIR" + +SED_SEARCH_STRING='// WARNING: This file has automatically been generated on' +SED_REPLACE_STRING='// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.' +grep -l -R "$SED_SEARCH_STRING" "$PKG_DIR" | grep -v "/gen/" | xargs sed -i -E "s#$SED_SEARCH_STRING.*\$#$SED_REPLACE_STRING#g" + +SED_SEARCH_STRING='// (.*) nvsandboxutils/nvsandboxutils.h:[0-9]+' +SED_REPLACE_STRING='// \1 nvsandboxutils/nvsandboxutils.h' +grep -l -RE "$SED_SEARCH_STRING" "$PKG_DIR" | grep -v "/gen/" | xargs sed -i -E "s#$SED_SEARCH_STRING\$#$SED_REPLACE_STRING#g" + diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci b/internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci new file mode 100644 index 00000000..c8028fb5 --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci @@ -0,0 +1,100 @@ +@patch@ +type WRAPPER_TYPE; +field list FIELDS; +identifier V; +expression E; +fresh identifier ST = "nvSandboxUtilsGenerated_struct___"; +fresh identifier TEMP_VAR = "nvSandboxUtilsGenerated_variable___" ## V; +@@ + +++ struct ST { +++ WRAPPER_TYPE TEMP_VAR; +++ FIELDS +++ }; ++ + +WRAPPER_TYPE +{ + ... +( +- struct { +- FIELDS +- } V[E]; ++ struct ST V[E]; + +| + +- struct { +- FIELDS +- } V; ++ struct ST V; +) + ... +}; + +@capture@ +type WRAPPER_TYPE; +identifier TEMP_VAR; +identifier ST =~ "^nvSandboxUtilsGenerated_struct___"; +@@ + +struct ST { + WRAPPER_TYPE TEMP_VAR; + ... +}; + +@script:python concat@ +WRAPPER_TYPE << capture.WRAPPER_TYPE; +TEMP_VAR << capture.TEMP_VAR; +ST << capture.ST; +T; +@@ + +def removePrefix(string, prefix): + if string.startswith(prefix): + return string[len(prefix):] + return string + +def removeSuffix(string, suffix): + if string.endswith(suffix): + return string[:-len(suffix)] + return string + +WRAPPER_TYPE = removeSuffix(WRAPPER_TYPE, "_t") +TEMP_VAR = removePrefix(TEMP_VAR, "nvSandboxUtilsGenerated_variable___") +coccinelle.T = cocci.make_type(WRAPPER_TYPE + TEMP_VAR[0].upper() + TEMP_VAR[1:] + "_t") + +@add_typedef@ +identifier capture.ST; +type concat.T; +type WRAPPER_TYPE; +identifier TEMP_VAR; +@@ + +- struct ST { ++ typedef struct { +- WRAPPER_TYPE TEMP_VAR; + ... +- }; ++ } T; + +@update@ +identifier capture.ST; +type concat.T; +identifier V; +expression E; +type WRAPPER_TYPE; +@@ + +WRAPPER_TYPE +{ + ... +( +- struct ST V[E]; ++ T V[E]; +| +- struct ST V; ++ T V; +) + ... +}; diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go b/internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go new file mode 100644 index 00000000..104b824b --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go @@ -0,0 +1,389 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "unicode" +) + +type GeneratableInterfacePoperties struct { + Type string + Interface string + Exclude []string + PackageMethodsAliasedFrom string +} + +var GeneratableInterfaces = []GeneratableInterfacePoperties{ + { + Type: "library", + Interface: "Interface", + PackageMethodsAliasedFrom: "libnvsandboxutils", + }, +} + +func main() { + sourceDir := flag.String("sourceDir", "", "Path to the source directory for all go files") + output := flag.String("output", "", "Path to the output file (default: stdout)") + flag.Parse() + + // Check if required flags are provided + if *sourceDir == "" { + flag.Usage() + return + } + + writer, closer, err := getWriter(*output) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + defer func() { + _ = closer() + }() + + header, err := generateHeader() + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, header) + + for i, p := range GeneratableInterfaces { + if p.PackageMethodsAliasedFrom != "" { + comment, err := generatePackageMethodsComment(p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, comment) + + output, err := generatePackageMethods(*sourceDir, p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprintf(writer, "%s\n", output) + } + + comment, err := generateInterfaceComment(p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, comment) + + output, err := generateInterface(*sourceDir, p) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + fmt.Fprint(writer, output) + + if i < (len(GeneratableInterfaces) - 1) { + fmt.Fprint(writer, "\n") + } + } +} + +func getWriter(outputFile string) (io.Writer, func() error, error) { + if outputFile == "" { + return os.Stdout, func() error { return nil }, nil + } + + file, err := os.Create(outputFile) + if err != nil { + return nil, nil, err + } + + return file, file.Close, nil +} + +func generateHeader() (string, error) { + lines := []string{ + "/**", + "# Copyright 2024 NVIDIA CORPORATION", + "#", + "# Licensed under the Apache License, Version 2.0 (the \"License\");", + "# you may not use this file except in compliance with the License.", + "# You may obtain a copy of the License at", + "#", + "# http://www.apache.org/licenses/LICENSE-2.0", + "#", + "# Unless required by applicable law or agreed to in writing, software", + "# distributed under the License is distributed on an \"AS IS\" BASIS,", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "# See the License for the specific language governing permissions and", + "# limitations under the License.", + "**/", + "", + "// Generated Code; DO NOT EDIT.", + "", + "package nvsandboxutils", + "", + "", + } + return strings.Join(lines, "\n"), nil +} + +func generatePackageMethodsComment(input GeneratableInterfacePoperties) (string, error) { + commentFmt := []string{ + "// The variables below represent package level methods from the %s type.", + } + + var signature strings.Builder + comment := strings.Join(commentFmt, "\n") + comment = fmt.Sprintf(comment, input.Type) + signature.WriteString(fmt.Sprintf("%s\n", comment)) + return signature.String(), nil +} + +func generateInterfaceComment(input GeneratableInterfacePoperties) (string, error) { + commentFmt := []string{ + "// %s represents the interface for the %s type.", + "//", + "//go:generate moq -out mock/%s.go -pkg mock . %s:%s", + } + + var signature strings.Builder + comment := strings.Join(commentFmt, "\n") + comment = fmt.Sprintf(comment, input.Interface, input.Type, strings.ToLower(input.Interface), input.Interface, input.Interface) + signature.WriteString(fmt.Sprintf("%s\n", comment)) + return signature.String(), nil +} + +func generatePackageMethods(sourceDir string, input GeneratableInterfacePoperties) (string, error) { + var signature strings.Builder + + signature.WriteString("var (\n") + + methods, err := extractMethodsFromPackage(sourceDir, input) + if err != nil { + return "", err + } + + for _, method := range methods { + name := method.Name.Name + formatted := fmt.Sprintf("\t%s = %s.%s\n", name, input.PackageMethodsAliasedFrom, name) + signature.WriteString(formatted) + } + + signature.WriteString(")\n") + + return signature.String(), nil +} + +func generateInterface(sourceDir string, input GeneratableInterfacePoperties) (string, error) { + var signature strings.Builder + + signature.WriteString(fmt.Sprintf("type %s interface {\n", input.Interface)) + + methods, err := extractMethodsFromPackage(sourceDir, input) + if err != nil { + return "", err + } + + for _, method := range methods { + formatted := fmt.Sprintf("\t%s\n", formatMethodSignature(method)) + signature.WriteString(formatted) + } + + signature.WriteString("}\n") + + return signature.String(), nil +} + +func getGoFiles(sourceDir string) (map[string][]byte, error) { + gofiles := make(map[string][]byte) + + err := filepath.WalkDir(sourceDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".go" { + return nil + } + + content, err := os.ReadFile(path) + if err != nil { + return err + } + + gofiles[path] = content + + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking %s: %w", sourceDir, err) + } + + return gofiles, nil +} + +func extractMethodsFromPackage(sourceDir string, input GeneratableInterfacePoperties) ([]*ast.FuncDecl, error) { + gofiles, err := getGoFiles(sourceDir) + if err != nil { + return nil, err + } + + var methods []*ast.FuncDecl + for file, content := range gofiles { + m, err := extractMethods(file, content, input) + if err != nil { + return nil, err + } + methods = append(methods, m...) + } + + sort.Slice(methods, func(i, j int) bool { + return methods[i].Name.Name < methods[j].Name.Name + }) + + return methods, nil +} + +func extractMethods(sourceFile string, sourceContent []byte, input GeneratableInterfacePoperties) ([]*ast.FuncDecl, error) { + // Parse source file + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, sourceFile, sourceContent, parser.ParseComments) + if err != nil { + return nil, err + } + + // Traverse AST to find type declarations and associated methods + var methods []*ast.FuncDecl + for _, decl := range node.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + + // Check if the function is a method associated with the specified type + if receiverType := funcDecl.Recv; receiverType != nil { + var ident *ast.Ident + + for _, field := range receiverType.List { + switch fieldType := field.Type.(type) { + case *ast.Ident: + ident = fieldType + case *ast.StarExpr: + // Update ident if it's a *ast.StarExpr + if newIdent, ok := fieldType.X.(*ast.Ident); ok { + // If the inner type is an *ast.Ident, update ident + ident = newIdent + } + } + + // No identifier found + if ident == nil { + continue + } + + // Identifier is not the one we are looking for + if ident.Name != input.Type { + continue + } + + // Ignore non-public methods + if !isPublic(funcDecl.Name.Name) { + continue + } + + // Ignore method in the exclude list + if slices.Contains(input.Exclude, funcDecl.Name.Name) { + continue + } + + methods = append(methods, funcDecl) + } + } + } + + return methods, nil +} + +func formatMethodSignature(decl *ast.FuncDecl) string { + var signature strings.Builder + + // Write method name + signature.WriteString(decl.Name.Name) + signature.WriteString("(") + + // Write parameters + if decl.Type.Params != nil { + for i, param := range decl.Type.Params.List { + if i > 0 { + signature.WriteString(", ") + } + signature.WriteString(formatFieldList(param)) + } + } + + signature.WriteString(")") + + // Write return types + if decl.Type.Results != nil { + signature.WriteString(" ") + if len(decl.Type.Results.List) > 1 { + signature.WriteString("(") + } + for i, result := range decl.Type.Results.List { + if i > 0 { + signature.WriteString(", ") + } + signature.WriteString(formatFieldList(result)) + } + if len(decl.Type.Results.List) > 1 { + signature.WriteString(")") + } + } + + return signature.String() +} + +func formatFieldList(field *ast.Field) string { + var builder strings.Builder + switch fieldType := field.Type.(type) { + case *ast.Ident: + builder.WriteString(fieldType.Name) + case *ast.ArrayType: + builder.WriteString("[]") + builder.WriteString(formatFieldList(&ast.Field{Type: fieldType.Elt})) + case *ast.StarExpr: + builder.WriteString("*") + builder.WriteString(formatFieldList(&ast.Field{Type: fieldType.X})) + } + return builder.String() +} + +func isPublic(name string) bool { + if len(name) == 0 { + return false + } + return unicode.IsUpper([]rune(name)[0]) +} diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h new file mode 100644 index 00000000..eb087156 --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVSANDBOXUTILS_H__ +#define __NVSANDBOXUTILS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define INPUT_LENGTH 256 +#define MAX_FILE_PATH 256 +#define MAX_NAME_LENGTH 256 + +/***************************************************************************************************/ +/** @defgroup enums Enumerations + * @{ + */ +/***************************************************************************************************/ + +/** + * Return types + */ +typedef enum +{ + NVSANDBOXUTILS_SUCCESS = 0, //!< The operation was successful + NVSANDBOXUTILS_ERROR_UNINITIALIZED = 1, //!< The library wasn't successfully initialized + NVSANDBOXUTILS_ERROR_NOT_SUPPORTED = 2, //!< The requested operation is not supported on target device + NVSANDBOXUTILS_ERROR_INVALID_ARG = 3, //!< A supplied argument is invalid + NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE = 4, //!< A supplied argument is not large enough + NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED = 5, //!< Requested library version is not supported + NVSANDBOXUTILS_ERROR_LIBRARY_LOAD = 6, //!< The library load failed + NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND = 7, //!< Called function was not found + NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND = 8, //!< Target device was not found + NVSANDBOXUTILS_ERROR_NVML_LIB_CALL = 9, //!< NVML library call failed + NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY = 10, //!< There is insufficient memory + NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND = 11, //!< A supplied file path was not found + NVSANDBOXUTILS_ERROR_UNKNOWN = 0xFFFF, //!< Unknown error occurred +} nvSandboxUtilsRet_t; + +/** + * Return if there is an error + */ +#define RETURN_ON_SANDBOX_ERROR(result) \ + if ((result) != NVSANDBOXUTILS_SUCCESS) { \ + NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \ + return result; \ + } + +/** + * Log levels + */ +typedef enum +{ + NVSANDBOXUTILS_LOG_LEVEL_FATAL = 0, //!< Log fatal errors + NVSANDBOXUTILS_LOG_LEVEL_ERROR = 1, //!< Log all errors + NVSANDBOXUTILS_LOG_LEVEL_WARN = 2, //!< Log all warnings + NVSANDBOXUTILS_LOG_LEVEL_DEBUG = 3, //!< Log all debug messages + NVSANDBOXUTILS_LOG_LEVEL_INFO = 4, //!< Log all info messages + NVSANDBOXUTILS_LOG_LEVEL_NONE = 0xFFFF, //!< Log none +} nvSandboxUtilsLogLevel_t; + +/** + * Input rootfs to help access files inside the driver container + */ +typedef enum +{ + NV_ROOTFS_DEFAULT, //!< Default no rootfs + NV_ROOTFS_PATH, //!< /run/nvidia/driver + NV_ROOTFS_PID, //!< /proc/PID/mountinfo +} nvSandboxUtilsRootfsInputType_t; + +/** + * File type + */ +typedef enum +{ + NV_DEV, //!< /dev file system + NV_PROC, //!< /proc file system + NV_SYS, //!< /sys file system +} nvSandboxUtilsFileType_t; + +/** + * File subtype + */ +typedef enum +{ + NV_DEV_NVIDIA, //!< /dev/nvidia0 + NV_DEV_DRI_CARD, //!< /dev/dri/card1 + NV_DEV_DRI_RENDERD, //!< /dev/dri/renderD128 + NV_DEV_DRI_CARD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-card + NV_DEV_DRI_RENDERD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-render + NV_DEV_NVIDIA_UVM, //!< /dev/nvidia-uvm + NV_DEV_NVIDIA_UVM_TOOLS, //!< /dev/nvidia-uvm-tools + NV_DEV_NVIDIA_MODESET, //!< /dev/nvidia-uvm-modeset + NV_DEV_NVIDIA_CTL, //!< /dev/nvidiactl + NV_DEV_GDRDRV, //!< /dev/gdrdrv + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP, //!< /dev/nvidia-caps/nvidia-cap22 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID, //!< /proc/driver/nvidia/gpus/0000:2d:00.0 + NV_PROC_DRIVER_NVIDIA_GPUS, //!< /proc/driver/nvidia/gpus (for mask out) + NV_PROC_NVIDIA_PARAMS, //!< /proc/driver/nvidia/params + NV_PROC_NVIDIA_CAPS_MIG_MINORS, //!< /proc/driver/nvidia-caps/mig-minors + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU, //!< /proc/driver/nvidia/capabilities/gpu0 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES, //!< /proc/driver/nvidia/capabilities (for mask out) + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS, //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID, //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0 + NV_SYS_MODULE_NVIDIA_DRIVER, //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out) + NV_NUM_SUBTYPE, // always at the end. +} nvSandboxUtilsFileSystemSubType_t; + +/** + * File module + */ +typedef enum +{ + NV_GPU, //!< Target device + NV_MIG, //!< Target device- MIG + NV_DRIVER_NVIDIA, //!< NVIDIA kernel driver + NV_DRIVER_NVIDIA_UVM, //!< NVIDIA kernel driver-UVM + NV_DRIVER_NVIDIA_MODESET, //!< NVIDIA kernel driver-modeset + NV_DRIVER_GDRDRV, //!< GDRDRV driver + NV_SYSTEM, //!< System module +} nvSandboxUtilsFileModule_t; + +/** + * Flag to provide additional details about the file + */ +typedef enum +{ + NV_FILE_FLAG_HINT = (1 << 0), //!< Default no hint + NV_FILE_FLAG_MASKOUT = (1 << 1), //!< For /proc/driver/nvidia/gpus + NV_FILE_FLAG_CONTENT = (1 << 2), //!< For /proc/driver/nvidia/params + //!< For SYMLINK + //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file + NV_FILE_FLAG_DEPRECTATED = (1 << 3), //!< For all the FIRMWARE GSP file + NV_FILE_FLAG_CANDIDATES = (1 << 4), //!< For libcuda.so +} nvSandboxUtilsFileFlag_t; + +/** + * Input type of the target device + */ +typedef enum +{ + NV_GPU_INPUT_GPU_UUID, //!< GPU UUID + NV_GPU_INPUT_MIG_UUID, //!< MIG UUID + NV_GPU_INPUT_PCI_ID, //!< PCIe DBDF ID + NV_GPU_INPUT_PCI_INDEX, //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF) +} nvSandboxUtilsGpuInputType_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup dataTypes Structures and Unions + * @{ + */ +/***************************************************************************************************/ + +/** + * Initalization input v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsRootfsInputType_t type; //!< One of \p nvSandboxUtilsRootfsInputType_t + char value[INPUT_LENGTH]; //!< String representation of input +} nvSandboxUtilsInitInput_v1_t; + +typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t; + +/** + * File system information + */ +typedef struct nvSandboxUtilsGpuFileInfo_v1_t +{ + struct nvSandboxUtilsGpuFileInfo_v1_t *next; //!< Pointer to the next node in the linked list + nvSandboxUtilsFileType_t fileType; //!< One of \p nvSandboxUtilsFileType_t + nvSandboxUtilsFileSystemSubType_t fileSubType; //!< One of \p nvSandboxUtilsFileSystemSubType_t + nvSandboxUtilsFileModule_t module; //!< One of \p nvSandboxUtilsFileModule_t + nvSandboxUtilsFileFlag_t flags; //!< One of \p nvSandboxUtilsFileFlag_t + char *filePath; //!< Relative file path to rootfs +}nvSandboxUtilsGpuFileInfo_v1_t; + +/** + * GPU resource request v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsGpuInputType_t inputType; //!< One of \p nvSandboxUtilsGpuInputType_t + char input[INPUT_LENGTH]; //!< String representation of input + nvSandboxUtilsGpuFileInfo_v1_t *files; //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t +} nvSandboxUtilsGpuRes_v1_t; + +typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup funcs Functions + * @{ + */ +/***************************************************************************************************/ + +/* ************************************************* + * Initialize library + * ************************************************* + */ +/** + * Prepare library resources before library API can be used. + * This initialization will not fail if one of the initialization prerequisites fails. + * @param input Reference to the called-supplied input struct that has initialization fields + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p input->value isn't a valid rootfs path + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p input->version isn't supported by the library + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND if any of the required file paths are not found during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY if there is insufficient system memory during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD on any error during loading the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input); + +/* ************************************************* + * Shutdown library + * ************************************************* + */ +/** + * Clean up library resources created by init call + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + */ +nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void); + +/* ************************************************* + * Get NVIDIA RM driver version + * ************************************************* + */ +/** + * Get NVIDIA RM driver version + * @param version Reference to caller-supplied buffer to return driver version string + * @param length The maximum allowed length of the string returned in \p version + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p version is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL on any error during driver version query from NVML + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length); + +/* ************************************************* + * Get /dev, /proc, /sys file system information + * ************************************************* + */ +/** + * Get /dev, /proc, /sys file system information + * @param request Reference to caller-supplied request struct to return the file system information + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p request->input doesn't match any device + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p request->version isn't supported by the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request); + +/* ************************************************* + * Get content of given file path + * ************************************************* + */ +/** + * Get file content of input file path + * @param filePath Reference to the file path + * @param content Reference to the caller-supplied buffer to return the file content + * @param contentSize Reference to the maximum allowed size of content. It is updated to the actual size of the content on return + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p filePath or \p content is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE if \p contentSize is too small + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND on an error while obtaining the content for the file path + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize); + +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif // __NVSANDBOXUTILS_H__ diff --git a/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml new file mode 100644 index 00000000..851db743 --- /dev/null +++ b/internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml @@ -0,0 +1,66 @@ +# Copyright (c) 2024, NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +GENERATOR: + PackageName: nvsandboxutils + PackageDescription: "Package NVSANDBOXUTILS bindings" + PackageLicense: |- + Copyright (c) 2024, NVIDIA CORPORATION + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Includes: ["nvsandboxutils.h"] + FlagGroups: + - {name: "LDFLAGS", traits: ["linux"], flags: ["-Wl,--export-dynamic","-Wl,--unresolved-symbols=ignore-in-object-files"]} + - {name: "LDFLAGS", traits: ["darwin"], flags: ["-Wl,-undefined,dynamic_lookup"]} +PARSER: + SourcesPaths: ["nvsandboxutils.h"] +TRANSLATOR: + ConstRules: + defines: eval + enum: eval + PtrTips: + function: + - {target: "^nvSandboxUtils", default: "sref"} + MemTips: + - {target: "^nvSandboxUtils", default: "raw"} + Rules: + const: + - {action: accept, from: "^NVSANDBOXUTILS_"} + - {action: accept, from: "^nvSandboxUtils"} + - {action: replace, from: "^NVSANDBOXUTILS_"} + - {action: replace, from: "^nvSandboxUtils"} + - {action: accept, from: "^NV"} + - {action: accept, from: "^MAX"} + - {action: accept, from: "^INPUT"} + - {action: replace, from: "_t$"} + - {transform: export} + type: + - {action: accept, from: "^nvSandboxUtils"} + - {action: replace, from: "^nvSandboxUtils"} + - {action: replace, from: "_t$"} + - {transform: export} + function: + - {action: accept, from: "^nvSandboxUtils"} + - {transform: unexport} diff --git a/internal/nvsandboxutils/gen/update-bindings.sh b/internal/nvsandboxutils/gen/update-bindings.sh new file mode 100755 index 00000000..5ac472ea --- /dev/null +++ b/internal/nvsandboxutils/gen/update-bindings.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file allows for the nvsandboxutils bindings to be updated using the tooling +# implemented in https://github.com/NVIDIA/go-nvml. +# To run this: +# cd internal/nvsandboxutils +# ./update-bindings.sh + +set -e + +BUILDIMAGE=bindings + +docker build \ + --build-arg GOLANG_VERSION=1.22.1 \ + --build-arg C_FOR_GO_TAG=8eeee8c3b71f9c3c90c4a73db54ed08b0bba971d \ + -t ${BUILDIMAGE} \ + -f docker/Dockerfile.devel \ + https://github.com/NVIDIA/go-nvml.git + + +docker run --rm -ti \ + -e GOCACHE=/tmp/.cache/go \ + -e GOMODCACHE=/tmp/.cache/gomod \ + -v $(pwd):/nvsandboxutils \ + -w /nvsandboxutils \ + -u $(id -u):$(id -g) \ + ${BUILDIMAGE} \ + ./gen/generate-bindings.sh From 978d439cf8fa67d8a0cb78bb8ac3665027596fc8 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:05:05 -0700 Subject: [PATCH 2/5] nvsandboxutils: Add internal bindings This change adds the internal bindings for Sandboxutils, some of which have been automatically generated with the help of c-for-go. The format followed is similar to what is used in go-nvml. These would need to be regenerated when the header file is modified and new APIs are added. Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- internal/nvsandboxutils/api.go | 45 +++ internal/nvsandboxutils/cgo_helpers.h | 25 ++ internal/nvsandboxutils/const.go | 156 +++++++++ internal/nvsandboxutils/doc.go | 23 ++ .../nvsandboxutils/dynamicLibrary_mock.go | 157 +++++++++ internal/nvsandboxutils/lib.go | 156 +++++++++ internal/nvsandboxutils/lib_test.go | 245 +++++++++++++ internal/nvsandboxutils/mock/interface.go | 325 ++++++++++++++++++ internal/nvsandboxutils/nvsandboxutils.go | 72 ++++ internal/nvsandboxutils/nvsandboxutils.h | 298 ++++++++++++++++ internal/nvsandboxutils/refcount.go | 31 ++ internal/nvsandboxutils/refcount_test.go | 139 ++++++++ internal/nvsandboxutils/return.go | 74 ++++ internal/nvsandboxutils/types_gen.go | 39 +++ internal/nvsandboxutils/zz_generated.api.go | 43 +++ 15 files changed, 1828 insertions(+) create mode 100644 internal/nvsandboxutils/api.go create mode 100644 internal/nvsandboxutils/cgo_helpers.h create mode 100644 internal/nvsandboxutils/const.go create mode 100644 internal/nvsandboxutils/doc.go create mode 100644 internal/nvsandboxutils/dynamicLibrary_mock.go create mode 100644 internal/nvsandboxutils/lib.go create mode 100644 internal/nvsandboxutils/lib_test.go create mode 100644 internal/nvsandboxutils/mock/interface.go create mode 100644 internal/nvsandboxutils/nvsandboxutils.go create mode 100644 internal/nvsandboxutils/nvsandboxutils.h create mode 100644 internal/nvsandboxutils/refcount.go create mode 100644 internal/nvsandboxutils/refcount_test.go create mode 100644 internal/nvsandboxutils/return.go create mode 100644 internal/nvsandboxutils/types_gen.go create mode 100644 internal/nvsandboxutils/zz_generated.api.go diff --git a/internal/nvsandboxutils/api.go b/internal/nvsandboxutils/api.go new file mode 100644 index 00000000..6275a5c2 --- /dev/null +++ b/internal/nvsandboxutils/api.go @@ -0,0 +1,45 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +// libraryOptions hold the parameters than can be set by a LibraryOption +type libraryOptions struct { + path string + flags int +} + +// LibraryOption represents a functional option to configure the underlying nvsandboxutils library +type LibraryOption func(*libraryOptions) + +// WithLibraryPath provides an option to set the library name to be used by the nvsandboxutils library. +func WithLibraryPath(path string) LibraryOption { + return func(o *libraryOptions) { + o.path = path + } +} + +// SetLibraryOptions applies the specified options to the nvsandboxutils library. +// If this is called when a library is already loaded, an error is raised. +func SetLibraryOptions(opts ...LibraryOption) error { + libnvsandboxutils.Lock() + defer libnvsandboxutils.Unlock() + if libnvsandboxutils.refcount != 0 { + return errLibraryAlreadyLoaded + } + libnvsandboxutils.init(opts...) + return nil +} diff --git a/internal/nvsandboxutils/cgo_helpers.h b/internal/nvsandboxutils/cgo_helpers.h new file mode 100644 index 00000000..23b3c256 --- /dev/null +++ b/internal/nvsandboxutils/cgo_helpers.h @@ -0,0 +1,25 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +#include "nvsandboxutils.h" +#include +#pragma once + +#define __CGOGEN 1 + diff --git a/internal/nvsandboxutils/const.go b/internal/nvsandboxutils/const.go new file mode 100644 index 00000000..9e8cdf3f --- /dev/null +++ b/internal/nvsandboxutils/const.go @@ -0,0 +1,156 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package nvsandboxutils + +/* +#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup +#include "nvsandboxutils.h" +#include +#include "cgo_helpers.h" +*/ +import "C" + +const ( + // INPUT_LENGTH as defined in nvsandboxutils/nvsandboxutils.h + INPUT_LENGTH = 256 + // MAX_FILE_PATH as defined in nvsandboxutils/nvsandboxutils.h + MAX_FILE_PATH = 256 + // MAX_NAME_LENGTH as defined in nvsandboxutils/nvsandboxutils.h + MAX_NAME_LENGTH = 256 +) + +// Ret as declared in nvsandboxutils/nvsandboxutils.h +type Ret int32 + +// Ret enumeration from nvsandboxutils/nvsandboxutils.h +const ( + SUCCESS Ret = iota + ERROR_UNINITIALIZED Ret = 1 + ERROR_NOT_SUPPORTED Ret = 2 + ERROR_INVALID_ARG Ret = 3 + ERROR_INSUFFICIENT_SIZE Ret = 4 + ERROR_VERSION_NOT_SUPPORTED Ret = 5 + ERROR_LIBRARY_LOAD Ret = 6 + ERROR_FUNCTION_NOT_FOUND Ret = 7 + ERROR_DEVICE_NOT_FOUND Ret = 8 + ERROR_NVML_LIB_CALL Ret = 9 + ERROR_OUT_OF_MEMORY Ret = 10 + ERROR_FILEPATH_NOT_FOUND Ret = 11 + ERROR_UNKNOWN Ret = 65535 +) + +// LogLevel as declared in nvsandboxutils/nvsandboxutils.h +type LogLevel int32 + +// LogLevel enumeration from nvsandboxutils/nvsandboxutils.h +const ( + LOG_LEVEL_FATAL LogLevel = iota + LOG_LEVEL_ERROR LogLevel = 1 + LOG_LEVEL_WARN LogLevel = 2 + LOG_LEVEL_DEBUG LogLevel = 3 + LOG_LEVEL_INFO LogLevel = 4 + LOG_LEVEL_NONE LogLevel = 65535 +) + +// RootfsInputType as declared in nvsandboxutils/nvsandboxutils.h +type RootfsInputType int32 + +// RootfsInputType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_ROOTFS_DEFAULT RootfsInputType = iota + NV_ROOTFS_PATH RootfsInputType = 1 + NV_ROOTFS_PID RootfsInputType = 2 +) + +// FileType as declared in nvsandboxutils/nvsandboxutils.h +type FileType int32 + +// FileType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_DEV FileType = iota + NV_PROC FileType = 1 + NV_SYS FileType = 2 +) + +// FileSystemSubType as declared in nvsandboxutils/nvsandboxutils.h +type FileSystemSubType int32 + +// FileSystemSubType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_DEV_NVIDIA FileSystemSubType = iota + NV_DEV_DRI_CARD FileSystemSubType = 1 + NV_DEV_DRI_RENDERD FileSystemSubType = 2 + NV_DEV_DRI_CARD_SYMLINK FileSystemSubType = 3 + NV_DEV_DRI_RENDERD_SYMLINK FileSystemSubType = 4 + NV_DEV_NVIDIA_UVM FileSystemSubType = 5 + NV_DEV_NVIDIA_UVM_TOOLS FileSystemSubType = 6 + NV_DEV_NVIDIA_MODESET FileSystemSubType = 7 + NV_DEV_NVIDIA_CTL FileSystemSubType = 8 + NV_DEV_GDRDRV FileSystemSubType = 9 + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP FileSystemSubType = 10 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID FileSystemSubType = 11 + NV_PROC_DRIVER_NVIDIA_GPUS FileSystemSubType = 12 + NV_PROC_NVIDIA_PARAMS FileSystemSubType = 13 + NV_PROC_NVIDIA_CAPS_MIG_MINORS FileSystemSubType = 14 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU FileSystemSubType = 15 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES FileSystemSubType = 16 + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS FileSystemSubType = 17 + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID FileSystemSubType = 18 + NV_SYS_MODULE_NVIDIA_DRIVER FileSystemSubType = 19 + NV_NUM_SUBTYPE FileSystemSubType = 20 +) + +// FileModule as declared in nvsandboxutils/nvsandboxutils.h +type FileModule int32 + +// FileModule enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_GPU FileModule = iota + NV_MIG FileModule = 1 + NV_DRIVER_NVIDIA FileModule = 2 + NV_DRIVER_NVIDIA_UVM FileModule = 3 + NV_DRIVER_NVIDIA_MODESET FileModule = 4 + NV_DRIVER_GDRDRV FileModule = 5 + NV_SYSTEM FileModule = 6 +) + +// FileFlag as declared in nvsandboxutils/nvsandboxutils.h +type FileFlag int32 + +// FileFlag enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_FILE_FLAG_HINT FileFlag = 1 + NV_FILE_FLAG_MASKOUT FileFlag = 2 + NV_FILE_FLAG_CONTENT FileFlag = 4 + NV_FILE_FLAG_DEPRECTATED FileFlag = 8 + NV_FILE_FLAG_CANDIDATES FileFlag = 16 +) + +// GpuInputType as declared in nvsandboxutils/nvsandboxutils.h +type GpuInputType int32 + +// GpuInputType enumeration from nvsandboxutils/nvsandboxutils.h +const ( + NV_GPU_INPUT_GPU_UUID GpuInputType = iota + NV_GPU_INPUT_MIG_UUID GpuInputType = 1 + NV_GPU_INPUT_PCI_ID GpuInputType = 2 + NV_GPU_INPUT_PCI_INDEX GpuInputType = 3 +) diff --git a/internal/nvsandboxutils/doc.go b/internal/nvsandboxutils/doc.go new file mode 100644 index 00000000..231c68c2 --- /dev/null +++ b/internal/nvsandboxutils/doc.go @@ -0,0 +1,23 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +/* +Package NVSANDBOXUTILS bindings +*/ +package nvsandboxutils diff --git a/internal/nvsandboxutils/dynamicLibrary_mock.go b/internal/nvsandboxutils/dynamicLibrary_mock.go new file mode 100644 index 00000000..a22e5669 --- /dev/null +++ b/internal/nvsandboxutils/dynamicLibrary_mock.go @@ -0,0 +1,157 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package nvsandboxutils + +import ( + "sync" +) + +// Ensure, that dynamicLibraryMock does implement dynamicLibrary. +// If this is not the case, regenerate this file with moq. +var _ dynamicLibrary = &dynamicLibraryMock{} + +// dynamicLibraryMock is a mock implementation of dynamicLibrary. +// +// func TestSomethingThatUsesdynamicLibrary(t *testing.T) { +// +// // make and configure a mocked dynamicLibrary +// mockeddynamicLibrary := &dynamicLibraryMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// LookupFunc: func(s string) error { +// panic("mock out the Lookup method") +// }, +// OpenFunc: func() error { +// panic("mock out the Open method") +// }, +// } +// +// // use mockeddynamicLibrary in code that requires dynamicLibrary +// // and then make assertions. +// +// } +type dynamicLibraryMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // LookupFunc mocks the Lookup method. + LookupFunc func(s string) error + + // OpenFunc mocks the Open method. + OpenFunc func() error + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // Lookup holds details about calls to the Lookup method. + Lookup []struct { + // S is the s argument value. + S string + } + // Open holds details about calls to the Open method. + Open []struct { + } + } + lockClose sync.RWMutex + lockLookup sync.RWMutex + lockOpen sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *dynamicLibraryMock) Close() error { + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + if mock.CloseFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockeddynamicLibrary.CloseCalls()) +func (mock *dynamicLibraryMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// Lookup calls LookupFunc. +func (mock *dynamicLibraryMock) Lookup(s string) error { + callInfo := struct { + S string + }{ + S: s, + } + mock.lockLookup.Lock() + mock.calls.Lookup = append(mock.calls.Lookup, callInfo) + mock.lockLookup.Unlock() + if mock.LookupFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.LookupFunc(s) +} + +// LookupCalls gets all the calls that were made to Lookup. +// Check the length with: +// +// len(mockeddynamicLibrary.LookupCalls()) +func (mock *dynamicLibraryMock) LookupCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockLookup.RLock() + calls = mock.calls.Lookup + mock.lockLookup.RUnlock() + return calls +} + +// Open calls OpenFunc. +func (mock *dynamicLibraryMock) Open() error { + callInfo := struct { + }{} + mock.lockOpen.Lock() + mock.calls.Open = append(mock.calls.Open, callInfo) + mock.lockOpen.Unlock() + if mock.OpenFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.OpenFunc() +} + +// OpenCalls gets all the calls that were made to Open. +// Check the length with: +// +// len(mockeddynamicLibrary.OpenCalls()) +func (mock *dynamicLibraryMock) OpenCalls() []struct { +} { + var calls []struct { + } + mock.lockOpen.RLock() + calls = mock.calls.Open + mock.lockOpen.RUnlock() + return calls +} diff --git a/internal/nvsandboxutils/lib.go b/internal/nvsandboxutils/lib.go new file mode 100644 index 00000000..3a85ef89 --- /dev/null +++ b/internal/nvsandboxutils/lib.go @@ -0,0 +1,156 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "fmt" + "sync" + + "github.com/NVIDIA/go-nvml/pkg/dl" +) + +const ( + defaultNvSandboxUtilsLibraryName = "libnvidia-sandboxutils.so.1" + defaultNvSandboxUtilsLibraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL +) + +var errLibraryNotLoaded = errors.New("library not loaded") +var errLibraryAlreadyLoaded = errors.New("library already loaded") + +// dynamicLibrary is an interface for abstacting the underlying library. +// This also allows for mocking and testing. + +//go:generate moq -rm -stub -out dynamicLibrary_mock.go . dynamicLibrary +type dynamicLibrary interface { + Lookup(string) error + Open() error + Close() error +} + +// library represents an nvsandboxutils library. +// This includes a reference to the underlying DynamicLibrary +type library struct { + sync.Mutex + path string + refcount refcount + dl dynamicLibrary +} + +// libnvsandboxutils is a global instance of the nvsandboxutils library. +var libnvsandboxutils = newLibrary() + +func New(opts ...LibraryOption) Interface { + return newLibrary(opts...) +} + +func newLibrary(opts ...LibraryOption) *library { + l := &library{} + l.init(opts...) + return l +} + +func (l *library) init(opts ...LibraryOption) { + o := libraryOptions{} + for _, opt := range opts { + opt(&o) + } + + if o.path == "" { + o.path = defaultNvSandboxUtilsLibraryName + } + if o.flags == 0 { + o.flags = defaultNvSandboxUtilsLibraryLoadFlags + } + + l.path = o.path + l.dl = dl.New(o.path, o.flags) +} + +// LookupSymbol checks whether the specified library symbol exists in the library. +// Note that this requires that the library be loaded. +func (l *library) LookupSymbol(name string) error { + if l == nil || l.refcount == 0 { + return fmt.Errorf("error looking up %s: %w", name, errLibraryNotLoaded) + } + return l.dl.Lookup(name) +} + +// load initializes the library and updates the versioned symbols. +// Multiple calls to an already loaded library will return without error. +func (l *library) load() (rerr error) { + l.Lock() + defer l.Unlock() + + defer func() { l.refcount.IncOnNoError(rerr) }() + if l.refcount > 0 { + return nil + } + + if err := l.dl.Open(); err != nil { + return fmt.Errorf("error opening %s: %w", l.path, err) + } + + // Update the errorStringFunc to point to nvsandboxutils.ErrorString + errorStringFunc = nvsanboxutilsErrorString + + // Update all versioned symbols + l.updateVersionedSymbols() + + return nil +} + +// close the underlying library and ensure that the global pointer to the +// library is set to nil to ensure that subsequent calls to open will reinitialize it. +// Multiple calls to an already closed nvsandboxutils library will return without error. +func (l *library) close() (rerr error) { + l.Lock() + defer l.Unlock() + + defer func() { l.refcount.DecOnNoError(rerr) }() + if l.refcount != 1 { + return nil + } + + if err := l.dl.Close(); err != nil { + return fmt.Errorf("error closing %s: %w", l.path, err) + } + + // Update the errorStringFunc to point to defaultErrorStringFunc + errorStringFunc = defaultErrorStringFunc + + return nil +} + +// Default all versioned APIs to v1 (to infer the types) +var ( +// Insert default versions for APIs here. +// Example: +// nvsandboxUtilsFunction = nvsandboxUtilsFunction_v1 +) + +// updateVersionedSymbols checks for versioned symbols in the loaded dynamic library. +// If newer versioned symbols exist, these replace the default `v1` symbols initialized above. +// When new versioned symbols are added, these would have to be initialized above and have +// corresponding checks and subsequent assignments added below. +func (l *library) updateVersionedSymbols() { + // Example: + // err := l.dl.Lookup("nvsandboxUtilsFunction_v2") + // if err == nil { + // nvsandboxUtilsFunction = nvsandboxUtilsFunction_v2 + // } +} diff --git a/internal/nvsandboxutils/lib_test.go b/internal/nvsandboxutils/lib_test.go new file mode 100644 index 00000000..b87c7242 --- /dev/null +++ b/internal/nvsandboxutils/lib_test.go @@ -0,0 +1,245 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestLibrary(dl dynamicLibrary) *library { + return &library{dl: dl} +} + +func TestLookupFromDefault(t *testing.T) { + errClose := errors.New("close error") + errOpen := errors.New("open error") + errLookup := errors.New("lookup error") + + testCases := []struct { + description string + dl dynamicLibrary + skipLoadLibrary bool + expectedLoadError error + expectedLookupErrror error + expectedCloseError error + }{ + { + description: "library not loaded yields error", + dl: &dynamicLibraryMock{}, + skipLoadLibrary: true, + expectedLookupErrror: errLibraryNotLoaded, + }, + { + description: "open error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errOpen + }, + }, + + expectedLoadError: errOpen, + expectedLookupErrror: errLibraryNotLoaded, + }, + { + description: "lookup error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return fmt.Errorf("%w: %s", errLookup, s) + }, + CloseFunc: func() error { + return nil + }, + }, + + expectedLookupErrror: errLookup, + }, + { + description: "lookup succeeds", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + }, + { + description: "lookup succeeds", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + }, + { + description: "close error is returned", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + LookupFunc: func(s string) error { + return nil + }, + CloseFunc: func() error { + return errClose + }, + }, + expectedCloseError: errClose, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l := newTestLibrary(tc.dl) + if !tc.skipLoadLibrary { + require.ErrorIs(t, l.load(), tc.expectedLoadError) + } + require.ErrorIs(t, l.LookupSymbol("symbol"), tc.expectedLookupErrror) + require.ErrorIs(t, l.close(), tc.expectedCloseError) + if tc.expectedCloseError == nil { + require.Equal(t, 0, int(l.refcount)) + } else { + require.Equal(t, 1, int(l.refcount)) + } + }) + } +} + +func TestLoadAndCloseNesting(t *testing.T) { + dl := &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return nil + }, + } + + l := newTestLibrary(dl) + + // When calling close before opening the library nothing happens. + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 0, len(dl.calls.Close)) + + // When calling load twice, the library was only opened once + require.Equal(t, 0, len(dl.calls.Open)) + require.Nil(t, l.load()) + require.Equal(t, 1, len(dl.calls.Open)) + require.Nil(t, l.load()) + require.Equal(t, 1, len(dl.calls.Open)) + + // Only after calling close twice, was the library closed + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 0, len(dl.calls.Close)) + require.Nil(t, l.close()) + require.Equal(t, 1, len(dl.calls.Close)) + + // Calling close again doesn't attempt to close the library again + require.Nil(t, l.close()) + require.Equal(t, 1, len(dl.calls.Close)) +} + +func TestLoadAndCloseWithErrors(t *testing.T) { + testCases := []struct { + description string + dl dynamicLibrary + expectedLoadRefcount refcount + expectedCloseRefcount refcount + }{ + { + description: "regular flow", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return nil + }, + }, + expectedLoadRefcount: 1, + expectedCloseRefcount: 0, + }, + { + description: "open error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errors.New("") + }, + CloseFunc: func() error { + return nil + }, + }, + expectedLoadRefcount: 0, + expectedCloseRefcount: 0, + }, + { + description: "close error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return nil + }, + CloseFunc: func() error { + return errors.New("") + }, + }, + expectedLoadRefcount: 1, + expectedCloseRefcount: 1, + }, + { + description: "open and close error", + dl: &dynamicLibraryMock{ + OpenFunc: func() error { + return errors.New("") + }, + CloseFunc: func() error { + return errors.New("") + }, + }, + expectedLoadRefcount: 0, + expectedCloseRefcount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l := newTestLibrary(tc.dl) + _ = l.load() + require.Equal(t, tc.expectedLoadRefcount, l.refcount) + _ = l.close() + require.Equal(t, tc.expectedCloseRefcount, l.refcount) + }) + } +} diff --git a/internal/nvsandboxutils/mock/interface.go b/internal/nvsandboxutils/mock/interface.go new file mode 100644 index 00000000..d7b27ebf --- /dev/null +++ b/internal/nvsandboxutils/mock/interface.go @@ -0,0 +1,325 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" + "sync" +) + +// Ensure, that Interface does implement nvsandboxutils.Interface. +// If this is not the case, regenerate this file with moq. +var _ nvsandboxutils.Interface = &Interface{} + +// Interface is a mock implementation of nvsandboxutils.Interface. +// +// func TestSomethingThatUsesInterface(t *testing.T) { +// +// // make and configure a mocked nvsandboxutils.Interface +// mockedInterface := &Interface{ +// ErrorStringFunc: func(ret nvsandboxutils.Ret) string { +// panic("mock out the ErrorString method") +// }, +// GetDriverVersionFunc: func() (string, nvsandboxutils.Ret) { +// panic("mock out the GetDriverVersion method") +// }, +// GetFileContentFunc: func(s string) (string, nvsandboxutils.Ret) { +// panic("mock out the GetFileContent method") +// }, +// GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { +// panic("mock out the GetGpuResource method") +// }, +// InitFunc: func(s string) nvsandboxutils.Ret { +// panic("mock out the Init method") +// }, +// LookupSymbolFunc: func(s string) error { +// panic("mock out the LookupSymbol method") +// }, +// ShutdownFunc: func() nvsandboxutils.Ret { +// panic("mock out the Shutdown method") +// }, +// } +// +// // use mockedInterface in code that requires nvsandboxutils.Interface +// // and then make assertions. +// +// } +type Interface struct { + // ErrorStringFunc mocks the ErrorString method. + ErrorStringFunc func(ret nvsandboxutils.Ret) string + + // GetDriverVersionFunc mocks the GetDriverVersion method. + GetDriverVersionFunc func() (string, nvsandboxutils.Ret) + + // GetFileContentFunc mocks the GetFileContent method. + GetFileContentFunc func(s string) (string, nvsandboxutils.Ret) + + // GetGpuResourceFunc mocks the GetGpuResource method. + GetGpuResourceFunc func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) + + // InitFunc mocks the Init method. + InitFunc func(s string) nvsandboxutils.Ret + + // LookupSymbolFunc mocks the LookupSymbol method. + LookupSymbolFunc func(s string) error + + // ShutdownFunc mocks the Shutdown method. + ShutdownFunc func() nvsandboxutils.Ret + + // calls tracks calls to the methods. + calls struct { + // ErrorString holds details about calls to the ErrorString method. + ErrorString []struct { + // Ret is the ret argument value. + Ret nvsandboxutils.Ret + } + // GetDriverVersion holds details about calls to the GetDriverVersion method. + GetDriverVersion []struct { + } + // GetFileContent holds details about calls to the GetFileContent method. + GetFileContent []struct { + // S is the s argument value. + S string + } + // GetGpuResource holds details about calls to the GetGpuResource method. + GetGpuResource []struct { + // S is the s argument value. + S string + } + // Init holds details about calls to the Init method. + Init []struct { + // S is the s argument value. + S string + } + // LookupSymbol holds details about calls to the LookupSymbol method. + LookupSymbol []struct { + // S is the s argument value. + S string + } + // Shutdown holds details about calls to the Shutdown method. + Shutdown []struct { + } + } + lockErrorString sync.RWMutex + lockGetDriverVersion sync.RWMutex + lockGetFileContent sync.RWMutex + lockGetGpuResource sync.RWMutex + lockInit sync.RWMutex + lockLookupSymbol sync.RWMutex + lockShutdown sync.RWMutex +} + +// ErrorString calls ErrorStringFunc. +func (mock *Interface) ErrorString(ret nvsandboxutils.Ret) string { + if mock.ErrorStringFunc == nil { + panic("Interface.ErrorStringFunc: method is nil but Interface.ErrorString was just called") + } + callInfo := struct { + Ret nvsandboxutils.Ret + }{ + Ret: ret, + } + mock.lockErrorString.Lock() + mock.calls.ErrorString = append(mock.calls.ErrorString, callInfo) + mock.lockErrorString.Unlock() + return mock.ErrorStringFunc(ret) +} + +// ErrorStringCalls gets all the calls that were made to ErrorString. +// Check the length with: +// +// len(mockedInterface.ErrorStringCalls()) +func (mock *Interface) ErrorStringCalls() []struct { + Ret nvsandboxutils.Ret +} { + var calls []struct { + Ret nvsandboxutils.Ret + } + mock.lockErrorString.RLock() + calls = mock.calls.ErrorString + mock.lockErrorString.RUnlock() + return calls +} + +// GetDriverVersion calls GetDriverVersionFunc. +func (mock *Interface) GetDriverVersion() (string, nvsandboxutils.Ret) { + if mock.GetDriverVersionFunc == nil { + panic("Interface.GetDriverVersionFunc: method is nil but Interface.GetDriverVersion was just called") + } + callInfo := struct { + }{} + mock.lockGetDriverVersion.Lock() + mock.calls.GetDriverVersion = append(mock.calls.GetDriverVersion, callInfo) + mock.lockGetDriverVersion.Unlock() + return mock.GetDriverVersionFunc() +} + +// GetDriverVersionCalls gets all the calls that were made to GetDriverVersion. +// Check the length with: +// +// len(mockedInterface.GetDriverVersionCalls()) +func (mock *Interface) GetDriverVersionCalls() []struct { +} { + var calls []struct { + } + mock.lockGetDriverVersion.RLock() + calls = mock.calls.GetDriverVersion + mock.lockGetDriverVersion.RUnlock() + return calls +} + +// GetFileContent calls GetFileContentFunc. +func (mock *Interface) GetFileContent(s string) (string, nvsandboxutils.Ret) { + if mock.GetFileContentFunc == nil { + panic("Interface.GetFileContentFunc: method is nil but Interface.GetFileContent was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockGetFileContent.Lock() + mock.calls.GetFileContent = append(mock.calls.GetFileContent, callInfo) + mock.lockGetFileContent.Unlock() + return mock.GetFileContentFunc(s) +} + +// GetFileContentCalls gets all the calls that were made to GetFileContent. +// Check the length with: +// +// len(mockedInterface.GetFileContentCalls()) +func (mock *Interface) GetFileContentCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockGetFileContent.RLock() + calls = mock.calls.GetFileContent + mock.lockGetFileContent.RUnlock() + return calls +} + +// GetGpuResource calls GetGpuResourceFunc. +func (mock *Interface) GetGpuResource(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + if mock.GetGpuResourceFunc == nil { + panic("Interface.GetGpuResourceFunc: method is nil but Interface.GetGpuResource was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockGetGpuResource.Lock() + mock.calls.GetGpuResource = append(mock.calls.GetGpuResource, callInfo) + mock.lockGetGpuResource.Unlock() + return mock.GetGpuResourceFunc(s) +} + +// GetGpuResourceCalls gets all the calls that were made to GetGpuResource. +// Check the length with: +// +// len(mockedInterface.GetGpuResourceCalls()) +func (mock *Interface) GetGpuResourceCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockGetGpuResource.RLock() + calls = mock.calls.GetGpuResource + mock.lockGetGpuResource.RUnlock() + return calls +} + +// Init calls InitFunc. +func (mock *Interface) Init(s string) nvsandboxutils.Ret { + if mock.InitFunc == nil { + panic("Interface.InitFunc: method is nil but Interface.Init was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockInit.Lock() + mock.calls.Init = append(mock.calls.Init, callInfo) + mock.lockInit.Unlock() + return mock.InitFunc(s) +} + +// InitCalls gets all the calls that were made to Init. +// Check the length with: +// +// len(mockedInterface.InitCalls()) +func (mock *Interface) InitCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockInit.RLock() + calls = mock.calls.Init + mock.lockInit.RUnlock() + return calls +} + +// LookupSymbol calls LookupSymbolFunc. +func (mock *Interface) LookupSymbol(s string) error { + if mock.LookupSymbolFunc == nil { + panic("Interface.LookupSymbolFunc: method is nil but Interface.LookupSymbol was just called") + } + callInfo := struct { + S string + }{ + S: s, + } + mock.lockLookupSymbol.Lock() + mock.calls.LookupSymbol = append(mock.calls.LookupSymbol, callInfo) + mock.lockLookupSymbol.Unlock() + return mock.LookupSymbolFunc(s) +} + +// LookupSymbolCalls gets all the calls that were made to LookupSymbol. +// Check the length with: +// +// len(mockedInterface.LookupSymbolCalls()) +func (mock *Interface) LookupSymbolCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockLookupSymbol.RLock() + calls = mock.calls.LookupSymbol + mock.lockLookupSymbol.RUnlock() + return calls +} + +// Shutdown calls ShutdownFunc. +func (mock *Interface) Shutdown() nvsandboxutils.Ret { + if mock.ShutdownFunc == nil { + panic("Interface.ShutdownFunc: method is nil but Interface.Shutdown was just called") + } + callInfo := struct { + }{} + mock.lockShutdown.Lock() + mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo) + mock.lockShutdown.Unlock() + return mock.ShutdownFunc() +} + +// ShutdownCalls gets all the calls that were made to Shutdown. +// Check the length with: +// +// len(mockedInterface.ShutdownCalls()) +func (mock *Interface) ShutdownCalls() []struct { +} { + var calls []struct { + } + mock.lockShutdown.RLock() + calls = mock.calls.Shutdown + mock.lockShutdown.RUnlock() + return calls +} diff --git a/internal/nvsandboxutils/nvsandboxutils.go b/internal/nvsandboxutils/nvsandboxutils.go new file mode 100644 index 00000000..29544bc9 --- /dev/null +++ b/internal/nvsandboxutils/nvsandboxutils.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package nvsandboxutils + +/* +#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup +#include "nvsandboxutils.h" +#include +#include "cgo_helpers.h" +*/ +import "C" +import "unsafe" + +// nvSandboxUtilsInit function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsInit(Input *InitInput) Ret { + cInput, _ := (*C.nvSandboxUtilsInitInput_t)(unsafe.Pointer(Input)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsInit(cInput) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsShutdown function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsShutdown() Ret { + __ret := C.nvSandboxUtilsShutdown() + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetDriverVersion function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetDriverVersion(Version *byte, Length uint32) Ret { + cVersion, _ := (*C.char)(unsafe.Pointer(Version)), cgoAllocsUnknown + cLength, _ := (C.uint)(Length), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetDriverVersion(cVersion, cLength) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetGpuResource function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetGpuResource(Request *GpuRes) Ret { + cRequest, _ := (*C.nvSandboxUtilsGpuRes_t)(unsafe.Pointer(Request)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetGpuResource(cRequest) + __v := (Ret)(__ret) + return __v +} + +// nvSandboxUtilsGetFileContent function as declared in nvsandboxutils/nvsandboxutils.h +func nvSandboxUtilsGetFileContent(FilePath *byte, Content *byte, ContentSize *uint32) Ret { + cFilePath, _ := (*C.char)(unsafe.Pointer(FilePath)), cgoAllocsUnknown + cContent, _ := (*C.char)(unsafe.Pointer(Content)), cgoAllocsUnknown + cContentSize, _ := (*C.uint)(unsafe.Pointer(ContentSize)), cgoAllocsUnknown + __ret := C.nvSandboxUtilsGetFileContent(cFilePath, cContent, cContentSize) + __v := (Ret)(__ret) + return __v +} diff --git a/internal/nvsandboxutils/nvsandboxutils.h b/internal/nvsandboxutils/nvsandboxutils.h new file mode 100644 index 00000000..3c66e159 --- /dev/null +++ b/internal/nvsandboxutils/nvsandboxutils.h @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVSANDBOXUTILS_H__ +#define __NVSANDBOXUTILS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define INPUT_LENGTH 256 +#define MAX_FILE_PATH 256 +#define MAX_NAME_LENGTH 256 + +/***************************************************************************************************/ +/** @defgroup enums Enumerations + * @{ + */ +/***************************************************************************************************/ + +/** + * Return types + */ +typedef enum +{ + NVSANDBOXUTILS_SUCCESS = 0, //!< The operation was successful + NVSANDBOXUTILS_ERROR_UNINITIALIZED = 1, //!< The library wasn't successfully initialized + NVSANDBOXUTILS_ERROR_NOT_SUPPORTED = 2, //!< The requested operation is not supported on target device + NVSANDBOXUTILS_ERROR_INVALID_ARG = 3, //!< A supplied argument is invalid + NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE = 4, //!< A supplied argument is not large enough + NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED = 5, //!< Requested library version is not supported + NVSANDBOXUTILS_ERROR_LIBRARY_LOAD = 6, //!< The library load failed + NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND = 7, //!< Called function was not found + NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND = 8, //!< Target device was not found + NVSANDBOXUTILS_ERROR_NVML_LIB_CALL = 9, //!< NVML library call failed + NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY = 10, //!< There is insufficient memory + NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND = 11, //!< A supplied file path was not found + NVSANDBOXUTILS_ERROR_UNKNOWN = 0xFFFF, //!< Unknown error occurred +} nvSandboxUtilsRet_t; + +/** + * Return if there is an error + */ +#define RETURN_ON_SANDBOX_ERROR(result) \ + if ((result) != NVSANDBOXUTILS_SUCCESS) { \ + NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \ + return result; \ + } + +/** + * Log levels + */ +typedef enum +{ + NVSANDBOXUTILS_LOG_LEVEL_FATAL = 0, //!< Log fatal errors + NVSANDBOXUTILS_LOG_LEVEL_ERROR = 1, //!< Log all errors + NVSANDBOXUTILS_LOG_LEVEL_WARN = 2, //!< Log all warnings + NVSANDBOXUTILS_LOG_LEVEL_DEBUG = 3, //!< Log all debug messages + NVSANDBOXUTILS_LOG_LEVEL_INFO = 4, //!< Log all info messages + NVSANDBOXUTILS_LOG_LEVEL_NONE = 0xFFFF, //!< Log none +} nvSandboxUtilsLogLevel_t; + +/** + * Input rootfs to help access files inside the driver container + */ +typedef enum +{ + NV_ROOTFS_DEFAULT, //!< Default no rootfs + NV_ROOTFS_PATH, //!< /run/nvidia/driver + NV_ROOTFS_PID, //!< /proc/PID/mountinfo +} nvSandboxUtilsRootfsInputType_t; + +/** + * File type + */ +typedef enum +{ + NV_DEV, //!< /dev file system + NV_PROC, //!< /proc file system + NV_SYS, //!< /sys file system +} nvSandboxUtilsFileType_t; + +/** + * File subtype + */ +typedef enum +{ + NV_DEV_NVIDIA, //!< /dev/nvidia0 + NV_DEV_DRI_CARD, //!< /dev/dri/card1 + NV_DEV_DRI_RENDERD, //!< /dev/dri/renderD128 + NV_DEV_DRI_CARD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-card + NV_DEV_DRI_RENDERD_SYMLINK, //!< /dev/dri/by-path/pci-0000:41:00.0-render + NV_DEV_NVIDIA_UVM, //!< /dev/nvidia-uvm + NV_DEV_NVIDIA_UVM_TOOLS, //!< /dev/nvidia-uvm-tools + NV_DEV_NVIDIA_MODESET, //!< /dev/nvidia-uvm-modeset + NV_DEV_NVIDIA_CTL, //!< /dev/nvidiactl + NV_DEV_GDRDRV, //!< /dev/gdrdrv + NV_DEV_NVIDIA_CAPS_NVIDIA_CAP, //!< /dev/nvidia-caps/nvidia-cap22 + NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID, //!< /proc/driver/nvidia/gpus/0000:2d:00.0 + NV_PROC_DRIVER_NVIDIA_GPUS, //!< /proc/driver/nvidia/gpus (for mask out) + NV_PROC_NVIDIA_PARAMS, //!< /proc/driver/nvidia/params + NV_PROC_NVIDIA_CAPS_MIG_MINORS, //!< /proc/driver/nvidia-caps/mig-minors + NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU, //!< /proc/driver/nvidia/capabilities/gpu0 + NV_PROC_DRIVER_NVIDIA_CAPABILITIES, //!< /proc/driver/nvidia/capabilities (for mask out) + NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS, //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access + NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID, //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0 + NV_SYS_MODULE_NVIDIA_DRIVER, //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out) + NV_NUM_SUBTYPE, // always at the end. +} nvSandboxUtilsFileSystemSubType_t; + +/** + * File module + */ +typedef enum +{ + NV_GPU, //!< Target device + NV_MIG, //!< Target device- MIG + NV_DRIVER_NVIDIA, //!< NVIDIA kernel driver + NV_DRIVER_NVIDIA_UVM, //!< NVIDIA kernel driver-UVM + NV_DRIVER_NVIDIA_MODESET, //!< NVIDIA kernel driver-modeset + NV_DRIVER_GDRDRV, //!< GDRDRV driver + NV_SYSTEM, //!< System module +} nvSandboxUtilsFileModule_t; + +/** + * Flag to provide additional details about the file + */ +typedef enum +{ + NV_FILE_FLAG_HINT = (1 << 0), //!< Default no hint + NV_FILE_FLAG_MASKOUT = (1 << 1), //!< For /proc/driver/nvidia/gpus + NV_FILE_FLAG_CONTENT = (1 << 2), //!< For /proc/driver/nvidia/params + //!< For SYMLINK + //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file + NV_FILE_FLAG_DEPRECTATED = (1 << 3), //!< For all the FIRMWARE GSP file + NV_FILE_FLAG_CANDIDATES = (1 << 4), //!< For libcuda.so +} nvSandboxUtilsFileFlag_t; + +/** + * Input type of the target device + */ +typedef enum +{ + NV_GPU_INPUT_GPU_UUID, //!< GPU UUID + NV_GPU_INPUT_MIG_UUID, //!< MIG UUID + NV_GPU_INPUT_PCI_ID, //!< PCIe DBDF ID + NV_GPU_INPUT_PCI_INDEX, //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF) +} nvSandboxUtilsGpuInputType_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup dataTypes Structures and Unions + * @{ + */ +/***************************************************************************************************/ + +/** + * Initalization input v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsRootfsInputType_t type; //!< One of \p nvSandboxUtilsRootfsInputType_t + char value[INPUT_LENGTH]; //!< String representation of input +} nvSandboxUtilsInitInput_v1_t; + +typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t; + +/** + * File system information + */ +typedef struct nvSandboxUtilsGpuFileInfo_v1_t +{ + struct nvSandboxUtilsGpuFileInfo_v1_t *next; //!< Pointer to the next node in the linked list + nvSandboxUtilsFileType_t fileType; //!< One of \p nvSandboxUtilsFileType_t + nvSandboxUtilsFileSystemSubType_t fileSubType; //!< One of \p nvSandboxUtilsFileSystemSubType_t + nvSandboxUtilsFileModule_t module; //!< One of \p nvSandboxUtilsFileModule_t + nvSandboxUtilsFileFlag_t flags; //!< One of \p nvSandboxUtilsFileFlag_t + char *filePath; //!< Relative file path to rootfs +}nvSandboxUtilsGpuFileInfo_v1_t; + +/** + * GPU resource request v1 + */ +typedef struct +{ + unsigned int version; //!< Version for the structure + nvSandboxUtilsGpuInputType_t inputType; //!< One of \p nvSandboxUtilsGpuInputType_t + char input[INPUT_LENGTH]; //!< String representation of input + nvSandboxUtilsGpuFileInfo_v1_t *files; //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t +} nvSandboxUtilsGpuRes_v1_t; + +typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t; + +/** @} */ + +/***************************************************************************************************/ +/** @defgroup funcs Functions + * @{ + */ +/***************************************************************************************************/ + +/* ************************************************* + * Initialize library + * ************************************************* + */ +/** + * Prepare library resources before library API can be used. + * This initialization will not fail if one of the initialization prerequisites fails. + * @param input Reference to the called-supplied input struct that has initialization fields + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p input->value isn't a valid rootfs path + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p input->version isn't supported by the library + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND if any of the required file paths are not found during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY if there is insufficient system memory during initialization + * @returns @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD on any error during loading the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input); + +/* ************************************************* + * Shutdown library + * ************************************************* + */ +/** + * Clean up library resources created by init call + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + */ +nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void); + +/* ************************************************* + * Get NVIDIA RM driver version + * ************************************************* + */ +/** + * Get NVIDIA RM driver version + * @param version Reference to caller-supplied buffer to return driver version string + * @param length The maximum allowed length of the string returned in \p version + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p version is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL on any error during driver version query from NVML + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length); + +/* ************************************************* + * Get /dev, /proc, /sys file system information + * ************************************************* + */ +/** + * Get /dev, /proc, /sys file system information + * @param request Reference to caller-supplied request struct to return the file system information + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p request->input doesn't match any device + * @returns @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED if \p request->version isn't supported by the library + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request); + +/* ************************************************* + * Get content of given file path + * ************************************************* + */ +/** + * Get file content of input file path + * @param filePath Reference to the file path + * @param content Reference to the caller-supplied buffer to return the file content + * @param contentSize Reference to the maximum allowed size of content. It is updated to the actual size of the content on return + * + * @returns @ref NVSANDBOXUTILS_SUCCESS on success + * @returns @ref NVSANDBOXUTILS_ERROR_INVALID_ARG if \p filePath or \p content is NULL + * @returns @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE if \p contentSize is too small + * @returns @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND on an error while obtaining the content for the file path + */ +nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize); + +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif // __NVSANDBOXUTILS_H__ diff --git a/internal/nvsandboxutils/refcount.go b/internal/nvsandboxutils/refcount.go new file mode 100644 index 00000000..f93107b0 --- /dev/null +++ b/internal/nvsandboxutils/refcount.go @@ -0,0 +1,31 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +type refcount int + +func (r *refcount) IncOnNoError(err error) { + if err == nil { + (*r)++ + } +} + +func (r *refcount) DecOnNoError(err error) { + if err == nil && (*r) > 0 { + (*r)-- + } +} diff --git a/internal/nvsandboxutils/refcount_test.go b/internal/nvsandboxutils/refcount_test.go new file mode 100644 index 00000000..bd311e10 --- /dev/null +++ b/internal/nvsandboxutils/refcount_test.go @@ -0,0 +1,139 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefcount(t *testing.T) { + testCases := []struct { + description string + workload func(r *refcount) + expectedRefcount refcount + }{ + { + description: "No inc or dec", + workload: func(r *refcount) {}, + expectedRefcount: refcount(0), + }, + { + description: "Single inc, no error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + }, + expectedRefcount: refcount(1), + }, + { + description: "Single inc, with error", + workload: func(r *refcount) { + r.IncOnNoError(errors.New("")) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, no error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + }, + expectedRefcount: refcount(2), + }, + { + description: "Double inc, one with error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(errors.New("")) + }, + expectedRefcount: refcount(1), + }, + { + description: "Single dec, no error", + workload: func(r *refcount) { + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Single dec, with error", + workload: func(r *refcount) { + r.DecOnNoError(errors.New("")) + }, + expectedRefcount: refcount(0), + }, + { + description: "Single inc, single dec, no errors", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, no errors", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, one inc error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(errors.New("")) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + { + description: "Double inc, Double dec, one dec error", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(nil) + r.DecOnNoError(errors.New("")) + }, + expectedRefcount: refcount(1), + }, + { + description: "Double inc, Tripple dec, one dec error early on", + workload: func(r *refcount) { + r.IncOnNoError(nil) + r.IncOnNoError(nil) + r.DecOnNoError(errors.New("")) + r.DecOnNoError(nil) + r.DecOnNoError(nil) + }, + expectedRefcount: refcount(0), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + var r refcount + tc.workload(&r) + require.Equal(t, tc.expectedRefcount, r) + }) + } +} diff --git a/internal/nvsandboxutils/return.go b/internal/nvsandboxutils/return.go new file mode 100644 index 00000000..90d4ed84 --- /dev/null +++ b/internal/nvsandboxutils/return.go @@ -0,0 +1,74 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "fmt" +) + +// nvsandboxutils.ErrorString() +func (l *library) ErrorString(r Ret) string { + return r.Error() +} + +// String returns the string representation of a Ret. +func (r Ret) String() string { + return r.Error() +} + +// Error returns the string representation of a Ret. +func (r Ret) Error() string { + return errorStringFunc(r) +} + +// Assigned to nvsandboxutils.ErrorString if the system nvsandboxutils library is in use. +var errorStringFunc = defaultErrorStringFunc + +// nvsanboxutilsErrorString is an alias for the default error string function. +var nvsanboxutilsErrorString = defaultErrorStringFunc + +// defaultErrorStringFunc provides a basic nvsandboxutils.ErrorString implementation. +// This allows the nvsandboxutils.ErrorString function to be used even if the nvsandboxutils library +// is not loaded. +var defaultErrorStringFunc = func(r Ret) string { + switch r { + case SUCCESS: + return "SUCCESS" + case ERROR_UNINITIALIZED: + return "ERROR_UNINITIALIZED" + case ERROR_NOT_SUPPORTED: + return "ERROR_NOT_SUPPORTED" + case ERROR_INVALID_ARG: + return "ERROR_INVALID_ARG" + case ERROR_INSUFFICIENT_SIZE: + return "ERROR_INSUFFICIENT_SIZE" + case ERROR_VERSION_NOT_SUPPORTED: + return "ERROR_VERSION_NOT_SUPPORTED" + case ERROR_LIBRARY_LOAD: + return "ERROR_LIBRARY_LOAD" + case ERROR_FUNCTION_NOT_FOUND: + return "ERROR_FUNCTION_NOT_FOUND" + case ERROR_DEVICE_NOT_FOUND: + return "ERROR_DEVICE_NOT_FOUND" + case ERROR_NVML_LIB_CALL: + return "ERROR_NVML_LIB_CALL" + case ERROR_UNKNOWN: + return "ERROR_UNKNOWN" + default: + return fmt.Sprintf("unknown return value: %d", r) + } +} diff --git a/internal/nvsandboxutils/types_gen.go b/internal/nvsandboxutils/types_gen.go new file mode 100644 index 00000000..90a00ed6 --- /dev/null +++ b/internal/nvsandboxutils/types_gen.go @@ -0,0 +1,39 @@ +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types.go + +package nvsandboxutils + +type InitInput_v1 struct { + Version uint32 + Type uint32 + Value [256]int8 +} + +type InitInput struct { + Version uint32 + Type uint32 + Value [256]int8 +} + +type GpuFileInfo_v1 struct { + Next *GpuFileInfo_v1 + FileType uint32 + FileSubType uint32 + Module uint32 + Flags uint32 + FilePath *int8 +} + +type GpuRes_v1 struct { + Version uint32 + InputType uint32 + Input [256]int8 + Files *GpuFileInfo_v1 +} + +type GpuRes struct { + Version uint32 + InputType uint32 + Input [256]int8 + Files *GpuFileInfo_v1 +} diff --git a/internal/nvsandboxutils/zz_generated.api.go b/internal/nvsandboxutils/zz_generated.api.go new file mode 100644 index 00000000..1cc2b01a --- /dev/null +++ b/internal/nvsandboxutils/zz_generated.api.go @@ -0,0 +1,43 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// Generated Code; DO NOT EDIT. + +package nvsandboxutils + +// The variables below represent package level methods from the library type. +var ( + ErrorString = libnvsandboxutils.ErrorString + GetDriverVersion = libnvsandboxutils.GetDriverVersion + GetFileContent = libnvsandboxutils.GetFileContent + GetGpuResource = libnvsandboxutils.GetGpuResource + Init = libnvsandboxutils.Init + LookupSymbol = libnvsandboxutils.LookupSymbol + Shutdown = libnvsandboxutils.Shutdown +) + +// Interface represents the interface for the library type. +// +//go:generate moq -out mock/interface.go -pkg mock . Interface:Interface +type Interface interface { + ErrorString(Ret) string + GetDriverVersion() (string, Ret) + GetFileContent(string) (string, Ret) + GetGpuResource(string) ([]GpuFileInfo, Ret) + Init(string) Ret + LookupSymbol(string) error + Shutdown() Ret +} From dcbf5bc81f41b1af46342d7db92f62d6949a7298 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:05:09 -0700 Subject: [PATCH 3/5] nvsandboxutils: Add implementation for the APIs This change adds manual wrappers around the generated bindings to make them into more user-friendly APIs for the caller. Some helper functions are also added. The APIs that are currently present in the library and implemented here are: nvSandboxUtilsInit nvSandboxUtilsShutdown nvSandboxUtilsGetDriverVersion nvSandboxUtilsGetGpuResource nvSandboxUtilsGetFileContent Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- internal/nvsandboxutils/cgo_helpers_static.go | 38 +++++++++++ internal/nvsandboxutils/gpu-resources.go | 66 +++++++++++++++++++ internal/nvsandboxutils/impl.go | 64 ++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 internal/nvsandboxutils/cgo_helpers_static.go create mode 100644 internal/nvsandboxutils/gpu-resources.go create mode 100644 internal/nvsandboxutils/impl.go diff --git a/internal/nvsandboxutils/cgo_helpers_static.go b/internal/nvsandboxutils/cgo_helpers_static.go new file mode 100644 index 00000000..5924d622 --- /dev/null +++ b/internal/nvsandboxutils/cgo_helpers_static.go @@ -0,0 +1,38 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +var cgoAllocsUnknown = new(struct{}) + +func clen(n []byte) int { + for i := 0; i < len(n); i++ { + if n[i] == 0 { + return i + } + } + return len(n) +} + +// Creates an int8 array of fixed input length to store the Go string. +// TODO: Add error check if input string has a length greater than INPUT_LENGTH +func convertStringToFixedArray(str string) [INPUT_LENGTH]int8 { + var output [INPUT_LENGTH]int8 + for i, s := range str { + output[i] = int8(s) + } + return output +} diff --git a/internal/nvsandboxutils/gpu-resources.go b/internal/nvsandboxutils/gpu-resources.go new file mode 100644 index 00000000..e47a9996 --- /dev/null +++ b/internal/nvsandboxutils/gpu-resources.go @@ -0,0 +1,66 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import ( + "strings" +) + +import "C" + +type GpuResource struct { + Version uint32 +} + +type GpuFileInfo struct { + Path string + Type FileType + SubType FileSystemSubType + Module FileModule + Flags FileFlag +} + +func (l *library) GetGpuResource(uuid string) ([]GpuFileInfo, Ret) { + deviceType := NV_GPU_INPUT_GPU_UUID + if strings.HasPrefix(uuid, "MIG-") { + deviceType = NV_GPU_INPUT_MIG_UUID + } + + request := GpuRes{ + Version: 1, + InputType: uint32(deviceType), + Input: convertStringToFixedArray(uuid), + } + + ret := nvSandboxUtilsGetGpuResource(&request) + if ret != SUCCESS { + return nil, ret + } + + var fileInfos []GpuFileInfo + for fileInfo := request.Files; fileInfo != nil; fileInfo = fileInfo.Next { + fi := GpuFileInfo{ + Path: C.GoString((*C.char)(fileInfo.FilePath)), + Type: FileType(fileInfo.FileType), + SubType: FileSystemSubType(fileInfo.FileSubType), + Module: FileModule(fileInfo.Module), + Flags: FileFlag(fileInfo.Flags), + } + fileInfos = append(fileInfos, fi) + } + return fileInfos, SUCCESS +} diff --git a/internal/nvsandboxutils/impl.go b/internal/nvsandboxutils/impl.go new file mode 100644 index 00000000..0f6948a2 --- /dev/null +++ b/internal/nvsandboxutils/impl.go @@ -0,0 +1,64 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvsandboxutils + +import "C" + +func (l *library) Init(path string) Ret { + if err := l.load(); err != nil { + return ERROR_LIBRARY_LOAD + } + + input := InitInput{ + Version: 1, + Type: uint32(NV_ROOTFS_PATH), + Value: convertStringToFixedArray(path), + } + + return nvSandboxUtilsInit(&input) +} + +func (l *library) Shutdown() Ret { + ret := nvSandboxUtilsShutdown() + if ret != SUCCESS { + return ret + } + + err := l.close() + if err != nil { + return ERROR_UNKNOWN + } + + return ret +} + +// TODO: Is this length specified in the header file? +const VERSION_LENGTH = 100 + +func (l *library) GetDriverVersion() (string, Ret) { + Version := make([]byte, VERSION_LENGTH) + ret := nvSandboxUtilsGetDriverVersion(&Version[0], VERSION_LENGTH) + return string(Version[:clen(Version)]), ret +} + +func (l *library) GetFileContent(path string) (string, Ret) { + Content := make([]byte, MAX_FILE_PATH) + FilePath := []byte(path + string(byte(0))) + Size := uint32(MAX_FILE_PATH) + ret := nvSandboxUtilsGetFileContent(&FilePath[0], &Content[0], &Size) + return string(Content[:clen(Content)]), ret +} From 7b770f63c3d5dddec43fb515b79fd34d7ec334e4 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:05:11 -0700 Subject: [PATCH 4/5] nvsandboxutils: Add usage of GetDriverVersion API This change includes the usage of Sandboxutils GetDriverVersion API to retrieve the CUDA driver version. If the library is not available on the system or the API call fails for some other reason, it will fallback to the NVML API to return the driver version. Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- pkg/nvcdi/lib-nvml.go | 14 ++++++++++++++ pkg/nvcdi/lib.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/pkg/nvcdi/lib-nvml.go b/pkg/nvcdi/lib-nvml.go index ab7cb8ba..d461aa51 100644 --- a/pkg/nvcdi/lib-nvml.go +++ b/pkg/nvcdi/lib-nvml.go @@ -27,6 +27,7 @@ import ( "tags.cncf.io/container-device-interface/specs-go" "github.com/NVIDIA/nvidia-container-toolkit/internal/edits" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec" ) @@ -52,6 +53,19 @@ func (l *nvmllib) GetAllDeviceSpecs() ([]specs.Device, error) { } }() + if l.nvsandboxutilslib != nil { + if r := l.nvsandboxutilslib.Init(l.driverRoot); r != nvsandboxutils.SUCCESS { + l.logger.Warningf("Failed to init nvsandboxutils: %v; ignoring", r) + l.nvsandboxutilslib = nil + } + defer func() { + if l.nvsandboxutilslib == nil { + return + } + _ = l.nvsandboxutilslib.Shutdown() + }() + } + gpuDeviceSpecs, err := l.getGPUDeviceSpecs() if err != nil { return nil, err diff --git a/pkg/nvcdi/lib.go b/pkg/nvcdi/lib.go index d2db3b6c..9e1c035b 100644 --- a/pkg/nvcdi/lib.go +++ b/pkg/nvcdi/lib.go @@ -26,6 +26,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform" @@ -43,6 +44,7 @@ type wrapper struct { type nvcdilib struct { logger logger.Interface nvmllib nvml.Interface + nvsandboxutilslib nvsandboxutils.Interface mode string devicelib device.Interface deviceNamers DeviceNamers @@ -107,6 +109,19 @@ func New(opts ...Option) (Interface, error) { } l.nvmllib = nvml.New(nvmlOpts...) } + if l.nvsandboxutilslib == nil { + var nvsandboxutilsOpts []nvsandboxutils.LibraryOption + // Set the library path for libnvidia-sandboxutils + candidates, err := l.driver.Libraries().Locate("libnvidia-sandboxutils.so.1") + if err != nil { + l.logger.Warningf("Ignoring error in locating libnvidia-sandboxutils.so.1: %v", err) + } else { + libNvidiaSandboxutilsPath := candidates[0] + l.logger.Infof("Using %v", libNvidiaSandboxutilsPath) + nvsandboxutilsOpts = append(nvsandboxutilsOpts, nvsandboxutils.WithLibraryPath(libNvidiaSandboxutilsPath)) + } + l.nvsandboxutilslib = nvsandboxutils.New(nvsandboxutilsOpts...) + } if l.devicelib == nil { l.devicelib = device.New(l.nvmllib) } @@ -214,6 +229,16 @@ func (l *nvcdilib) resolveMode() (rmode string) { // getCudaVersion returns the CUDA version of the current system. func (l *nvcdilib) getCudaVersion() (string, error) { + version, err := l.getCudaVersionNvsandboxutils() + if err == nil { + return version, err + } + + // Fallback to NVML + return l.getCudaVersionNvml() +} + +func (l *nvcdilib) getCudaVersionNvml() (string, error) { if hasNVML, reason := l.infolib.HasNvml(); !hasNVML { return "", fmt.Errorf("nvml not detected: %v", reason) } @@ -236,3 +261,12 @@ func (l *nvcdilib) getCudaVersion() (string, error) { } return version, nil } + +func (l *nvcdilib) getCudaVersionNvsandboxutils() (string, error) { + // Sandboxutils initialization should happen before this function is called + version, ret := l.nvsandboxutilslib.GetDriverVersion() + if ret != nvsandboxutils.SUCCESS { + return "", fmt.Errorf("%v", ret) + } + return version, nil +} From 563db0e0beaa1da8017617da88403c4dfcfb3bb1 Mon Sep 17 00:00:00 2001 From: Sananya Majumder Date: Tue, 24 Sep 2024 10:05:14 -0700 Subject: [PATCH 5/5] nvsandboxutils: Add usage of GetGpuResource and GetFileContent APIs This change adds a new discoverer for Sandboxutils to report the file system paths and associated symbolic links using GetGpuResource and GetFileContent APIs. Both GPU and MIG devices are supported. If the Sandboxutils discoverer fails, the NVML discoverer is used to report the file system information. Signed-off-by: Evan Lezar Signed-off-by: Huy Nguyen Signed-off-by: Sananya Majumder --- .golangci.yml | 5 + internal/discover/cache.go | 80 ++++++++ internal/discover/first-valid.go | 72 ++++++++ internal/platform-support/dgpu/dgpu.go | 64 ++++++- .../platform-support/dgpu/nvsandboxutils.go | 131 +++++++++++++ .../dgpu/nvsandboxutils_test.go | 174 ++++++++++++++++++ internal/platform-support/dgpu/options.go | 11 ++ pkg/nvcdi/full-gpu-nvml.go | 1 + pkg/nvcdi/mig-device-nvml.go | 1 + 9 files changed, 536 insertions(+), 3 deletions(-) create mode 100644 internal/discover/cache.go create mode 100644 internal/discover/first-valid.go create mode 100644 internal/platform-support/dgpu/nvsandboxutils.go create mode 100644 internal/platform-support/dgpu/nvsandboxutils_test.go diff --git a/.golangci.yml b/.golangci.yml index b4c2f1c2..825d94c3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -36,3 +36,8 @@ issues: linters: - errcheck text: config.Delete + # RENDERD refers to the Render Device and not the past tense of render. + - path: .*.go + linters: + - misspell + text: "`RENDERD` is a misspelling of `RENDERED`" diff --git a/internal/discover/cache.go b/internal/discover/cache.go new file mode 100644 index 00000000..e9e4b615 --- /dev/null +++ b/internal/discover/cache.go @@ -0,0 +1,80 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import "sync" + +type cache struct { + d Discover + + sync.Mutex + devices []Device + hooks []Hook + mounts []Mount +} + +var _ Discover = (*cache)(nil) + +// WithCache decorates the specified disoverer with a cache. +func WithCache(d Discover) Discover { + if d == nil { + return None{} + } + return &cache{d: d} +} + +func (c *cache) Devices() ([]Device, error) { + c.Lock() + defer c.Unlock() + + if c.devices == nil { + devices, err := c.d.Devices() + if err != nil { + return nil, err + } + c.devices = devices + } + return c.devices, nil +} + +func (c *cache) Hooks() ([]Hook, error) { + c.Lock() + defer c.Unlock() + + if c.hooks == nil { + hooks, err := c.d.Hooks() + if err != nil { + return nil, err + } + c.hooks = hooks + } + return c.hooks, nil +} + +func (c *cache) Mounts() ([]Mount, error) { + c.Lock() + defer c.Unlock() + + if c.mounts == nil { + mounts, err := c.d.Mounts() + if err != nil { + return nil, err + } + c.mounts = mounts + } + return c.mounts, nil +} diff --git a/internal/discover/first-valid.go b/internal/discover/first-valid.go new file mode 100644 index 00000000..36de9204 --- /dev/null +++ b/internal/discover/first-valid.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import "errors" + +type firstOf []Discover + +// FirstValid returns a discoverer that returns the first non-error result from a list of discoverers. +func FirstValid(discoverers ...Discover) Discover { + var f firstOf + for _, d := range discoverers { + if d == nil { + continue + } + f = append(f, d) + } + return f +} + +func (f firstOf) Devices() ([]Device, error) { + var errs error + for _, d := range f { + devices, err := d.Devices() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return devices, nil + } + return nil, errs +} + +func (f firstOf) Hooks() ([]Hook, error) { + var errs error + for _, d := range f { + hooks, err := d.Hooks() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return hooks, nil + } + return nil, errs +} + +func (f firstOf) Mounts() ([]Mount, error) { + var errs error + for _, d := range f { + mounts, err := d.Mounts() + if err != nil { + errs = errors.Join(errs, err) + continue + } + return mounts, nil + } + return nil, nil +} diff --git a/internal/platform-support/dgpu/dgpu.go b/internal/platform-support/dgpu/dgpu.go index b79f6bd4..de411bd6 100644 --- a/internal/platform-support/dgpu/dgpu.go +++ b/internal/platform-support/dgpu/dgpu.go @@ -17,6 +17,8 @@ package dgpu import ( + "errors" + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" @@ -25,22 +27,78 @@ import ( ) // NewForDevice creates a discoverer for the specified Device. +// nvsandboxutils is used for discovery if specified, otherwise NVML is used. func NewForDevice(d device.Device, opts ...Option) (discover.Discover, error) { o := new(opts...) - return o.newNvmlDGPUDiscoverer(&toRequiredInfo{d}) + var discoverers []discover.Discover + var errs error + nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(d) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvsandboxutilsDiscoverer != nil { + discoverers = append(discoverers, nvsandboxutilsDiscoverer) + } + + nvmlDiscoverer, err := o.newNvmlDGPUDiscoverer(&toRequiredInfo{d}) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvmlDiscoverer != nil { + discoverers = append(discoverers, nvmlDiscoverer) + } + + if len(discoverers) == 0 { + return nil, errs + } + + return discover.WithCache( + discover.FirstValid( + discoverers..., + ), + ), nil } -// NewForDevice creates a discoverer for the specified device and its associated MIG device. +// NewForMigDevice creates a discoverer for the specified device and its associated MIG device. +// nvsandboxutils is used for discovery if specified, otherwise NVML is used. func NewForMigDevice(d device.Device, mig device.MigDevice, opts ...Option) (discover.Discover, error) { o := new(opts...) + o.isMigDevice = true - return o.newNvmlMigDiscoverer( + var discoverers []discover.Discover + var errs error + nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(mig) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvsandboxutilsDiscoverer != nil { + discoverers = append(discoverers, nvsandboxutilsDiscoverer) + } + + nvmlDiscoverer, err := o.newNvmlMigDiscoverer( &toRequiredMigInfo{ MigDevice: mig, parent: &toRequiredInfo{d}, }, ) + if err != nil { + // TODO: Log a warning + errs = errors.Join(errs, err) + } else if nvmlDiscoverer != nil { + discoverers = append(discoverers, nvmlDiscoverer) + } + + if len(discoverers) == 0 { + return nil, errs + } + + return discover.WithCache( + discover.FirstValid( + discoverers..., + ), + ), nil + } func new(opts ...Option) *options { diff --git a/internal/platform-support/dgpu/nvsandboxutils.go b/internal/platform-support/dgpu/nvsandboxutils.go new file mode 100644 index 00000000..7022deab --- /dev/null +++ b/internal/platform-support/dgpu/nvsandboxutils.go @@ -0,0 +1,131 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package dgpu + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" +) + +type nvsandboxutilsDGPU struct { + lib nvsandboxutils.Interface + uuid string + devRoot string + isMig bool + nvidiaCDIHookPath string + deviceLinks []string +} + +var _ discover.Discover = (*nvsandboxutilsDGPU)(nil) + +type UUIDer interface { + GetUUID() (string, nvml.Return) +} + +func (o *options) newNvsandboxutilsDGPUDiscoverer(d UUIDer) (discover.Discover, error) { + if o.nvsandboxutilslib == nil { + return nil, nil + } + + uuid, nvmlRet := d.GetUUID() + if nvmlRet != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device UUID: %w", nvmlRet) + } + + nvd := nvsandboxutilsDGPU{ + lib: o.nvsandboxutilslib, + uuid: uuid, + devRoot: strings.TrimSuffix(filepath.Clean(o.devRoot), "/dev"), + isMig: o.isMigDevice, + nvidiaCDIHookPath: o.nvidiaCDIHookPath, + } + + return &nvd, nil +} + +func (d *nvsandboxutilsDGPU) Devices() ([]discover.Device, error) { + gpuFileInfos, ret := d.lib.GetGpuResource(d.uuid) + if ret != nvsandboxutils.SUCCESS { + return nil, fmt.Errorf("failed to get GPU resource: %w", ret) + } + + var devices []discover.Device + for _, info := range gpuFileInfos { + switch { + case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD: + if d.isMig { + continue + } + fallthrough + case info.SubType == nvsandboxutils.NV_DEV_NVIDIA, info.SubType == nvsandboxutils.NV_DEV_NVIDIA_CAPS_NVIDIA_CAP: + containerPath := info.Path + if d.devRoot != "/" { + containerPath = strings.TrimPrefix(containerPath, d.devRoot) + } + + // TODO: Extend discover.Device with additional information. + device := discover.Device{ + HostPath: info.Path, + Path: containerPath, + } + devices = append(devices, device) + case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD_SYMLINK, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD_SYMLINK: + if d.isMig { + continue + } + if info.Flags == nvsandboxutils.NV_FILE_FLAG_CONTENT { + targetPath, ret := d.lib.GetFileContent(info.Path) + if ret != nvsandboxutils.SUCCESS { + return nil, fmt.Errorf("failed to get symlink: %w", ret) + } + d.deviceLinks = append(d.deviceLinks, fmt.Sprintf("%v::%v", targetPath, info.Path)) + } + } + } + + return devices, nil +} + +// Hooks returns a hook to create the by-path symlinks for the discovered devices. +func (d *nvsandboxutilsDGPU) Hooks() ([]discover.Hook, error) { + if len(d.deviceLinks) == 0 { + return nil, nil + } + + var args []string + for _, l := range d.deviceLinks { + args = append(args, "--link", l) + } + + hook := discover.CreateNvidiaCDIHook( + d.nvidiaCDIHookPath, + "create-symlinks", + args..., + ) + + return []discover.Hook{hook}, nil +} + +func (d *nvsandboxutilsDGPU) Mounts() ([]discover.Mount, error) { + return nil, nil +} diff --git a/internal/platform-support/dgpu/nvsandboxutils_test.go b/internal/platform-support/dgpu/nvsandboxutils_test.go new file mode 100644 index 00000000..fb9df187 --- /dev/null +++ b/internal/platform-support/dgpu/nvsandboxutils_test.go @@ -0,0 +1,174 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package dgpu + +import ( + "testing" + + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" + "github.com/NVIDIA/go-nvml/pkg/nvml" + mocknvml "github.com/NVIDIA/go-nvml/pkg/nvml/mock" + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" + mocknvsandboxutils "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils/mock" +) + +func TestNewNvsandboxutilsDGPUDiscoverer(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + nvmllib := &mocknvml.Interface{} + devicelib := device.New( + nvmllib, + ) + + testCases := []struct { + description string + devRoot string + device nvml.Device + nvsandboxutils nvsandboxutils.Interface + expectedError error + expectedDevices []discover.Device + expectedHooks []discover.Hook + expectedMounts []discover.Mount + }{ + { + description: "detects host devices", + device: &mocknvml.Device{ + GetUUIDFunc: func() (string, nvml.Return) { + return "GPU-1234", nvml.SUCCESS + }, + }, + nvsandboxutils: &mocknvsandboxutils.Interface{ + GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + infos := []nvsandboxutils.GpuFileInfo{ + { + Path: "/dev/nvidia0", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidiactl", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidia-uvm", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/dev/nvidia-uvm-tools", + Type: nvsandboxutils.NV_DEV, + }, + } + return infos, nvsandboxutils.SUCCESS + }, + }, + expectedDevices: []discover.Device{ + { + Path: "/dev/nvidia0", + HostPath: "/dev/nvidia0", + }, + { + Path: "/dev/nvidiactl", + HostPath: "/dev/nvidiactl", + }, + { + Path: "/dev/nvidia-uvm", + HostPath: "/dev/nvidia-uvm", + }, + { + Path: "/dev/nvidia-uvm-tools", + HostPath: "/dev/nvidia-uvm-tools", + }, + }, + }, + { + description: "detects container devices", + devRoot: "/some/root", + device: &mocknvml.Device{ + GetUUIDFunc: func() (string, nvml.Return) { + return "GPU-1234", nvml.SUCCESS + }, + }, + nvsandboxutils: &mocknvsandboxutils.Interface{ + GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) { + infos := []nvsandboxutils.GpuFileInfo{ + { + Path: "/some/root/dev/nvidia0", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidiactl", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidia-uvm", + Type: nvsandboxutils.NV_DEV, + }, + { + Path: "/some/root/dev/nvidia-uvm-tools", + Type: nvsandboxutils.NV_DEV, + }, + } + return infos, nvsandboxutils.SUCCESS + }, + }, + expectedDevices: []discover.Device{ + { + Path: "/dev/nvidia0", + HostPath: "/some/root/dev/nvidia0", + }, + { + Path: "/dev/nvidiactl", + HostPath: "/some/root/dev/nvidiactl", + }, + { + Path: "/dev/nvidia-uvm", + HostPath: "/some/root/dev/nvidia-uvm", + }, + { + Path: "/dev/nvidia-uvm-tools", + HostPath: "/some/root/dev/nvidia-uvm-tools", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + o := &options{ + logger: logger, + devRoot: tc.devRoot, + nvsandboxutilslib: tc.nvsandboxutils, + } + + device, err := devicelib.NewDevice(tc.device) + require.NoError(t, err) + + d, err := o.newNvsandboxutilsDGPUDiscoverer(device) + require.ErrorIs(t, err, tc.expectedError) + + devices, _ := d.Devices() + require.EqualValues(t, tc.expectedDevices, devices) + hooks, _ := d.Hooks() + require.EqualValues(t, tc.expectedHooks, hooks) + mounts, _ := d.Mounts() + require.EqualValues(t, tc.expectedMounts, mounts) + }) + } +} diff --git a/internal/platform-support/dgpu/options.go b/internal/platform-support/dgpu/options.go index 41e4d7a9..2fd1c01b 100644 --- a/internal/platform-support/dgpu/options.go +++ b/internal/platform-support/dgpu/options.go @@ -19,6 +19,7 @@ package dgpu import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps" + "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils" ) type options struct { @@ -26,10 +27,13 @@ type options struct { devRoot string nvidiaCDIHookPath string + isMigDevice bool // migCaps stores the MIG capabilities for the system. // If MIG is not available, this is nil. migCaps nvcaps.MigCaps migCapsError error + + nvsandboxutilslib nvsandboxutils.Interface } type Option func(*options) @@ -61,3 +65,10 @@ func WithMIGCaps(migCaps nvcaps.MigCaps) Option { l.migCaps = migCaps } } + +// WithNvsandboxuitilsLib sets the nvsandboxutils library implementation. +func WithNvsandboxuitilsLib(nvsandboxutilslib nvsandboxutils.Interface) Option { + return func(l *options) { + l.nvsandboxutilslib = nvsandboxutilslib + } +} diff --git a/pkg/nvcdi/full-gpu-nvml.go b/pkg/nvcdi/full-gpu-nvml.go index 881d102d..003515ca 100644 --- a/pkg/nvcdi/full-gpu-nvml.go +++ b/pkg/nvcdi/full-gpu-nvml.go @@ -72,6 +72,7 @@ func (l *nvmllib) newFullGPUDiscoverer(d device.Device) (discover.Discover, erro dgpu.WithDevRoot(l.devRoot), dgpu.WithLogger(l.logger), dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath), + dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib), ) if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err) diff --git a/pkg/nvcdi/mig-device-nvml.go b/pkg/nvcdi/mig-device-nvml.go index 91fe879c..5c1a504c 100644 --- a/pkg/nvcdi/mig-device-nvml.go +++ b/pkg/nvcdi/mig-device-nvml.go @@ -55,6 +55,7 @@ func (l *nvmllib) GetMIGDeviceEdits(parent device.Device, mig device.MigDevice) dgpu.WithDevRoot(l.devRoot), dgpu.WithLogger(l.logger), dgpu.WithNVIDIACDIHookPath(l.nvidiaCDIHookPath), + dgpu.WithNvsandboxuitilsLib(l.nvsandboxutilslib), ) if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err)