Merge pull request #878 from elezar/bump-release-v1.17.4

Bump version for v1.17.4 release

.github/workflows/code_scanning.yaml

@@ -0,0 +1,52 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "CodeQL"
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - main
+      - release-*
+  push:
+    branches:
+      - main
+      - release-*
+
+jobs:
+  analyze:
+    name: Analyze Go code with CodeQL
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      security-events: write
+      packages: read
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: go
+        build-mode: manual
+    - shell: bash
+      run: |
+        make build
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:go"

.github/workflows/golang.yaml

@@ -49,7 +49,9 @@ jobs:
         args: -v --timeout 5m
         skip-cache: true
     - name: Check golang modules
-      run: make check-vendor
+      run: | 
+        make check-vendor
+        make -C deployments/devel check-modules
   test:
     name: Unit test
     runs-on: ubuntu-latest

.golangci.yml

@@ -1,5 +1,5 @@
 run:
-  deadline: 10m
+  timeout: 10m
 
 linters:
   enable:
@@ -36,3 +36,8 @@ issues:
     linters:
     - errcheck
     text: config.Delete
+  # RENDERD refers to the Render Device and not the past tense of render.
+  - path: .*.go
+    linters:
+    - misspell
+    text: "`RENDERD` is a misspelling of `RENDERED`"

CHANGELOG.md

@@ -1,5 +1,72 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.17.4
+- Disable mounting of compat libs from container by default
+- Add allow-cuda-compat-libs-from-container feature flag
+- Skip graphics modifier in CSV mode
+- Properly pass configSearchPaths to a Driver constructor
+- Add support for containerd version 3 config
+- Add string TOML source
+
+### Changes in libnvidia-container
+- Add no-cntlibs CLI option to nvidia-container-cli
+
+### Changes in the Toolkit Container
+- Bump CUDA base image version to 12.6.3
+
+## v1.17.3
+- Only allow host-relative LDConfig paths by default.
+### Changes in libnvidia-container
+- Create virtual copy of host ldconfig binary before calling fexecve()
+
+## v1.17.2
+- Fixed a bug where legacy images would set imex channels as `all`.
+
+## v1.17.1
+- Fixed a bug where specific symlinks existing in a container image could cause a container to fail to start.
+- Fixed a bug on Tegra-based systems where a container would fail to start.
+- Fixed a bug where the default container runtime config path was not properly set.
+
+### Changes in the Toolkit Container
+- Fallback to using a config file if the current runtime config can not be determined from the command line.
+
+## v1.17.0
+- Promote v1.17.0-rc.2 to v1.17.0
+- Fix bug when using just-in-time CDI spec generation
+- Check for valid paths in create-symlinks hook
+
+## v1.17.0-rc.2
+- Fix bug in locating libcuda.so from ldcache
+- Fix bug in sorting of symlink chain
+- Remove unsupported print-ldcache command
+- Remove csv-filename support from create-symlinks
+
+### Changes in the Toolkit Container
+- Fallback to `crio-status` if `crio status` does not work when configuring the crio runtime
+
+## v1.17.0-rc.1
+- Allow IMEX channels to be requested as volume mounts
+- Fix typo in error message
+- Add disable-imex-channel-creation feature flag
+- Add -z,lazy to LDFLAGS
+- Add imex channels to management CDI spec
+- Add support to fetch current container runtime config from the command line.
+- Add creation of select driver symlinks to CDI spec generation.
+- Remove support for config overrides when configuring runtimes.
+- Skip explicit creation of libnvidia-allocator.so.1 symlink
+- Add vdpau as as a driver library search path.
+- Add support for using libnvsandboxutils to generate CDI specifications.
+
+### Changes in the Toolkit Container
+
+- Allow opt-in features to be selected when deploying the toolkit-container.
+- Bump CUDA base image version to 12.6.2
+- Remove support for config overrides when configuring runtimes.
+
+### Changes in libnvidia-container
+
+- Add no-create-imex-channels command line option.
+
 ## v1.16.2
 - Exclude libnvidia-allocator from graphics mounts. This fixes a bug that leaks mounts when a container is started with bi-directional mount propagation.
 - Use empty string for default runtime-config-override. This removes a redundant warning for runtimes (e.g. Docker) where this is not applicable.
@@ -135,7 +202,7 @@
 ## v1.14.0-rc.2
 * Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
 * Remove dependency on coreutils when installing package on RPM-based systems.
-* Create ouput folders if required when running `nvidia-ctk runtime configure`
+* Create output folders if required when running `nvidia-ctk runtime configure`
 * Generate default config as post-install step.
 * Added support for detecting GSP firmware at custom paths when generating CDI specifications.
 * Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.

Makefile

@@ -60,7 +60,7 @@ endif
 cmds: $(CMD_TARGETS)
 
 ifneq ($(shell uname),Darwin)
-EXTLDFLAGS = -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files
+EXTLDFLAGS = -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files -Wl,-z,lazy
 else
 EXTLDFLAGS = -Wl,-undefined,dynamic_lookup
 endif

cmd/nvidia-cdi-hook/create-symlinks/create-symlinks.go

@@ -17,18 +17,18 @@
 package symlinks
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/moby/sys/symlink"
 	"github.com/urfave/cli/v2"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
 )
 
 type command struct {
@@ -36,8 +36,6 @@ type command struct {
 }
 
 type config struct {
-	hostRoot      string
-	filenames     cli.StringSlice
 	links         cli.StringSlice
 	containerSpec string
 }
@@ -50,39 +48,30 @@ func NewCommand(logger logger.Interface) *cli.Command {
 	return c.build()
 }
 
-// build
+// build creates the create-symlink command.
 func (m command) build() *cli.Command {
 	cfg := config{}
 
-	// Create the '' command
 	c := cli.Command{
 		Name:  "create-symlinks",
-		Usage: "A hook to create symlinks in the container. This can be used to process CSV mount specs",
+		Usage: "A hook to create symlinks in the container.",
 		Action: func(c *cli.Context) error {
 			return m.run(c, &cfg)
 		},
 	}
 
 	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "host-root",
-			Usage:       "The root on the host filesystem to use to resolve symlinks",
-			Destination: &cfg.hostRoot,
-		},
-		&cli.StringSliceFlag{
-			Name:        "csv-filename",
-			Usage:       "Specify a (CSV) filename to process",
-			Destination: &cfg.filenames,
-		},
 		&cli.StringSliceFlag{
 			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as target::link",
+			Usage:       "Specify a specific link to create. The link is specified as target::link. If the link exists in the container root, it is removed.",
 			Destination: &cfg.links,
 		},
+		// The following flags are testing-only flags.
 		&cli.StringFlag{
 			Name:        "container-spec",
-			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN. This is only intended for testing.",
 			Destination: &cfg.containerSpec,
+			Hidden:      true,
 		},
 	}
 
@@ -100,90 +89,65 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
-	csvFiles := cfg.filenames.Value()
-
-	chainLocator := lookup.NewSymlinkChainLocator(
-		lookup.WithLogger(m.logger),
-		lookup.WithRoot(cfg.hostRoot),
-	)
-
-	var candidates []string
-	for _, file := range csvFiles {
-		mountSpecs, err := csv.NewCSVFileParser(m.logger, file).Parse()
-		if err != nil {
-			m.logger.Debugf("Skipping CSV file %v: %v", file, err)
-			continue
-		}
-
-		for _, ms := range mountSpecs {
-			if ms.Type != csv.MountSpecSym {
-				continue
-			}
-			targets, err := chainLocator.Locate(ms.Path)
-			if err != nil {
-				m.logger.Warningf("Failed to locate symlink %v", ms.Path)
-			}
-			candidates = append(candidates, targets...)
-		}
-	}
-
 	created := make(map[string]bool)
-	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
-	for _, candidate := range candidates {
-		target, err := symlinks.Resolve(candidate)
-		if err != nil {
-			m.logger.Debugf("Skipping invalid link: %v", err)
-			continue
-		} else if target == candidate {
-			m.logger.Debugf("%v is not a symlink", candidate)
+	for _, l := range cfg.links.Value() {
+		if created[l] {
+			m.logger.Debugf("Link %v already processed", l)
 			continue
 		}
-
-		err = m.createLink(created, cfg.hostRoot, containerRoot, target, candidate)
-		if err != nil {
-			m.logger.Warningf("Failed to create link %v: %v", []string{target, candidate}, err)
-		}
-	}
-
-	links := cfg.links.Value()
-	for _, l := range links {
 		parts := strings.Split(l, "::")
 		if len(parts) != 2 {
-			m.logger.Warningf("Invalid link specification %v", l)
-			continue
+			return fmt.Errorf("invalid symlink specification %v", l)
 		}
 
-		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
+		err := m.createLink(containerRoot, parts[0], parts[1])
 		if err != nil {
-			m.logger.Warningf("Failed to create link %v: %v", parts, err)
+			return fmt.Errorf("failed to create link %v: %w", parts, err)
 		}
+		created[l] = true
 	}
-
 	return nil
-
 }
 
-func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
-	linkPath, err := changeRoot(hostRoot, containerRoot, link)
+// createLink creates a symbolic link in the specified container root.
+// This is equivalent to:
+//
+//	chroot {{ .containerRoot }} ln -f -s {{ .target }} {{ .link }}
+//
+// If the specified link already exists and points to the same target, this
+// operation is a no-op.
+// If a file exists at the link path or the link points to a different target
+// this file is removed before creating the link.
+//
+// Note that if the link path resolves to an absolute path oudside of the
+// specified root, this is treated as an absolute path in this root.
+func (m command) createLink(containerRoot string, targetPath string, link string) error {
+	linkPath := filepath.Join(containerRoot, link)
+
+	exists, err := linkExists(targetPath, linkPath)
 	if err != nil {
-		m.logger.Warningf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
+		return fmt.Errorf("failed to check if link exists: %w", err)
 	}
-	if created[linkPath] {
-		m.logger.Debugf("Link %v already created", linkPath)
+	if exists {
+		m.logger.Debugf("Link %s already exists", linkPath)
 		return nil
 	}
 
-	targetPath, err := changeRoot(hostRoot, "/", target)
+	// We resolve the parent of the symlink that we're creating in the container root.
+	// If we resolve the full link path, an existing link at the location itself
+	// is also resolved here and we are unable to force create the link.
+	resolvedLinkParent, err := symlink.FollowSymlinkInScope(filepath.Dir(linkPath), containerRoot)
 	if err != nil {
-		m.logger.Warningf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
+		return fmt.Errorf("failed to follow path for link %v relative to %v: %w", link, containerRoot, err)
 	}
+	resolvedLinkPath := filepath.Join(resolvedLinkParent, filepath.Base(linkPath))
 
-	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)
-	err = os.MkdirAll(filepath.Dir(linkPath), 0755)
+	m.logger.Infof("Symlinking %v to %v", resolvedLinkPath, targetPath)
+	err = os.MkdirAll(filepath.Dir(resolvedLinkPath), 0755)
 	if err != nil {
 		return fmt.Errorf("failed to create directory: %v", err)
 	}
-	err = os.Symlink(target, linkPath)
+	err = symlinks.ForceCreate(targetPath, resolvedLinkPath)
 	if err != nil {
 		return fmt.Errorf("failed to create symlink: %v", err)
 	}
@@ -191,41 +155,18 @@ func (m command) createLink(created map[string]bool, hostRoot string, containerR
 	return nil
 }
 
-func changeRoot(current string, new string, path string) (string, error) {
-	if !filepath.IsAbs(path) {
-		return path, nil
+// linkExists checks whether the specified link exists.
+// A link exists if the path exists, is a symlink, and points to the specified target.
+func linkExists(target string, link string) (bool, error) {
+	currentTarget, err := symlinks.Resolve(link)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
 	}
-
-	relative := path
-	if current != "" {
-		r, err := filepath.Rel(current, path)
-		if err != nil {
-			return "", err
-		}
-		relative = r
-	}
-
-	return filepath.Join(new, relative), nil
-}
-
-// Locate returns the link target of the specified filename or an empty slice if the
-// specified filename is not a symlink.
-func (m command) Locate(filename string) ([]string, error) {
-	info, err := os.Lstat(filename)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get file info: %v", info)
+		return false, fmt.Errorf("failed to resolve existing symlink %s: %w", link, err)
 	}
-	if info.Mode()&os.ModeSymlink == 0 {
-		m.logger.Debugf("%v is not a symlink", filename)
-		return nil, nil
+	if currentTarget == target {
+		return true, nil
 	}
-
-	target, err := os.Readlink(filename)
-	if err != nil {
-		return nil, fmt.Errorf("error checking symlink: %v", err)
-	}
-
-	m.logger.Debugf("Resolved link: '%v' => '%v'", filename, target)
-
-	return []string{target}, nil
+	return false, nil
 }

cmd/nvidia-cdi-hook/create-symlinks/create-symlinks_test.go

@@ -0,0 +1,297 @@
+package symlinks
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+)
+
+func TestLinkExist(t *testing.T) {
+	tmpDir := t.TempDir()
+	require.NoError(
+		t,
+		makeFs(tmpDir,
+			dirOrLink{path: "/a/b/c", target: "d"},
+			dirOrLink{path: "/a/b/e", target: "/a/b/f"},
+		),
+	)
+
+	exists, err := linkExists("d", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.True(t, exists)
+
+	exists, err = linkExists("/a/b/f", filepath.Join(tmpDir, "/a/b/e"))
+	require.NoError(t, err)
+	require.True(t, exists)
+
+	exists, err = linkExists("different-target", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.False(t, exists)
+
+	exists, err = linkExists("/a/b/d", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.False(t, exists)
+
+	exists, err = linkExists("foo", filepath.Join(tmpDir, "/a/b/does-not-exist"))
+	require.NoError(t, err)
+	require.False(t, exists)
+}
+
+func TestCreateLink(t *testing.T) {
+	type link struct {
+		path   string
+		target string
+	}
+	type expectedLink struct {
+		link
+		err error
+	}
+
+	testCases := []struct {
+		description         string
+		containerContents   []dirOrLink
+		link                link
+		expectedCreateError error
+		expectedLinks       []expectedLink
+	}{
+		{
+			description: "link to / resolves to container root",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "libfoo.so.1",
+					},
+				},
+			},
+		},
+		{
+			description: "link to / resolves to container root; parent relative link",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "../libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "../libfoo.so.1",
+					},
+				},
+			},
+		},
+		{
+			description: "link to / resolves to container root; absolute link",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "/a-path-in-container/foo/libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "/a-path-in-container/foo/libfoo.so.1",
+					},
+				},
+				{
+					// We also check that the target is NOT created.
+					link: link{
+						path: "{{ .containerRoot }}/a-path-in-container/foo/libfoo.so.1",
+					},
+					err: os.ErrNotExist,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			hostRoot := filepath.Join(tmpDir, "/host-root/")
+			containerRoot := filepath.Join(tmpDir, "/container-root")
+
+			require.NoError(t, makeFs(hostRoot))
+			require.NoError(t, makeFs(containerRoot, tc.containerContents...))
+
+			// nvidia-cdi-hook create-symlinks --link linkSpec
+			err := getTestCommand().createLink(containerRoot, tc.link.target, tc.link.path)
+			// TODO: We may be able to replace this with require.ErrorIs.
+			if tc.expectedCreateError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			for _, expectedLink := range tc.expectedLinks {
+				path := strings.ReplaceAll(expectedLink.path, "{{ .containerRoot }}", containerRoot)
+				path = strings.ReplaceAll(path, "{{ .hostRoot }}", hostRoot)
+				if expectedLink.target != "" {
+					target, err := symlinks.Resolve(path)
+					require.ErrorIs(t, err, expectedLink.err)
+					require.Equal(t, expectedLink.target, target)
+				} else {
+					_, err := os.Stat(path)
+					require.ErrorIs(t, err, expectedLink.err)
+				}
+			}
+		})
+	}
+}
+
+func TestCreateLinkRelativePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root/")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t, makeFs(hostRoot))
+	require.NoError(t, makeFs(containerRoot, dirOrLink{path: "/lib/"}))
+
+	// nvidia-cdi-hook create-symlinks --link libfoo.so.1::/lib/libfoo.so
+	err := getTestCommand().createLink(containerRoot, "libfoo.so.1", "/lib/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "libfoo.so.1", target)
+}
+
+func TestCreateLinkAbsolutePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root/")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t, makeFs(hostRoot))
+	require.NoError(t, makeFs(containerRoot, dirOrLink{path: "/lib/"}))
+
+	// nvidia-cdi-hook create-symlinks --link /lib/libfoo.so.1::/lib/libfoo.so
+	err := getTestCommand().createLink(containerRoot, "/lib/libfoo.so.1", "/lib/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "/lib/libfoo.so.1", target)
+}
+
+func TestCreateLinkAlreadyExists(t *testing.T) {
+	testCases := []struct {
+		description       string
+		containerContents []dirOrLink
+		shouldExist       []string
+	}{
+		{
+			description:       "link already exists with correct target",
+			containerContents: []dirOrLink{{path: "/lib/libfoo.so", target: "libfoo.so.1"}},
+			shouldExist:       []string{},
+		},
+		{
+			description:       "link already exists with different target",
+			containerContents: []dirOrLink{{path: "/lib/libfoo.so", target: "different-target"}, {path: "different-target"}},
+			shouldExist:       []string{"{{ .containerRoot }}/different-target"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			hostRoot := filepath.Join(tmpDir, "/host-root/")
+			containerRoot := filepath.Join(tmpDir, "/container-root")
+			require.NoError(t, makeFs(hostRoot))
+			require.NoError(t, makeFs(containerRoot, tc.containerContents...))
+
+			// nvidia-cdi-hook create-symlinks --link libfoo.so.1::/lib/libfoo.so
+			err := getTestCommand().createLink(containerRoot, "libfoo.so.1", "/lib/libfoo.so")
+			require.NoError(t, err)
+			target, err := symlinks.Resolve(filepath.Join(containerRoot, "lib/libfoo.so"))
+			require.NoError(t, err)
+			require.Equal(t, "libfoo.so.1", target)
+
+			for _, p := range tc.shouldExist {
+				require.DirExists(t, strings.ReplaceAll(p, "{{ .containerRoot }}", containerRoot))
+			}
+		})
+	}
+}
+
+func TestCreateLinkOutOfBounds(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t,
+		makeFs(hostRoot,
+			dirOrLink{path: "libfoo.so"},
+		),
+	)
+	require.NoError(t,
+		makeFs(containerRoot,
+			dirOrLink{path: "/lib"},
+			dirOrLink{path: "/lib/foo", target: hostRoot},
+		),
+	)
+
+	path, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/foo"))
+	require.NoError(t, err)
+	require.Equal(t, hostRoot, path)
+
+	// nvidia-cdi-hook create-symlinks --link ../libfoo.so.1::/lib/foo/libfoo.so
+	_ = getTestCommand().createLink(containerRoot, "../libfoo.so.1", "/lib/foo/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, hostRoot, "libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "../libfoo.so.1", target)
+
+	require.DirExists(t, filepath.Join(hostRoot, "libfoo.so"))
+}
+
+type dirOrLink struct {
+	path   string
+	target string
+}
+
+func makeFs(tmpdir string, fs ...dirOrLink) error {
+	if err := os.MkdirAll(tmpdir, 0o755); err != nil {
+		return err
+	}
+	for _, s := range fs {
+		s.path = filepath.Join(tmpdir, s.path)
+		if s.target == "" {
+			_ = os.MkdirAll(s.path, 0o755)
+			continue
+		}
+		if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
+			return err
+		}
+		if err := os.Symlink(s.target, s.path); err != nil && !os.IsExist(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// getTestCommand creates a command for running tests against.
+func getTestCommand() *command {
+	logger, _ := testlog.NewNullLogger()
+	return &command{
+		logger: logger,
+	}
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -6,8 +6,6 @@ import (
 	"log"
 	"os"
 	"path"
-	"path/filepath"
-	"strings"
 
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
@@ -15,31 +13,15 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 )
 
-const (
-	envCUDAVersion          = "CUDA_VERSION"
-	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
-	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
-	envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
-	envNVMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
-	envNVMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
-	envNVImexChannels       = "NVIDIA_IMEX_CHANNELS"
-	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
-)
-
 const (
 	capSysAdmin = "CAP_SYS_ADMIN"
 )
 
-const (
-	deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
-)
-
 type nvidiaConfig struct {
-	Devices            string
+	Devices            []string
 	MigConfigDevices   string
 	MigMonitorDevices  string
-	ImexChannels       string
+	ImexChannels       []string
 	DriverCapabilities string
 	// Requirements defines the requirements DSL for the container to run.
 	// This is empty if no specific requirements are needed, or if requirements are
@@ -77,23 +59,14 @@ type LinuxCapabilities struct {
 	Ambient     []string `json:"ambient,omitempty" platform:"linux"`
 }
 
-// Mount from OCI runtime spec
-// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L103
-type Mount struct {
-	Destination string   `json:"destination"`
-	Type        string   `json:"type,omitempty" platform:"linux,solaris"`
-	Source      string   `json:"source,omitempty"`
-	Options     []string `json:"options,omitempty"`
-}
-
 // Spec from OCI runtime spec
 // We use pointers to structs, similarly to the latest version of runtime-spec:
 // https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L5-L28
 type Spec struct {
-	Version *string  `json:"ociVersion"`
-	Process *Process `json:"process,omitempty"`
-	Root    *Root    `json:"root,omitempty"`
-	Mounts  []Mount  `json:"mounts,omitempty"`
+	Version *string       `json:"ociVersion"`
+	Process *Process      `json:"process,omitempty"`
+	Root    *Root         `json:"root,omitempty"`
+	Mounts  []specs.Mount `json:"mounts,omitempty"`
 }
 
 // HookState holds state information about the hook
@@ -172,82 +145,30 @@ func isPrivileged(s *Spec) bool {
 	return image.IsPrivileged(&fullSpec)
 }
 
-func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
+func getDevicesFromEnvvar(containerImage image.CUDA, swarmResourceEnvvars []string) []string {
 	// We check if the image has at least one of the Swarm resource envvars defined and use this
 	// if specified.
-	var hasSwarmEnvvar bool
 	for _, envvar := range swarmResourceEnvvars {
-		if image.HasEnvvar(envvar) {
-			hasSwarmEnvvar = true
-			break
+		if containerImage.HasEnvvar(envvar) {
+			return containerImage.DevicesFromEnvvars(swarmResourceEnvvars...).List()
 		}
 	}
 
-	var devices []string
-	if hasSwarmEnvvar {
-		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
-	} else {
-		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
-	}
-
-	if len(devices) == 0 {
-		return nil
-	}
-
-	devicesString := strings.Join(devices, ",")
-
-	return &devicesString
+	return containerImage.VisibleDevicesFromEnvVar()
 }
 
-func getDevicesFromMounts(mounts []Mount) *string {
-	var devices []string
-	for _, m := range mounts {
-		root := filepath.Clean(deviceListAsVolumeMountsRoot)
-		source := filepath.Clean(m.Source)
-		destination := filepath.Clean(m.Destination)
-
-		// Only consider mounts who's host volume is /dev/null
-		if source != "/dev/null" {
-			continue
-		}
-		// Only consider container mount points that begin with 'root'
-		if len(destination) < len(root) {
-			continue
-		}
-		if destination[:len(root)] != root {
-			continue
-		}
-		// Grab the full path beyond 'root' and add it to the list of devices
-		device := destination[len(root):]
-		if len(device) > 0 && device[0] == '/' {
-			device = device[1:]
-		}
-		if len(device) == 0 {
-			continue
-		}
-		devices = append(devices, device)
-	}
-
-	if devices == nil {
-		return nil
-	}
-
-	ret := strings.Join(devices, ",")
-	return &ret
-}
-
-func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
+func (hookConfig *hookConfig) getDevices(image image.CUDA, privileged bool) []string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
-		devices := getDevicesFromMounts(mounts)
-		if devices != nil {
+		devices := image.VisibleDevicesFromMounts()
+		if len(devices) > 0 {
 			return devices
 		}
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
 	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
-	if devices == nil {
+	if len(devices) == 0 {
 		return nil
 	}
 	if privileged || hookConfig.AcceptEnvvarUnprivileged {
@@ -260,12 +181,12 @@ func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privil
 	return nil
 }
 
-func getMigConfigDevices(image image.CUDA) *string {
-	return getMigDevices(image, envNVMigConfigDevices)
+func getMigConfigDevices(i image.CUDA) *string {
+	return getMigDevices(i, image.EnvVarNvidiaMigConfigDevices)
 }
 
-func getMigMonitorDevices(image image.CUDA) *string {
-	return getMigDevices(image, envNVMigMonitorDevices)
+func getMigMonitorDevices(i image.CUDA) *string {
+	return getMigDevices(i, image.EnvVarNvidiaMigMonitorDevices)
 }
 
 func getMigDevices(image image.CUDA, envvar string) *string {
@@ -276,23 +197,35 @@ func getMigDevices(image image.CUDA, envvar string) *string {
 	return &devices
 }
 
-func getImexChannels(image image.CUDA) *string {
-	if !image.HasEnvvar(envNVImexChannels) {
+func (hookConfig *hookConfig) getImexChannels(image image.CUDA, privileged bool) []string {
+	// If enabled, try and get the device list from volume mounts first
+	if hookConfig.AcceptDeviceListAsVolumeMounts {
+		devices := image.ImexChannelsFromMounts()
+		if len(devices) > 0 {
+			return devices
+		}
+	}
+	devices := image.ImexChannelsFromEnvVar()
+	if len(devices) == 0 {
 		return nil
 	}
-	chans := image.Getenv(envNVImexChannels)
-	return &chans
+
+	if privileged || hookConfig.AcceptEnvvarUnprivileged {
+		return devices
+	}
+
+	return nil
 }
 
-func (c *HookConfig) getDriverCapabilities(cudaImage image.CUDA, legacyImage bool) image.DriverCapabilities {
+func (hookConfig *hookConfig) getDriverCapabilities(cudaImage image.CUDA, legacyImage bool) image.DriverCapabilities {
 	// We use the default driver capabilities by default. This is filtered to only include the
 	// supported capabilities
-	supportedDriverCapabilities := image.NewDriverCapabilities(c.SupportedDriverCapabilities)
+	supportedDriverCapabilities := image.NewDriverCapabilities(hookConfig.SupportedDriverCapabilities)
 
 	capabilities := supportedDriverCapabilities.Intersection(image.DefaultDriverCapabilities)
 
-	capsEnvSpecified := cudaImage.HasEnvvar(envNVDriverCapabilities)
-	capsEnv := cudaImage.Getenv(envNVDriverCapabilities)
+	capsEnvSpecified := cudaImage.HasEnvvar(image.EnvVarNvidiaDriverCapabilities)
+	capsEnv := cudaImage.Getenv(image.EnvVarNvidiaDriverCapabilities)
 
 	if !capsEnvSpecified && legacyImage {
 		// Environment variable unset with legacy image: set all capabilities.
@@ -311,14 +244,12 @@ func (c *HookConfig) getDriverCapabilities(cudaImage image.CUDA, legacyImage boo
 	return capabilities
 }
 
-func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
+func (hookConfig *hookConfig) getNvidiaConfig(image image.CUDA, privileged bool) *nvidiaConfig {
 	legacyImage := image.IsLegacy()
 
-	var devices string
-	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
-		devices = *d
-	} else {
-		// 'nil' devices means this is not a GPU container.
+	devices := hookConfig.getDevices(image, privileged)
+	if len(devices) == 0 {
+		// empty devices means this is not a GPU container.
 		return nil
 	}
 
@@ -338,10 +269,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	var imexChannels string
-	if c := getImexChannels(image); c != nil {
-		imexChannels = *c
-	}
+	imexChannels := hookConfig.getImexChannels(image, privileged)
 
 	driverCapabilities := hookConfig.getDriverCapabilities(image, legacyImage).String()
 
@@ -360,7 +288,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	}
 }
 
-func getContainerConfig(hook HookConfig) (config containerConfig) {
+func (hookConfig *hookConfig) getContainerConfig() (config containerConfig) {
 	var h HookState
 	d := json.NewDecoder(os.Stdin)
 	if err := d.Decode(&h); err != nil {
@@ -376,7 +304,8 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	image, err := image.New(
 		image.WithEnv(s.Process.Env),
-		image.WithDisableRequire(hook.DisableRequire),
+		image.WithMounts(s.Mounts),
+		image.WithDisableRequire(hookConfig.DisableRequire),
 	)
 	if err != nil {
 		log.Panicln(err)
@@ -387,6 +316,6 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
 		Image:  image,
-		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
+		Nvidia: hookConfig.getNvidiaConfig(image, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -17,16 +17,10 @@ const (
 	driverPath = "/run/nvidia/driver"
 )
 
-// HookConfig : options for the nvidia-container-runtime-hook.
-type HookConfig config.Config
-
-func getDefaultHookConfig() (HookConfig, error) {
-	defaultCfg, err := config.GetDefault()
-	if err != nil {
-		return HookConfig{}, err
-	}
-
-	return *(*HookConfig)(defaultCfg), nil
+// hookConfig wraps the toolkit config.
+// This allows for functions to be defined on the local type.
+type hookConfig struct {
+	*config.Config
 }
 
 // loadConfig loads the required paths for the hook config.
@@ -56,12 +50,12 @@ func loadConfig() (*config.Config, error) {
 	return config.GetDefault()
 }
 
-func getHookConfig() (*HookConfig, error) {
+func getHookConfig() (*hookConfig, error) {
 	cfg, err := loadConfig()
 	if err != nil {
 		return nil, fmt.Errorf("failed to load config: %v", err)
 	}
-	config := (*HookConfig)(cfg)
+	config := &hookConfig{cfg}
 
 	allSupportedDriverCapabilities := image.SupportedDriverCapabilities
 	if config.SupportedDriverCapabilities == "all" {
@@ -79,7 +73,7 @@ func getHookConfig() (*HookConfig, error) {
 
 // getConfigOption returns the toml config option associated with the
 // specified struct field.
-func (c HookConfig) getConfigOption(fieldName string) string {
+func (c hookConfig) getConfigOption(fieldName string) string {
 	t := reflect.TypeOf(c)
 	f, ok := t.FieldByName(fieldName)
 	if !ok {
@@ -93,7 +87,7 @@ func (c HookConfig) getConfigOption(fieldName string) string {
 }
 
 // getSwarmResourceEnvvars returns the swarm resource envvars for the config.
-func (c *HookConfig) getSwarmResourceEnvvars() []string {
+func (c *hookConfig) getSwarmResourceEnvvars() []string {
 	if c.SwarmResource == "" {
 		return nil
 	}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 )
 
@@ -89,10 +90,10 @@ func TestGetHookConfig(t *testing.T) {
 				}
 			}
 
-			var config HookConfig
+			var cfg hookConfig
 			getHookConfig := func() {
 				c, _ := getHookConfig()
-				config = *c
+				cfg = *c
 			}
 
 			if tc.expectedPanic {
@@ -102,7 +103,7 @@ func TestGetHookConfig(t *testing.T) {
 
 			getHookConfig()
 
-			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+			require.EqualValues(t, tc.expectedDriverCapabilities, cfg.SupportedDriverCapabilities)
 		})
 	}
 }
@@ -144,8 +145,10 @@ func TestGetSwarmResourceEnvvars(t *testing.T) {
 
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			c := &HookConfig{
-				SwarmResource: tc.value,
+			c := &hookConfig{
+				Config: &config.Config{
+					SwarmResource: tc.value,
+				},
 			}
 
 			envvars := c.getSwarmResourceEnvvars()

cmd/nvidia-container-runtime-hook/main.go

@@ -75,7 +75,7 @@ func doPrestart() {
 	}
 	cli := hook.NVIDIAContainerCLIConfig
 
-	container := getContainerConfig(*hook)
+	container := hook.getContainerConfig()
 	nvidia := container.Nvidia
 	if nvidia == nil {
 		// Not a GPU container, nothing to do.
@@ -95,6 +95,9 @@ func doPrestart() {
 	if cli.LoadKmods {
 		args = append(args, "--load-kmods")
 	}
+	if hook.Features.DisableImexChannelCreation.IsEnabled() {
+		args = append(args, "--no-create-imex-channels")
+	}
 	if cli.NoPivot {
 		args = append(args, "--no-pivot")
 	}
@@ -111,14 +114,17 @@ func doPrestart() {
 	}
 	args = append(args, "configure")
 
+	if !hook.Features.AllowCUDACompatLibsFromContainer.IsEnabled() {
+		args = append(args, "--no-cntlibs")
+	}
 	if ldconfigPath := cli.NormalizeLDConfigPath(); ldconfigPath != "" {
 		args = append(args, fmt.Sprintf("--ldconfig=%s", ldconfigPath))
 	}
 	if cli.NoCgroups {
 		args = append(args, "--no-cgroups")
 	}
-	if len(nvidia.Devices) > 0 {
-		args = append(args, fmt.Sprintf("--device=%s", nvidia.Devices))
+	if devicesString := strings.Join(nvidia.Devices, ","); len(devicesString) > 0 {
+		args = append(args, fmt.Sprintf("--device=%s", devicesString))
 	}
 	if len(nvidia.MigConfigDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-config=%s", nvidia.MigConfigDevices))
@@ -126,8 +132,8 @@ func doPrestart() {
 	if len(nvidia.MigMonitorDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
 	}
-	if len(nvidia.ImexChannels) > 0 {
-		args = append(args, fmt.Sprintf("--imex-channel=%s", nvidia.ImexChannels))
+	if imexString := strings.Join(nvidia.ImexChannels, ","); len(imexString) > 0 {
+		args = append(args, fmt.Sprintf("--imex-channel=%s", imexString))
 	}
 
 	for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -17,7 +17,6 @@
 package configure
 
 import (
-	"encoding/json"
 	"fmt"
 	"path/filepath"
 
@@ -29,6 +28,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 const (
@@ -44,13 +44,17 @@ const (
 	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
 	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
 	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
+
+	defaultConfigSource = configSourceFile
+	configSourceCommand = "command"
+	configSourceFile    = "file"
 )
 
 type command struct {
 	logger logger.Interface
 }
 
-// NewCommand constructs an configure command with the specified logger
+// NewCommand constructs a configure command with the specified logger
 func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
@@ -64,6 +68,7 @@ type config struct {
 	dryRun         bool
 	runtime        string
 	configFilePath string
+	configSource   string
 	mode           string
 	hookFilePath   string
 
@@ -120,6 +125,12 @@ func (m command) build() *cli.Command {
 			Usage:       "the config mode for runtimes that support multiple configuration mechanisms",
 			Destination: &config.mode,
 		},
+		&cli.StringFlag{
+			Name:        "config-source",
+			Usage:       "the source to retrieve the container runtime configuration; one of [command, file]\"",
+			Destination: &config.configSource,
+			Value:       defaultConfigSource,
+		},
 		&cli.StringFlag{
 			Name:        "oci-hook-path",
 			Usage:       "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
@@ -156,13 +167,6 @@ func (m command) build() *cli.Command {
 			Usage:       "Enable CDI in the configured runtime",
 			Destination: &config.cdi.enabled,
 		},
-		&cli.StringFlag{
-			Name:        "runtime-config-override",
-			Destination: &config.runtimeConfigOverrideJSON,
-			Usage:       "specify additional runtime options as a JSON string. The paths are relative to the runtime config.",
-			Value:       "",
-			EnvVars:     []string{"RUNTIME_CONFIG_OVERRIDE"},
-		},
 	}
 
 	return &configure
@@ -209,6 +213,29 @@ func (m command) validateFlags(c *cli.Context, config *config) error {
 		config.runtimeConfigOverrideJSON = ""
 	}
 
+	switch config.configSource {
+	case configSourceCommand:
+		if config.runtime == "docker" {
+			m.logger.Warningf("A %v Config Source is not supported for %v; using %v", config.configSource, config.runtime, configSourceFile)
+			config.configSource = configSourceFile
+		}
+	case configSourceFile:
+		break
+	default:
+		return fmt.Errorf("unrecognized Config Source: %v", config.configSource)
+	}
+
+	if config.configFilePath == "" {
+		switch config.runtime {
+		case "containerd":
+			config.configFilePath = defaultContainerdConfigFilePath
+		case "crio":
+			config.configFilePath = defaultCrioConfigFilePath
+		case "docker":
+			config.configFilePath = defaultDockerConfigFilePath
+		}
+	}
+
 	return nil
 }
 
@@ -225,25 +252,29 @@ func (m command) configureWrapper(c *cli.Context, config *config) error {
 
 // configureConfigFile updates the specified container engine config file to enable the NVIDIA runtime.
 func (m command) configureConfigFile(c *cli.Context, config *config) error {
-	configFilePath := config.resolveConfigFilePath()
+	configSource, err := config.resolveConfigSource()
+	if err != nil {
+		return err
+	}
 
 	var cfg engine.Interface
-	var err error
 	switch config.runtime {
 	case "containerd":
 		cfg, err = containerd.New(
 			containerd.WithLogger(m.logger),
-			containerd.WithPath(configFilePath),
+			containerd.WithPath(config.configFilePath),
+			containerd.WithConfigSource(configSource),
 		)
 	case "crio":
 		cfg, err = crio.New(
 			crio.WithLogger(m.logger),
-			crio.WithPath(configFilePath),
+			crio.WithPath(config.configFilePath),
+			crio.WithConfigSource(configSource),
 		)
 	case "docker":
 		cfg, err = docker.New(
 			docker.WithLogger(m.logger),
-			docker.WithPath(configFilePath),
+			docker.WithPath(config.configFilePath),
 		)
 	default:
 		err = fmt.Errorf("unrecognized runtime '%v'", config.runtime)
@@ -252,16 +283,10 @@ func (m command) configureConfigFile(c *cli.Context, config *config) error {
 		return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
 	}
 
-	runtimeConfigOverride, err := config.runtimeConfigOverride()
-	if err != nil {
-		return fmt.Errorf("unable to parse config overrides: %w", err)
-	}
-
 	err = cfg.AddRuntime(
 		config.nvidiaRuntime.name,
 		config.nvidiaRuntime.path,
 		config.nvidiaRuntime.setAsDefault,
-		runtimeConfigOverride,
 	)
 	if err != nil {
 		return fmt.Errorf("unable to update config: %v", err)
@@ -272,7 +297,7 @@ func (m command) configureConfigFile(c *cli.Context, config *config) error {
 		return fmt.Errorf("failed to enable CDI in %s: %w", config.runtime, err)
 	}
 
-	outputPath := config.getOuputConfigPath()
+	outputPath := config.getOutputConfigPath()
 	n, err := cfg.Save(outputPath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)
@@ -290,42 +315,35 @@ func (m command) configureConfigFile(c *cli.Context, config *config) error {
 	return nil
 }
 
-// resolveConfigFilePath returns the default config file path for the configured container engine
-func (c *config) resolveConfigFilePath() string {
-	if c.configFilePath != "" {
-		return c.configFilePath
+// resolveConfigSource returns the default config source or the user provided config source
+func (c *config) resolveConfigSource() (toml.Loader, error) {
+	switch c.configSource {
+	case configSourceCommand:
+		return c.getCommandConfigSource(), nil
+	case configSourceFile:
+		return toml.FromFile(c.configFilePath), nil
+	default:
+		return nil, fmt.Errorf("unrecognized config source: %s", c.configSource)
 	}
-	switch c.runtime {
-	case "containerd":
-		return defaultContainerdConfigFilePath
-	case "crio":
-		return defaultCrioConfigFilePath
-	case "docker":
-		return defaultDockerConfigFilePath
-	}
-	return ""
 }
 
-// getOuputConfigPath returns the configured config path or "" if dry-run is enabled
-func (c *config) getOuputConfigPath() string {
+// getConfigSourceCommand returns the default cli command to fetch the current runtime config
+func (c *config) getCommandConfigSource() toml.Loader {
+	switch c.runtime {
+	case "containerd":
+		return containerd.CommandLineSource("")
+	case "crio":
+		return crio.CommandLineSource("")
+	}
+	return toml.Empty
+}
+
+// getOutputConfigPath returns the configured config path or "" if dry-run is enabled
+func (c *config) getOutputConfigPath() string {
 	if c.dryRun {
 		return ""
 	}
-	return c.resolveConfigFilePath()
-}
-
-// runtimeConfigOverride converts the specified runtimeConfigOverride JSON string to a map.
-func (o *config) runtimeConfigOverride() (map[string]interface{}, error) {
-	if o.runtimeConfigOverrideJSON == "" {
-		return nil, nil
-	}
-
-	runtimeOptions := make(map[string]interface{})
-	if err := json.Unmarshal([]byte(o.runtimeConfigOverrideJSON), &runtimeOptions); err != nil {
-		return nil, fmt.Errorf("failed to read %v as JSON: %w", o.runtimeConfigOverrideJSON, err)
-	}
-
-	return runtimeOptions, nil
+	return c.configFilePath
 }
 
 // configureOCIHook creates and configures the OCI hook for the NVIDIA runtime

cmd/nvidia-ctk/system/print-ldcache/print-ldcache.go

@@ -1,102 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package createdevicenodes
-
-import (
-	"fmt"
-
-	"github.com/urfave/cli/v2"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-)
-
-type command struct {
-	logger logger.Interface
-}
-
-type options struct {
-	driverRoot string
-}
-
-// NewCommand constructs a command sub-command with the specified logger
-func NewCommand(logger logger.Interface) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	opts := options{}
-
-	c := cli.Command{
-		Name:  "print-ldcache",
-		Usage: "A utility to print the contents of the ldcache",
-		Before: func(c *cli.Context) error {
-			return m.validateFlags(c, &opts)
-		},
-		Action: func(c *cli.Context) error {
-			return m.run(c, &opts)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "driver-root",
-			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
-			Value:       "/",
-			Destination: &opts.driverRoot,
-			EnvVars:     []string{"NVIDIA_DRIVER_ROOT", "DRIVER_ROOT"},
-		},
-	}
-
-	return &c
-}
-
-func (m command) validateFlags(r *cli.Context, opts *options) error {
-	return nil
-}
-
-func (m command) run(c *cli.Context, opts *options) error {
-	cache, err := ldcache.New(m.logger, opts.driverRoot)
-	if err != nil {
-		return fmt.Errorf("failed to create ldcache: %v", err)
-	}
-
-	lib32, lib64 := cache.List()
-
-	if len(lib32) == 0 {
-		m.logger.Info("No 32-bit libraries found")
-	} else {
-		m.logger.Infof("%d 32-bit libraries found", len(lib32))
-		for _, lib := range lib32 {
-			m.logger.Infof("%v", lib)
-		}
-	}
-	if len(lib64) == 0 {
-		m.logger.Info("No 64-bit libraries found")
-	} else {
-		m.logger.Infof("%d 64-bit libraries found", len(lib64))
-		for _, lib := range lib64 {
-			m.logger.Infof("%v", lib)
-		}
-	}
-
-	return nil
-}

cmd/nvidia-ctk/system/system.go

@@ -21,7 +21,6 @@ import (
 
 	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
 	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
-	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/print-ldcache"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
@@ -47,7 +46,6 @@ func (m command) build() *cli.Command {
 	system.Subcommands = []*cli.Command{
 		devchar.NewCommand(m.logger),
 		devicenodes.NewCommand(m.logger),
-		ldcache.NewCommand(m.logger),
 	}
 
 	return &system

deployments/container/Dockerfile.packaging

@@ -14,7 +14,7 @@
 
 ARG GOLANG_VERSION=x.x.x
 
-FROM nvidia/cuda:12.6.0-base-ubuntu20.04
+FROM nvidia/cuda:12.6.3-base-ubuntu20.04
 
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/

deployments/container/Dockerfile.ubi8

@@ -15,7 +15,7 @@
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-FROM nvidia/cuda:12.6.0-base-ubi8 as build
+FROM nvidia/cuda:12.6.3-base-ubi8 as build
 
 RUN yum install -y \
     wget make git gcc \
@@ -48,7 +48,7 @@ COPY . .
 RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
 
 
-FROM nvidia/cuda:12.6.0-base-ubi8
+FROM nvidia/cuda:12.6.3-base-ubi8
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=void

deployments/container/Dockerfile.ubuntu

@@ -15,7 +15,7 @@
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-FROM nvidia/cuda:12.6.0-base-ubuntu20.04 as build
+FROM nvidia/cuda:12.6.3-base-ubuntu20.04 as build
 
 RUN apt-get update && \
     apt-get install -y wget make git gcc \
@@ -47,7 +47,7 @@ COPY . .
 RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
 
 
-FROM nvcr.io/nvidia/cuda:12.6.0-base-ubuntu20.04
+FROM nvcr.io/nvidia/cuda:12.6.3-base-ubuntu20.04
 
 # Remove the CUDA repository configurations to avoid issues with rotated GPG keys
 RUN rm -f /etc/apt/sources.list.d/cuda.list

deployments/devel/Dockerfile

@@ -14,7 +14,7 @@
 
 # This Dockerfile is also used to define the golang version used in this project
 # This allows dependabot to manage this version in addition to other images.
-FROM golang:1.23.1
+FROM golang:1.23.4
 
 WORKDIR /work
 COPY * .

deployments/devel/go.mod

@@ -1,27 +1,27 @@
 module github.com/NVIDIA/k8s-device-plugin/deployments/devel
 
-go 1.22.1
+go 1.23
 
 toolchain go1.23.1
 
 require (
-	github.com/golangci/golangci-lint v1.60.1
-	github.com/matryer/moq v0.3.4
+	github.com/golangci/golangci-lint v1.61.0
+	github.com/matryer/moq v0.5.0
 )
 
 require (
 	4d63.com/gocheckcompilerdirectives v1.2.1 // indirect
 	4d63.com/gochecknoglobals v0.2.1 // indirect
 	github.com/4meepo/tagalign v1.3.4 // indirect
-	github.com/Abirdcfly/dupword v0.0.14 // indirect
+	github.com/Abirdcfly/dupword v0.1.1 // indirect
 	github.com/Antonboom/errname v0.1.13 // indirect
 	github.com/Antonboom/nilnil v0.1.9 // indirect
 	github.com/Antonboom/testifylint v1.4.3 // indirect
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
-	github.com/Crocmagnon/fatcontext v0.4.0 // indirect
+	github.com/Crocmagnon/fatcontext v0.5.2 // indirect
 	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
 	github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 // indirect
-	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/semver/v3 v3.3.0 // indirect
 	github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect
 	github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
@@ -42,9 +42,9 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/charithe/durationcheck v0.0.10 // indirect
 	github.com/chavacava/garif v0.1.0 // indirect
-	github.com/ckaznocha/intrange v0.1.2 // indirect
+	github.com/ckaznocha/intrange v0.2.0 // indirect
 	github.com/curioswitch/go-reassign v0.2.0 // indirect
-	github.com/daixiang0/gci v0.13.4 // indirect
+	github.com/daixiang0/gci v0.13.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/denis-tingaikin/go-header v0.5.0 // indirect
 	github.com/ettle/strcase v0.2.0 // indirect
@@ -62,13 +62,13 @@ require (
 	github.com/go-toolsmith/astp v1.1.0 // indirect
 	github.com/go-toolsmith/strparse v1.1.0 // indirect
 	github.com/go-toolsmith/typep v1.1.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.1.0 // indirect
 	github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
-	github.com/golangci/gofmt v0.0.0-20231018234816-f50ced29576e // indirect
+	github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9 // indirect
 	github.com/golangci/misspell v0.6.0 // indirect
 	github.com/golangci/modinfo v0.3.4 // indirect
 	github.com/golangci/plugin-module-register v0.1.1 // indirect
@@ -119,25 +119,25 @@ require (
 	github.com/nunnatsa/ginkgolinter v0.16.2 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polyfloyd/go-errorlint v1.6.0 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/quasilyte/go-ruleguard v0.4.2 // indirect
+	github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 // indirect
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect
 	github.com/quasilyte/gogrep v0.5.0 // indirect
 	github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
 	github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
-	github.com/ryancurrah/gomodguard v1.3.3 // indirect
+	github.com/ryancurrah/gomodguard v1.3.5 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
 	github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
 	github.com/sashamelentyev/usestdlibvars v1.27.0 // indirect
-	github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9 // indirect
+	github.com/securego/gosec/v2 v2.21.2 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sivchari/containedctx v1.0.3 // indirect
@@ -156,10 +156,10 @@ require (
 	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/tdakkota/asciicheck v0.2.0 // indirect
-	github.com/tetafro/godot v1.4.16 // indirect
+	github.com/tetafro/godot v1.4.17 // indirect
 	github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 // indirect
 	github.com/timonwong/loggercheck v0.9.4 // indirect
-	github.com/tomarrell/wrapcheck/v2 v2.8.3 // indirect
+	github.com/tomarrell/wrapcheck/v2 v2.9.0 // indirect
 	github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
 	github.com/ultraware/funlen v0.1.0 // indirect
 	github.com/ultraware/whitespace v0.1.1 // indirect
@@ -175,18 +175,18 @@ require (
 	go.uber.org/automaxprocs v1.5.3 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
+	golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect
 	golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect
-	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	honnef.co/go/tools v0.5.0 // indirect
-	mvdan.cc/gofumpt v0.6.0 // indirect
+	honnef.co/go/tools v0.5.1 // indirect
+	mvdan.cc/gofumpt v0.7.0 // indirect
 	mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect
 )

deployments/devel/go.sum

@@ -37,8 +37,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/4meepo/tagalign v1.3.4 h1:P51VcvBnf04YkHzjfclN6BbsopfJR5rxs1n+5zHt+w8=
 github.com/4meepo/tagalign v1.3.4/go.mod h1:M+pnkHH2vG8+qhE5bVc/zeP7HS/j910Fwa9TUSyZVI0=
-github.com/Abirdcfly/dupword v0.0.14 h1:3U4ulkc8EUo+CaT105/GJ1BQwtgyj6+VaBVbAX11Ba8=
-github.com/Abirdcfly/dupword v0.0.14/go.mod h1:VKDAbxdY8YbKUByLGg8EETzYSuC4crm9WwI6Y3S0cLI=
+github.com/Abirdcfly/dupword v0.1.1 h1:Bsxe0fIw6OwBtXMIncaTxCLHYO5BB+3mcsR5E8VXloY=
+github.com/Abirdcfly/dupword v0.1.1/go.mod h1:B49AcJdTYYkpd4HjgAcutNGG9HZ2JWwKunH9Y2BA6sM=
 github.com/Antonboom/errname v0.1.13 h1:JHICqsewj/fNckzrfVSe+T33svwQxmjC+1ntDsHOVvM=
 github.com/Antonboom/errname v0.1.13/go.mod h1:uWyefRYRN54lBg6HseYCFhs6Qjcy41Y3Jl/dVhA87Ns=
 github.com/Antonboom/nilnil v0.1.9 h1:eKFMejSxPSA9eLSensFmjW2XTgTwJMjZ8hUHtV4s/SQ=
@@ -49,14 +49,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Crocmagnon/fatcontext v0.4.0 h1:4ykozu23YHA0JB6+thiuEv7iT6xq995qS1vcuWZq0tg=
-github.com/Crocmagnon/fatcontext v0.4.0/go.mod h1:ZtWrXkgyfsYPzS6K3O88va6t2GEglG93vnII/F94WC0=
+github.com/Crocmagnon/fatcontext v0.5.2 h1:vhSEg8Gqng8awhPju2w7MKHqMlg4/NI+gSDHtR3xgwA=
+github.com/Crocmagnon/fatcontext v0.5.2/go.mod h1:87XhRMaInHP44Q7Tlc7jkgKKB7kZAOPiDkFMdKCC+74=
 github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM=
 github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 h1:/fTUt5vmbkAcMBt4YQiuC23cV0kEsN1MVMNqeOW43cU=
 github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0/go.mod h1:ONJg5sxcbsdQQ4pOW8TGdTidT2TMAUy/2Xhr8mrYaao=
-github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
-github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0=
+github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
 github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
@@ -115,15 +115,15 @@ github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+U
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/ckaznocha/intrange v0.1.2 h1:3Y4JAxcMntgb/wABQ6e8Q8leMd26JbX2790lIss9MTI=
-github.com/ckaznocha/intrange v0.1.2/go.mod h1:RWffCw/vKBwHeOEwWdCikAtY0q4gGt8VhJZEEA5n+RE=
+github.com/ckaznocha/intrange v0.2.0 h1:FykcZuJ8BD7oX93YbO1UY9oZtkRbp+1/kJcDjkefYLs=
+github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdcuRFeevn1oE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
 github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc=
-github.com/daixiang0/gci v0.13.4 h1:61UGkmpoAcxHM2hhNkZEf5SzwQtWJXTSws7jaPyqwlw=
-github.com/daixiang0/gci v0.13.4/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
+github.com/daixiang0/gci v0.13.5 h1:kThgmH1yBmZSBCh1EJVxQ7JsHpm5Oms0AMed/0LaH4c=
+github.com/daixiang0/gci v0.13.5/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -141,8 +141,8 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/firefart/nonamedreturns v1.0.5 h1:tM+Me2ZaXs8tfdDw3X6DOX++wMCOqzYUho6tUTYIdRA=
 github.com/firefart/nonamedreturns v1.0.5/go.mod h1:gHJjDqhGM4WyPt639SOZs+G89Ko7QKH5R5BhnO6xJhw=
-github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
-github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
+github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
@@ -160,8 +160,10 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
-github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
@@ -184,8 +186,8 @@ github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQi
 github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ=
 github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus=
 github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig=
-github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc=
-github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w=
+github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U=
 github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
@@ -224,10 +226,10 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/gofmt v0.0.0-20231018234816-f50ced29576e h1:ULcKCDV1LOZPFxGZaA6TlQbiM3J2GCPnkx/bGF6sX/g=
-github.com/golangci/gofmt v0.0.0-20231018234816-f50ced29576e/go.mod h1:Pm5KhLPA8gSnQwrQ6ukebRcapGb/BG9iUkdaiCcGHJM=
-github.com/golangci/golangci-lint v1.60.1 h1:DRKNqNTQRLBJZ1il5u4fvgLQCjQc7QFs0DbhksJtVJE=
-github.com/golangci/golangci-lint v1.60.1/go.mod h1:jDIPN1rYaIA+ijp9OZcUmUCoQOtZ76pOlFbi15FlLJY=
+github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9 h1:/1322Qns6BtQxUZDTAT4SdcoxknUki7IAoK4SAXr8ME=
+github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9/go.mod h1:Oesb/0uFAyWoaw1U1qS5zyjCg5NP9C9iwjnI4tIsXEE=
+github.com/golangci/golangci-lint v1.61.0 h1:VvbOLaRVWmyxCnUIMTbf1kDsaJbTzH20FAMXTAlQGu8=
+github.com/golangci/golangci-lint v1.61.0/go.mod h1:e4lztIrJJgLPhWvFPDkhiMwEFRrWlmFbrZea3FsJyN8=
 github.com/golangci/misspell v0.6.0 h1:JCle2HUTNWirNlDIAUO44hUsKhOFqGPoC4LZxlaSXDs=
 github.com/golangci/misspell v0.6.0/go.mod h1:keMNyY6R9isGaSAu+4Q8NMBwMPkh15Gtc8UCVoDtAWo=
 github.com/golangci/modinfo v0.3.4 h1:oU5huX3fbxqQXdfspamej74DFX0kyGLkw1ppvXoJ8GA=
@@ -264,8 +266,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
-github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -359,8 +361,8 @@ github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 h1:gWg6ZQ4JhDfJPqlo2
 github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
 github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
-github.com/matryer/moq v0.3.4 h1:czCFIos9rI2tyOehN9ktc/6bQ76N9J4xQ2n3dk063ac=
-github.com/matryer/moq v0.3.4/go.mod h1:wqm9QObyoMuUtH81zFfs3EK6mXEcByy+TjvSROOXJ2U=
+github.com/matryer/moq v0.5.0 h1:h2PJUYjZSiyEahzVogDRmrgL9Bsx9xYAl8l+LPfmwL8=
+github.com/matryer/moq v0.5.0/go.mod h1:39GTnrD0mVWHPvWdYj5ki/lxfhLQEtHcLh+tWoYF/iE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -395,10 +397,10 @@ github.com/nunnatsa/ginkgolinter v0.16.2 h1:8iLqHIZvN4fTLDC0Ke9tbSZVcyVHoBs0HIbn
 github.com/nunnatsa/ginkgolinter v0.16.2/go.mod h1:4tWRinDN1FeJgU+iJANW/kz7xKN5nYRAOfJDQUS9dOQ=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU=
-github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc=
-github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
-github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
+github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=
+github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
+github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
+github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
@@ -408,8 +410,8 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
-github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -442,8 +444,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/quasilyte/go-ruleguard v0.4.2 h1:htXcXDK6/rO12kiTHKfHuqR4kr3Y4M0J0rOL6CH/BYs=
-github.com/quasilyte/go-ruleguard v0.4.2/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI=
+github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 h1:+Wl/0aFp0hpuHM3H//KMft64WQ1yX9LdJY64Qm/gFCo=
+github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI=
 github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
 github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
 github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo=
@@ -456,8 +458,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryancurrah/gomodguard v1.3.3 h1:eiSQdJVNr9KTNxY2Niij8UReSwR8Xrte3exBrAZfqpg=
-github.com/ryancurrah/gomodguard v1.3.3/go.mod h1:rsKQjj4l3LXe8N344Ow7agAy5p9yjsWOtRzUMYmA0QY=
+github.com/ryancurrah/gomodguard v1.3.5 h1:cShyguSwUEeC0jS7ylOiG/idnd1TpJ1LfHGpV3oJmPU=
+github.com/ryancurrah/gomodguard v1.3.5/go.mod h1:MXlEPQRxgfPQa62O8wzK3Ozbkv9Rkqr+wKjSxTdsNJE=
 github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU=
 github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ=
 github.com/sanposhiho/wastedassign/v2 v2.0.7 h1:J+6nrY4VW+gC9xFzUc+XjPD3g3wF3je/NsJFwFK7Uxc=
@@ -468,8 +470,8 @@ github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tM
 github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ=
 github.com/sashamelentyev/usestdlibvars v1.27.0 h1:t/3jZpSXtRPRf2xr0m63i32ZrusyurIGT9E5wAvXQnI=
 github.com/sashamelentyev/usestdlibvars v1.27.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8=
-github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9 h1:rnO6Zp1YMQwv8AyxzuwsVohljJgp4L0ZqiCgtACsPsc=
-github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9/go.mod h1:dg7lPlu/xK/Ut9SedURCoZbVCR4yC7fM65DtH9/CDHs=
+github.com/securego/gosec/v2 v2.21.2 h1:deZp5zmYf3TWwU7A7cR2+SolbTpZ3HQiwFqnzQyEl3M=
+github.com/securego/gosec/v2 v2.21.2/go.mod h1:au33kg78rNseF5PwPnTWhuYBFf534bvJRvOrgZ/bFzU=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
@@ -527,14 +529,14 @@ github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA
 github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0=
 github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag=
 github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
-github.com/tetafro/godot v1.4.16 h1:4ChfhveiNLk4NveAZ9Pu2AN8QZ2nkUGFuadM9lrr5D0=
-github.com/tetafro/godot v1.4.16/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio=
+github.com/tetafro/godot v1.4.17 h1:pGzu+Ye7ZUEFx7LHU0dAKmCOXWsPjl7qA6iMGndsjPs=
+github.com/tetafro/godot v1.4.17/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio=
 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M=
 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ=
 github.com/timonwong/loggercheck v0.9.4 h1:HKKhqrjcVj8sxL7K77beXh0adEm6DLjV/QOGeMXEVi4=
 github.com/timonwong/loggercheck v0.9.4/go.mod h1:caz4zlPcgvpEkXgVnAJGowHAMW2NwHaNlpS8xDbVhTg=
-github.com/tomarrell/wrapcheck/v2 v2.8.3 h1:5ov+Cbhlgi7s/a42BprYoxsr73CbdMUTzE3bRDFASUs=
-github.com/tomarrell/wrapcheck/v2 v2.8.3/go.mod h1:g9vNIyhb5/9TQgumxQyOEqDHsmGYcGsVMOx/xGkqdMo=
+github.com/tomarrell/wrapcheck/v2 v2.9.0 h1:801U2YCAjLhdN8zhZ/7tdjB3EnAoRlJHt/s+9hijLQ4=
+github.com/tomarrell/wrapcheck/v2 v2.9.0/go.mod h1:g9vNIyhb5/9TQgumxQyOEqDHsmGYcGsVMOx/xGkqdMo=
 github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw=
 github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
 github.com/ultraware/funlen v0.1.0 h1:BuqclbkY6pO+cvxoq7OsktIXZpgBSkYTQtmwhAK81vI=
@@ -599,8 +601,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=
-golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk=
+golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
 golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8=
@@ -633,8 +635,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
-golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -749,8 +751,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
-golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -767,8 +769,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -913,8 +915,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -941,10 +943,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.5.0 h1:29uoiIormS3Z6R+t56STz/oI4v+mB51TSmEOdJPgRnE=
-honnef.co/go/tools v0.5.0/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
-mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo=
-mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA=
+honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I=
+honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
+mvdan.cc/gofumpt v0.7.0 h1:bg91ttqXmi9y2xawvkuMXyvAA/1ZGJqYAEGjXuP0JXU=
+mvdan.cc/gofumpt v0.7.0/go.mod h1:txVFJy/Sc/mvaycET54pV8SW8gWxTlUuGHVEcncmNUo=
 mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f h1:lMpcwN6GxNbWtbpI1+xzFLSW8XzX0u72NttUGVFjO3U=
 mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f/go.mod h1:RSLa7mKKCNeTTMHBw5Hsy2rfJmd6O2ivt9Dw9ZqCQpQ=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

go.mod

@@ -4,21 +4,22 @@ go 1.20
 
 require (
 	github.com/NVIDIA/go-nvlib v0.6.1
-	github.com/NVIDIA/go-nvml v0.12.4-0
+	github.com/NVIDIA/go-nvml v0.12.4-1
 	github.com/fsnotify/fsnotify v1.7.0
+	github.com/moby/sys/symlink v0.3.0
 	github.com/opencontainers/runtime-spec v1.2.0
 	github.com/pelletier/go-toml v1.9.5
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
-	github.com/urfave/cli/v2 v2.27.4
+	github.com/urfave/cli/v2 v2.27.5
 	golang.org/x/mod v0.20.0
-	golang.org/x/sys v0.24.0
+	golang.org/x/sys v0.26.0
 	tags.cncf.io/container-device-interface v0.8.0
 	tags.cncf.io/container-device-interface/specs-go v0.8.0
 )
 
 require (
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect

go.sum

@@ -1,11 +1,11 @@
 github.com/NVIDIA/go-nvlib v0.6.1 h1:0/5FvaKvDJoJeJ+LFlh+NDQMxMlVw9wOXrOVrGXttfE=
 github.com/NVIDIA/go-nvlib v0.6.1/go.mod h1:9UrsLGx/q1OrENygXjOuM5Ey5KCtiZhbvBlbUIxtGWY=
-github.com/NVIDIA/go-nvml v0.12.4-0 h1:4tkbB3pT1O77JGr0gQ6uD8FrsUPqP1A/EOEm2wI1TUg=
-github.com/NVIDIA/go-nvml v0.12.4-0/go.mod h1:8Llmj+1Rr+9VGGwZuRer5N/aCjxGuR5nPb/9ebBiIEQ=
+github.com/NVIDIA/go-nvml v0.12.4-1 h1:WKUvqshhWSNTfm47ETRhv0A0zJyr1ncCuHiXwoTrBEc=
+github.com/NVIDIA/go-nvml v0.12.4-1/go.mod h1:8Llmj+1Rr+9VGGwZuRer5N/aCjxGuR5nPb/9ebBiIEQ=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -28,6 +28,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
+github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU=
+github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk=
@@ -58,8 +60,8 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
-github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -74,8 +76,8 @@ golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

hack/golang-version.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../hack && pwd )"
 
 DOCKERFILE_ROOT=${SCRIPTS_DIR}/../deployments/devel
 

internal/config/cli.go

@@ -17,6 +17,7 @@
 package config
 
 import (
+	"fmt"
 	"os"
 	"strings"
 )
@@ -34,29 +35,62 @@ type ContainerCLIConfig struct {
 	NoPivot   bool   `toml:"no-pivot,omitempty"`
 	NoCgroups bool   `toml:"no-cgroups"`
 	User      string `toml:"user"`
-	Ldconfig  string `toml:"ldconfig"`
+	// Ldconfig represents the path to the ldconfig binary to be used to update
+	// the ldcache in a container as it is being created.
+	// If this path starts with a '@' the path is relative to the host and if
+	// not it is treated as a container path.
+	//
+	// Note that the use of container paths are disabled by default and if this
+	// is required, the features.allow-ldconfig-from-container feature gate must
+	// be enabled explicitly.
+	Ldconfig ldconfigPath `toml:"ldconfig"`
 }
 
 // NormalizeLDConfigPath returns the resolved path of the configured LDConfig binary.
 // This is only done for host LDConfigs and is required to handle systems where
 // /sbin/ldconfig is a wrapper around /sbin/ldconfig.real.
 func (c *ContainerCLIConfig) NormalizeLDConfigPath() string {
-	return NormalizeLDConfigPath(c.Ldconfig)
+	return string(c.Ldconfig.normalize())
+}
+
+// An ldconfigPath is used to represent the path to ldconfig.
+type ldconfigPath string
+
+func (p ldconfigPath) assertValid(allowContainerRelativePath bool) error {
+	if p.isHostRelative() {
+		return nil
+	}
+	if allowContainerRelativePath {
+		return nil
+	}
+	return fmt.Errorf("nvidia-container-cli.ldconfig value %q is not host-relative (does not start with a '@')", p)
+}
+
+func (p ldconfigPath) isHostRelative() bool {
+	return strings.HasPrefix(string(p), "@")
+}
+
+// normalize returns the resolved path of the configured LDConfig binary.
+// This is only done for host LDConfigs and is required to handle systems where
+// /sbin/ldconfig is a wrapper around /sbin/ldconfig.real.
+func (p ldconfigPath) normalize() ldconfigPath {
+	if !p.isHostRelative() {
+		return p
+	}
+
+	path := string(p)
+	trimmedPath := strings.TrimSuffix(strings.TrimPrefix(path, "@"), ".real")
+	// If the .real path exists, we return that.
+	if _, err := os.Stat(trimmedPath + ".real"); err == nil {
+		return ldconfigPath("@" + trimmedPath + ".real")
+	}
+	// If the .real path does not exists (or cannot be read) we return the non-.real path.
+	return ldconfigPath("@" + trimmedPath)
 }
 
 // NormalizeLDConfigPath returns the resolved path of the configured LDConfig binary.
 // This is only done for host LDConfigs and is required to handle systems where
 // /sbin/ldconfig is a wrapper around /sbin/ldconfig.real.
 func NormalizeLDConfigPath(path string) string {
-	if !strings.HasPrefix(path, "@") {
-		return path
-	}
-
-	trimmedPath := strings.TrimSuffix(strings.TrimPrefix(path, "@"), ".real")
-	// If the .real path exists, we return that.
-	if _, err := os.Stat(trimmedPath + ".real"); err == nil {
-		return "@" + trimmedPath + ".real"
-	}
-	// If the .real path does not exists (or cannot be read) we return the non-.real path.
-	return "@" + trimmedPath
+	return string(ldconfigPath(path).normalize())
 }

internal/config/cli_test.go

@@ -33,7 +33,7 @@ func TestNormalizeLDConfigPath(t *testing.T) {
 
 	testCases := []struct {
 		description string
-		ldconfig    string
+		ldconfig    ldconfigPath
 		expected    string
 	}{
 		{
@@ -51,12 +51,12 @@ func TestNormalizeLDConfigPath(t *testing.T) {
 		},
 		{
 			description: "host .real file exists is returned",
-			ldconfig:    "@" + filepath.Join(testDir, "exists.real"),
+			ldconfig:    ldconfigPath("@" + filepath.Join(testDir, "exists.real")),
 			expected:    "@" + filepath.Join(testDir, "exists.real"),
 		},
 		{
 			description: "host resolves .real file",
-			ldconfig:    "@" + filepath.Join(testDir, "exists"),
+			ldconfig:    ldconfigPath("@" + filepath.Join(testDir, "exists")),
 			expected:    "@" + filepath.Join(testDir, "exists.real"),
 		},
 		{

internal/config/config.go

@@ -18,6 +18,7 @@ package config
 
 import (
 	"bufio"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -51,6 +52,8 @@ var (
 	NVIDIAContainerToolkitExecutable = "nvidia-container-toolkit"
 )
 
+var errInvalidConfig = errors.New("invalid config value")
+
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
@@ -127,8 +130,20 @@ func GetDefault() (*Config, error) {
 	return &d, nil
 }
 
-func getLdConfigPath() string {
-	return NormalizeLDConfigPath("@/sbin/ldconfig")
+// assertValid checks for a valid config.
+func (c *Config) assertValid() error {
+	err := c.NVIDIAContainerCLIConfig.Ldconfig.assertValid(c.Features.AllowLDConfigFromContainer.IsEnabled())
+	if err != nil {
+		return errors.Join(err, errInvalidConfig)
+	}
+	return nil
+}
+
+// getLdConfigPath allows us to override this function for testing.
+var getLdConfigPath = getLdConfigPathStub
+
+func getLdConfigPathStub() ldconfigPath {
+	return ldconfigPath("@/sbin/ldconfig").normalize()
 }
 
 func getUserGroup() string {

internal/config/config_test.go

@@ -44,23 +44,21 @@ func TestGetConfigWithCustomConfig(t *testing.T) {
 
 func TestGetConfig(t *testing.T) {
 	testCases := []struct {
-		description     string
-		contents        []string
-		expectedError   error
-		inspectLdconfig bool
-		distIdsLike     []string
-		expectedConfig  *Config
+		description    string
+		contents       []string
+		expectedError  error
+		distIdsLike    []string
+		expectedConfig *Config
 	}{
 		{
-			description:     "empty config is default",
-			inspectLdconfig: true,
+			description: "empty config is default",
 			expectedConfig: &Config{
 				AcceptEnvvarUnprivileged:    true,
 				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root:      "",
 					LoadKmods: true,
-					Ldconfig:  "WAS_CHECKED",
+					Ldconfig:  "@/test/ld/config/path",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/dev/null",
@@ -93,7 +91,7 @@ func TestGetConfig(t *testing.T) {
 				"supported-driver-capabilities = \"compute,utility\"",
 				"nvidia-container-cli.root = \"/bar/baz\"",
 				"nvidia-container-cli.load-kmods = false",
-				"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
+				"nvidia-container-cli.ldconfig = \"@/foo/bar/ldconfig\"",
 				"nvidia-container-cli.user = \"foo:bar\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
@@ -113,7 +111,7 @@ func TestGetConfig(t *testing.T) {
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root:      "/bar/baz",
 					LoadKmods: false,
-					Ldconfig:  "/foo/bar/ldconfig",
+					Ldconfig:  "@/foo/bar/ldconfig",
 					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
@@ -146,6 +144,53 @@ func TestGetConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			description: "feature allows ldconfig to be overridden",
+			contents: []string{
+				"[nvidia-container-cli]",
+				"ldconfig = \"/foo/bar/ldconfig\"",
+				"[features]",
+				"allow-ldconfig-from-container = true",
+			},
+			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Ldconfig:  "/foo/bar/ldconfig",
+					LoadKmods: true,
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+						CDI: cdiModeConfig{
+							DefaultKind: "nvidia.com/gpu",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+							},
+							SpecDirs: []string{
+								"/etc/cdi",
+								"/var/run/cdi",
+							},
+						},
+					},
+				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+				Features: features{
+					AllowLDConfigFromContainer: ptr(feature(true)),
+				},
+			},
+		},
 		{
 			description: "config options set in section",
 			contents: []string{
@@ -154,7 +199,7 @@ func TestGetConfig(t *testing.T) {
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
 				"load-kmods = false",
-				"ldconfig = \"/foo/bar/ldconfig\"",
+				"ldconfig = \"@/foo/bar/ldconfig\"",
 				"user = \"foo:bar\"",
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
@@ -179,7 +224,7 @@ func TestGetConfig(t *testing.T) {
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root:      "/bar/baz",
 					LoadKmods: false,
-					Ldconfig:  "/foo/bar/ldconfig",
+					Ldconfig:  "@/foo/bar/ldconfig",
 					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
@@ -213,16 +258,15 @@ func TestGetConfig(t *testing.T) {
 			},
 		},
 		{
-			description:     "suse config",
-			distIdsLike:     []string{"suse", "opensuse"},
-			inspectLdconfig: true,
+			description: "suse config",
+			distIdsLike: []string{"suse", "opensuse"},
 			expectedConfig: &Config{
 				AcceptEnvvarUnprivileged:    true,
 				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root:      "",
 					LoadKmods: true,
-					Ldconfig:  "WAS_CHECKED",
+					Ldconfig:  "@/test/ld/config/path",
 					User:      "root:video",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
@@ -250,9 +294,8 @@ func TestGetConfig(t *testing.T) {
 			},
 		},
 		{
-			description:     "suse config overrides user",
-			distIdsLike:     []string{"suse", "opensuse"},
-			inspectLdconfig: true,
+			description: "suse config overrides user",
+			distIdsLike: []string{"suse", "opensuse"},
 			contents: []string{
 				"nvidia-container-cli.user = \"foo:bar\"",
 			},
@@ -262,7 +305,7 @@ func TestGetConfig(t *testing.T) {
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root:      "",
 					LoadKmods: true,
-					Ldconfig:  "WAS_CHECKED",
+					Ldconfig:  "@/test/ld/config/path",
 					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
@@ -293,6 +336,7 @@ func TestGetConfig(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
+			defer setGetLdConfigPathForTest()()
 			defer setGetDistIDLikeForTest(tc.distIdsLike)()
 			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
 
@@ -305,21 +349,63 @@ func TestGetConfig(t *testing.T) {
 			cfg, err := tomlCfg.Config()
 			require.NoError(t, err)
 
-			// We first handle the ldconfig path since this is currently system-dependent.
-			if tc.inspectLdconfig {
-				ldconfig := cfg.NVIDIAContainerCLIConfig.Ldconfig
-				require.True(t, strings.HasPrefix(ldconfig, "@/sbin/ldconfig"))
-				remaining := strings.TrimPrefix(ldconfig, "@/sbin/ldconfig")
-				require.True(t, remaining == ".real" || remaining == "")
-
-				cfg.NVIDIAContainerCLIConfig.Ldconfig = "WAS_CHECKED"
-			}
-
 			require.EqualValues(t, tc.expectedConfig, cfg)
 		})
 	}
 }
 
+func TestAssertValid(t *testing.T) {
+	defer setGetLdConfigPathForTest()()
+
+	testCases := []struct {
+		description   string
+		config        *Config
+		expectedError error
+	}{
+		{
+			description: "default is valid",
+			config: func() *Config {
+				config, _ := GetDefault()
+				return config
+			}(),
+		},
+		{
+			description: "alternative host ldconfig path is valid",
+			config: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Ldconfig: "@/some/host/path",
+				},
+			},
+		},
+		{
+			description: "non-host path is invalid",
+			config: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Ldconfig: "/non/host/path",
+				},
+			},
+			expectedError: errInvalidConfig,
+		},
+		{
+			description: "feature flag allows non-host path",
+			config: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Ldconfig: "/non/host/path",
+				},
+				Features: features{
+					AllowLDConfigFromContainer: ptr(feature(true)),
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			require.ErrorIs(t, tc.config.assertValid(), tc.expectedError)
+		})
+	}
+}
+
 // setGetDistIDsLikeForTest overrides the distribution IDs that would normally be read from the /etc/os-release file.
 func setGetDistIDLikeForTest(ids []string) func() {
 	if ids == nil {
@@ -335,3 +421,18 @@ func setGetDistIDLikeForTest(ids []string) func() {
 		getDistIDLike = original
 	}
 }
+
+// prt returns a reference to whatever type is passed into it
+func ptr[T any](x T) *T {
+	return &x
+}
+
+func setGetLdConfigPathForTest() func() {
+	previous := getLdConfigPath
+	getLdConfigPath = func() ldconfigPath {
+		return "@/test/ld/config/path"
+	}
+	return func() {
+		getLdConfigPath = previous
+	}
+}

internal/config/features.go

@@ -16,70 +16,26 @@
 
 package config
 
-type featureName string
-
-const (
-	FeatureGDS      = featureName("gds")
-	FeatureMOFED    = featureName("mofed")
-	FeatureNVSWITCH = featureName("nvswitch")
-	FeatureGDRCopy  = featureName("gdrcopy")
-)
-
 // features specifies a set of named features.
 type features struct {
-	GDS      *feature `toml:"gds,omitempty"`
-	MOFED    *feature `toml:"mofed,omitempty"`
-	NVSWITCH *feature `toml:"nvswitch,omitempty"`
-	GDRCopy  *feature `toml:"gdrcopy,omitempty"`
+	// AllowCUDACompatLibsFromContainer allows CUDA compat libs from a container
+	// to override certain driver library mounts from the host.
+	AllowCUDACompatLibsFromContainer *feature `toml:"allow-cuda-compat-libs-from-container,omitempty"`
+	// AllowLDConfigFromContainer allows non-host ldconfig paths to be used.
+	// If this feature flag is not set to 'true' only host-rooted config paths
+	// (i.e. paths starting with an '@' are considered valid)
+	AllowLDConfigFromContainer *feature `toml:"allow-ldconfig-from-container,omitempty"`
+	// DisableImexChannelCreation ensures that the implicit creation of
+	// requested IMEX channels is skipped when invoking the nvidia-container-cli.
+	DisableImexChannelCreation *feature `toml:"disable-imex-channel-creation,omitempty"`
 }
 
 type feature bool
 
-// IsEnabled checks whether a specified named feature is enabled.
-// An optional list of environments to check for feature-specific environment
-// variables can also be supplied.
-func (fs features) IsEnabled(n featureName, in ...getenver) bool {
-	featureEnvvars := map[featureName]string{
-		FeatureGDS:      "NVIDIA_GDS",
-		FeatureMOFED:    "NVIDIA_MOFED",
-		FeatureNVSWITCH: "NVIDIA_NVSWITCH",
-		FeatureGDRCopy:  "NVIDIA_GDRCOPY",
-	}
-
-	envvar := featureEnvvars[n]
-	switch n {
-	case FeatureGDS:
-		return fs.GDS.isEnabled(envvar, in...)
-	case FeatureMOFED:
-		return fs.MOFED.isEnabled(envvar, in...)
-	case FeatureNVSWITCH:
-		return fs.NVSWITCH.isEnabled(envvar, in...)
-	case FeatureGDRCopy:
-		return fs.GDRCopy.isEnabled(envvar, in...)
-	default:
-		return false
-	}
-}
-
-// isEnabled checks whether a feature is enabled.
-// If the enabled value is explicitly set, this is returned, otherwise the
-// associated envvar is checked in the specified getenver for the string "enabled"
-// A CUDA container / image can be passed here.
-func (f *feature) isEnabled(envvar string, ins ...getenver) bool {
+// IsEnabled checks whether a feature is explicitly enabled.
+func (f *feature) IsEnabled() bool {
 	if f != nil {
 		return bool(*f)
 	}
-	if envvar == "" {
-		return false
-	}
-	for _, in := range ins {
-		if in.Getenv(envvar) == "enabled" {
-			return true
-		}
-	}
 	return false
 }
-
-type getenver interface {
-	Getenv(string) string
-}

internal/config/hook.go

@@ -24,13 +24,3 @@ type RuntimeHookConfig struct {
 	// SkipModeDetection disables the mode check for the runtime hook.
 	SkipModeDetection bool `toml:"skip-mode-detection"`
 }
-
-// GetDefaultRuntimeHookConfig defines the default values for the config
-func GetDefaultRuntimeHookConfig() (*RuntimeHookConfig, error) {
-	cfg, err := GetDefault()
-	if err != nil {
-		return nil, err
-	}
-
-	return &cfg.NVIDIAContainerRuntimeHookConfig, nil
-}

internal/config/image/builder.go

@@ -47,7 +47,7 @@ func New(opt ...Option) (CUDA, error) {
 // build creates a CUDA image from the builder.
 func (b builder) build() (CUDA, error) {
 	if b.disableRequire {
-		b.env[envNVDisableRequire] = "true"
+		b.env[EnvVarNvidiaDisableRequire] = "true"
 	}
 
 	c := CUDA{

internal/config/image/cuda_image.go

@@ -28,12 +28,10 @@ import (
 )
 
 const (
-	envCUDAVersion          = "CUDA_VERSION"
-	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
-	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
-	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
-	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
+	DeviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
+
+	volumeMountDevicePrefixCDI  = "cdi/"
+	volumeMountDevicePrefixImex = "imex/"
 )
 
 // CUDA represents a CUDA image that can be used for GPU computing. This wraps
@@ -80,8 +78,8 @@ func (i CUDA) HasEnvvar(key string) bool {
 // image is considered legacy if it has a CUDA_VERSION environment variable defined
 // and no NVIDIA_REQUIRE_CUDA environment variable defined.
 func (i CUDA) IsLegacy() bool {
-	legacyCudaVersion := i.env[envCUDAVersion]
-	cudaRequire := i.env[envNVRequireCUDA]
+	legacyCudaVersion := i.env[EnvVarCudaVersion]
+	cudaRequire := i.env[EnvVarNvidiaRequireCuda]
 	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
 }
 
@@ -95,7 +93,7 @@ func (i CUDA) GetRequirements() ([]string, error) {
 	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
 	var requirements []string
 	for name, value := range i.env {
-		if strings.HasPrefix(name, envNVRequirePrefix) && !strings.HasPrefix(name, envNVRequireJetpack) {
+		if strings.HasPrefix(name, NvidiaRequirePrefix) && !strings.HasPrefix(name, EnvVarNvidiaRequireJetpack) {
 			requirements = append(requirements, value)
 		}
 	}
@@ -113,7 +111,7 @@ func (i CUDA) GetRequirements() ([]string, error) {
 // HasDisableRequire checks for the value of the NVIDIA_DISABLE_REQUIRE. If set
 // to a valid (true) boolean value this can be used to disable the requirement checks
 func (i CUDA) HasDisableRequire() bool {
-	if disable, exists := i.env[envNVDisableRequire]; exists {
+	if disable, exists := i.env[EnvVarNvidiaDisableRequire]; exists {
 		// i.logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", disable)
 		d, _ := strconv.ParseBool(disable)
 		return d
@@ -157,7 +155,7 @@ func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
 
 // GetDriverCapabilities returns the requested driver capabilities.
 func (i CUDA) GetDriverCapabilities() DriverCapabilities {
-	env := i.env[envNVDriverCapabilities]
+	env := i.env[EnvVarNvidiaDriverCapabilities]
 
 	capabilities := make(DriverCapabilities)
 	for _, c := range strings.Split(env, ",") {
@@ -168,7 +166,7 @@ func (i CUDA) GetDriverCapabilities() DriverCapabilities {
 }
 
 func (i CUDA) legacyVersion() (string, error) {
-	cudaVersion := i.env[envCUDAVersion]
+	cudaVersion := i.env[EnvVarCudaVersion]
 	majorMinor, err := parseMajorMinorVersion(cudaVersion)
 	if err != nil {
 		return "", fmt.Errorf("invalid CUDA version %v: %v", cudaVersion, err)
@@ -202,7 +200,7 @@ func parseMajorMinorVersion(version string) (string, error) {
 // OnlyFullyQualifiedCDIDevices returns true if all devices requested in the image are requested as CDI devices/
 func (i CUDA) OnlyFullyQualifiedCDIDevices() bool {
 	var hasCDIdevice bool
-	for _, device := range i.DevicesFromEnvvars("NVIDIA_VISIBLE_DEVICES").List() {
+	for _, device := range i.VisibleDevicesFromEnvVar() {
 		if !parser.IsQualifiedName(device) {
 			return false
 		}
@@ -218,14 +216,31 @@ func (i CUDA) OnlyFullyQualifiedCDIDevices() bool {
 	return hasCDIdevice
 }
 
-const (
-	deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
-)
+// VisibleDevicesFromEnvVar returns the set of visible devices requested through
+// the NVIDIA_VISIBLE_DEVICES environment variable.
+func (i CUDA) VisibleDevicesFromEnvVar() []string {
+	return i.DevicesFromEnvvars(EnvVarNvidiaVisibleDevices).List()
+}
+
+// VisibleDevicesFromMounts returns the set of visible devices requested as mounts.
+func (i CUDA) VisibleDevicesFromMounts() []string {
+	var devices []string
+	for _, device := range i.DevicesFromMounts() {
+		switch {
+		case strings.HasPrefix(device, volumeMountDevicePrefixCDI):
+			continue
+		case strings.HasPrefix(device, volumeMountDevicePrefixImex):
+			continue
+		}
+		devices = append(devices, device)
+	}
+	return devices
+}
 
 // DevicesFromMounts returns a list of device specified as mounts.
 // TODO: This should be merged with getDevicesFromMounts used in the NVIDIA Container Runtime
 func (i CUDA) DevicesFromMounts() []string {
-	root := filepath.Clean(deviceListAsVolumeMountsRoot)
+	root := filepath.Clean(DeviceListAsVolumeMountsRoot)
 	seen := make(map[string]bool)
 	var devices []string
 	for _, m := range i.mounts {
@@ -260,10 +275,10 @@ func (i CUDA) DevicesFromMounts() []string {
 func (i CUDA) CDIDevicesFromMounts() []string {
 	var devices []string
 	for _, mountDevice := range i.DevicesFromMounts() {
-		if !strings.HasPrefix(mountDevice, "cdi/") {
+		if !strings.HasPrefix(mountDevice, volumeMountDevicePrefixCDI) {
 			continue
 		}
-		parts := strings.SplitN(strings.TrimPrefix(mountDevice, "cdi/"), "/", 3)
+		parts := strings.SplitN(strings.TrimPrefix(mountDevice, volumeMountDevicePrefixCDI), "/", 3)
 		if len(parts) != 3 {
 			continue
 		}
@@ -274,3 +289,24 @@ func (i CUDA) CDIDevicesFromMounts() []string {
 	}
 	return devices
 }
+
+// ImexChannelsFromEnvVar returns the list of IMEX channels requested for the image.
+func (i CUDA) ImexChannelsFromEnvVar() []string {
+	imexChannels := i.DevicesFromEnvvars(EnvVarNvidiaImexChannels).List()
+	if len(imexChannels) == 1 && imexChannels[0] == "all" {
+		return nil
+	}
+	return imexChannels
+}
+
+// ImexChannelsFromMounts returns the list of IMEX channels requested for the image.
+func (i CUDA) ImexChannelsFromMounts() []string {
+	var channels []string
+	for _, mountDevice := range i.DevicesFromMounts() {
+		if !strings.HasPrefix(mountDevice, volumeMountDevicePrefixImex) {
+			continue
+		}
+		channels = append(channels, strings.TrimPrefix(mountDevice, volumeMountDevicePrefixImex))
+	}
+	return channels
+}

internal/config/image/cuda_image_test.go

@@ -17,8 +17,10 @@
 package image
 
 import (
+	"path/filepath"
 	"testing"
 
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/require"
 )
 
@@ -130,3 +132,116 @@ func TestGetRequirements(t *testing.T) {
 
 	}
 }
+
+func TestGetVisibleDevicesFromMounts(t *testing.T) {
+	var tests = []struct {
+		description     string
+		mounts          []specs.Mount
+		expectedDevices []string
+	}{
+		{
+			description:     "No mounts",
+			mounts:          nil,
+			expectedDevices: nil,
+		},
+		{
+			description: "Host path is not /dev/null",
+			mounts: []specs.Mount{
+				{
+					Source:      "/not/dev/null",
+					Destination: filepath.Join(DeviceListAsVolumeMountsRoot, "GPU0"),
+				},
+			},
+			expectedDevices: nil,
+		},
+		{
+			description: "Container path is not prefixed by 'root'",
+			mounts: []specs.Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join("/other/prefix", "GPU0"),
+				},
+			},
+			expectedDevices: nil,
+		},
+		{
+			description: "Container path is only 'root'",
+			mounts: []specs.Mount{
+				{
+					Source:      "/dev/null",
+					Destination: DeviceListAsVolumeMountsRoot,
+				},
+			},
+			expectedDevices: nil,
+		},
+		{
+			description:     "Discover 2 devices",
+			mounts:          makeTestMounts("GPU0", "GPU1"),
+			expectedDevices: []string{"GPU0", "GPU1"},
+		},
+		{
+			description:     "Discover 2 devices with slashes in the name",
+			mounts:          makeTestMounts("GPU0-MIG0/0/1", "GPU1-MIG0/0/1"),
+			expectedDevices: []string{"GPU0-MIG0/0/1", "GPU1-MIG0/0/1"},
+		},
+		{
+			description:     "cdi devices are ignored",
+			mounts:          makeTestMounts("GPU0", "cdi/nvidia.com/gpu=all", "GPU1"),
+			expectedDevices: []string{"GPU0", "GPU1"},
+		},
+		{
+			description:     "imex devices are ignored",
+			mounts:          makeTestMounts("GPU0", "imex/0", "GPU1"),
+			expectedDevices: []string{"GPU0", "GPU1"},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			image, _ := New(WithMounts(tc.mounts))
+			require.Equal(t, tc.expectedDevices, image.VisibleDevicesFromMounts())
+		})
+	}
+}
+
+func TestImexChannelsFromEnvVar(t *testing.T) {
+	testCases := []struct {
+		description string
+		env         []string
+		expected    []string
+	}{
+		{
+			description: "no imex channels specified",
+		},
+		{
+			description: "imex channel specified",
+			env: []string{
+				"NVIDIA_IMEX_CHANNELS=3,4",
+			},
+			expected: []string{"3", "4"},
+		},
+	}
+
+	for _, tc := range testCases {
+		for id, baseEnvvars := range map[string][]string{"": nil, "legacy": {"CUDA_VERSION=1.2.3"}} {
+			t.Run(tc.description+id, func(t *testing.T) {
+				i, err := NewCUDAImageFromEnv(append(baseEnvvars, tc.env...))
+				require.NoError(t, err)
+
+				channels := i.ImexChannelsFromEnvVar()
+				require.EqualValues(t, tc.expected, channels)
+			})
+		}
+	}
+}
+
+func makeTestMounts(paths ...string) []specs.Mount {
+	var mounts []specs.Mount
+	for _, path := range paths {
+		mount := specs.Mount{
+			Source:      "/dev/null",
+			Destination: filepath.Join(DeviceListAsVolumeMountsRoot, path),
+		}
+		mounts = append(mounts, mount)
+	}
+	return mounts
+}

internal/config/image/envvars.go

@@ -0,0 +1,31 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+const (
+	EnvVarCudaVersion              = "CUDA_VERSION"
+	EnvVarNvidiaDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	EnvVarNvidiaDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
+	EnvVarNvidiaImexChannels       = "NVIDIA_IMEX_CHANNELS"
+	EnvVarNvidiaMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
+	EnvVarNvidiaMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
+	EnvVarNvidiaRequireCuda        = NvidiaRequirePrefix + "CUDA"
+	EnvVarNvidiaRequireJetpack     = NvidiaRequirePrefix + "JETPACK"
+	EnvVarNvidiaVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
+
+	NvidiaRequirePrefix = "NVIDIA_REQUIRE_"
+)

internal/config/runtime.go

@@ -45,13 +45,3 @@ type cdiModeConfig struct {
 type csvModeConfig struct {
 	MountSpecPath string `toml:"mount-spec-path"`
 }
-
-// GetDefaultRuntimeConfig defines the default values for the config
-func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
-	cfg, err := GetDefault()
-	if err != nil {
-		return nil, err
-	}
-
-	return &cfg.NVIDIAContainerRuntimeConfig, nil
-}

internal/config/toml.go

@@ -108,6 +108,19 @@ func loadConfigTomlFrom(reader io.Reader) (*Toml, error) {
 
 // Config returns the typed config associated with the toml tree.
 func (t *Toml) Config() (*Config, error) {
+	cfg, err := t.configNoOverrides()
+	if err != nil {
+		return nil, err
+	}
+	if err := cfg.assertValid(); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// configNoOverrides returns the typed config associated with the toml tree.
+// This config does not include feature-specific overrides.
+func (t *Toml) configNoOverrides() (*Config, error) {
 	cfg, err := GetDefault()
 	if err != nil {
 		return nil, err
@@ -170,11 +183,22 @@ func (t *Toml) Get(key string) interface{} {
 	return (*toml.Tree)(t).Get(key)
 }
 
+// GetDefault returns the value for the specified key and falls back to the default value if the Get call fails
+func (t *Toml) GetDefault(key string, def interface{}) interface{} {
+	return (*toml.Tree)(t).GetDefault(key, def)
+}
+
 // Set sets the specified key to the specified value in the TOML config.
 func (t *Toml) Set(key string, value interface{}) {
 	(*toml.Tree)(t).Set(key, value)
 }
 
+// WriteTo encode the Tree as Toml and writes it to the writer w.
+// Returns the number of bytes written in case of success, or an error if anything happened.
+func (t *Toml) WriteTo(w io.Writer) (int64, error) {
+	return (*toml.Tree)(t).WriteTo(w)
+}
+
 // commentDefaults applies the required comments for default values to the Toml.
 func (t *Toml) commentDefaults() *Toml {
 	asToml := (*toml.Tree)(t)

internal/config/toml_test.go

@@ -198,9 +198,12 @@ func TestTomlContents(t *testing.T) {
 }
 
 func TestConfigFromToml(t *testing.T) {
+	defer setGetLdConfigPathForTest()()
+
 	testCases := []struct {
 		description    string
 		contents       map[string]interface{}
+		expectedError  error
 		expectedConfig *Config
 	}{
 		{
@@ -226,13 +229,39 @@ func TestConfigFromToml(t *testing.T) {
 				return c
 			}(),
 		},
+		{
+			description: "invalid ldconfig value raises error",
+			contents: map[string]interface{}{
+				"nvidia-container-cli": map[string]interface{}{
+					"ldconfig": "/some/ldconfig/path",
+				},
+			},
+			expectedError: errInvalidConfig,
+		},
+		{
+			description: "feature allows ldconfig override",
+			contents: map[string]interface{}{
+				"nvidia-container-cli": map[string]interface{}{
+					"ldconfig": "/some/ldconfig/path",
+				},
+				"features": map[string]interface{}{
+					"allow-ldconfig-from-container": true,
+				},
+			},
+			expectedConfig: func() *Config {
+				c, _ := GetDefault()
+				c.NVIDIAContainerCLIConfig.Ldconfig = "/some/ldconfig/path"
+				c.Features.AllowLDConfigFromContainer = ptr(feature(true))
+				return c
+			}(),
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			tomlCfg := fromMap(tc.contents)
 			config, err := tomlCfg.Config()
-			require.NoError(t, err)
+			require.ErrorIs(t, err, tc.expectedError)
 			require.EqualValues(t, tc.expectedConfig, config)
 		})
 	}

internal/discover/cache.go

@@ -0,0 +1,80 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "sync"
+
+type cache struct {
+	d Discover
+
+	sync.Mutex
+	devices []Device
+	hooks   []Hook
+	mounts  []Mount
+}
+
+var _ Discover = (*cache)(nil)
+
+// WithCache decorates the specified disoverer with a cache.
+func WithCache(d Discover) Discover {
+	if d == nil {
+		return None{}
+	}
+	return &cache{d: d}
+}
+
+func (c *cache) Devices() ([]Device, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.devices == nil {
+		devices, err := c.d.Devices()
+		if err != nil {
+			return nil, err
+		}
+		c.devices = devices
+	}
+	return c.devices, nil
+}
+
+func (c *cache) Hooks() ([]Hook, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.hooks == nil {
+		hooks, err := c.d.Hooks()
+		if err != nil {
+			return nil, err
+		}
+		c.hooks = hooks
+	}
+	return c.hooks, nil
+}
+
+func (c *cache) Mounts() ([]Mount, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.mounts == nil {
+		mounts, err := c.d.Mounts()
+		if err != nil {
+			return nil, err
+		}
+		c.mounts = mounts
+	}
+	return c.mounts, nil
+}

internal/discover/first-valid.go

@@ -0,0 +1,72 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "errors"
+
+type firstOf []Discover
+
+// FirstValid returns a discoverer that returns the first non-error result from a list of discoverers.
+func FirstValid(discoverers ...Discover) Discover {
+	var f firstOf
+	for _, d := range discoverers {
+		if d == nil {
+			continue
+		}
+		f = append(f, d)
+	}
+	return f
+}
+
+func (f firstOf) Devices() ([]Device, error) {
+	var errs error
+	for _, d := range f {
+		devices, err := d.Devices()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return devices, nil
+	}
+	return nil, errs
+}
+
+func (f firstOf) Hooks() ([]Hook, error) {
+	var errs error
+	for _, d := range f {
+		hooks, err := d.Hooks()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return hooks, nil
+	}
+	return nil, errs
+}
+
+func (f firstOf) Mounts() ([]Mount, error) {
+	var errs error
+	for _, d := range f {
+		mounts, err := d.Mounts()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return mounts, nil
+	}
+	return nil, nil
+}

internal/discover/graphics.go

@@ -180,10 +180,10 @@ func (d graphicsDriverLibraries) Hooks() ([]Hook, error) {
 		switch {
 		case d.isDriverLibrary(filename, "libnvidia-allocator.so"):
 			// gbm/nvidia-drm_gbm.so is a symlink to ../libnvidia-allocator.so.1 which
-			// in turn symlinks to libnvidia-allocator.so.RM_VERSION and is created
-			// when ldconfig is run in the container.
-			// create libnvidia-allocate.so.1 -> libnvidia-allocate.so.RM_VERSION symlink
-			links = append(links, fmt.Sprintf("%s::%s", filename, filepath.Join(dir, "libnvidia-allocator.so.1")))
+			// in turn symlinks to libnvidia-allocator.so.RM_VERSION.
+			// The libnvidia-allocator.so.1 -> libnvidia-allocator.so.RM_VERSION symlink
+			// is created when ldconfig is run against the container and there
+			// is no explicit need to create it.
 			// create gbm/nvidia-drm_gbm.so -> ../libnvidia-allocate.so.1 symlink
 			linkPath := filepath.Join(dir, "gbm", "nvidia-drm_gbm.so")
 			links = append(links, fmt.Sprintf("%s::%s", "../libnvidia-allocator.so.1", linkPath))

internal/discover/graphics_test.go

@@ -68,7 +68,6 @@ func TestGraphicsLibrariesDiscoverer(t *testing.T) {
 					Lifecycle: "createContainer",
 					Path:      "/usr/bin/nvidia-cdi-hook",
 					Args: []string{"nvidia-cdi-hook", "create-symlinks",
-						"--link", "libnvidia-allocator.so.123.45.67::/usr/lib64/libnvidia-allocator.so.1",
 						"--link", "../libnvidia-allocator.so.1::/usr/lib64/gbm/nvidia-drm_gbm.so",
 					},
 				},
@@ -126,7 +125,6 @@ func TestGraphicsLibrariesDiscoverer(t *testing.T) {
 					Lifecycle: "createContainer",
 					Path:      "/usr/bin/nvidia-cdi-hook",
 					Args: []string{"nvidia-cdi-hook", "create-symlinks",
-						"--link", "libnvidia-allocator.so.123.45.67::/usr/lib64/libnvidia-allocator.so.1",
 						"--link", "../libnvidia-allocator.so.1::/usr/lib64/gbm/nvidia-drm_gbm.so",
 						"--link", "libnvidia-vulkan-producer.so.123.45.67::/usr/lib64/libnvidia-vulkan-producer.so",
 					},

internal/discover/none.go

@@ -24,15 +24,15 @@ var _ Discover = (*None)(nil)
 
 // Devices returns an empty list of devices
 func (e None) Devices() ([]Device, error) {
-	return []Device{}, nil
+	return nil, nil
 }
 
 // Mounts returns an empty list of mounts
 func (e None) Mounts() ([]Mount, error) {
-	return []Mount{}, nil
+	return nil, nil
 }
 
 // Hooks returns an empty list of hooks
 func (e None) Hooks() ([]Hook, error) {
-	return []Hook{}, nil
+	return nil, nil
 }

internal/discover/symlinks.go

@@ -0,0 +1,108 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"path/filepath"
+)
+
+type additionalSymlinks struct {
+	Discover
+	version           string
+	nvidiaCDIHookPath string
+}
+
+// WithDriverDotSoSymlinks decorates the provided discoverer.
+// A hook is added that checks for specific driver symlinks that need to be created.
+func WithDriverDotSoSymlinks(mounts Discover, version string, nvidiaCDIHookPath string) Discover {
+	if version == "" {
+		version = "*.*"
+	}
+	return &additionalSymlinks{
+		Discover:          mounts,
+		nvidiaCDIHookPath: nvidiaCDIHookPath,
+		version:           version,
+	}
+}
+
+// Hooks returns a hook to create the additional symlinks based on the mounts.
+func (d *additionalSymlinks) Hooks() ([]Hook, error) {
+	mounts, err := d.Discover.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get library mounts: %v", err)
+	}
+	hooks, err := d.Discover.Hooks()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get hooks: %v", err)
+	}
+
+	var links []string
+	processedPaths := make(map[string]bool)
+	processedLinks := make(map[string]bool)
+	for _, mount := range mounts {
+		if processedPaths[mount.Path] {
+			continue
+		}
+		processedPaths[mount.Path] = true
+
+		for _, link := range d.getLinksForMount(mount.Path) {
+			if processedLinks[link] {
+				continue
+			}
+			processedLinks[link] = true
+			links = append(links, link)
+		}
+	}
+
+	if len(links) == 0 {
+		return hooks, nil
+	}
+
+	hook := CreateCreateSymlinkHook(d.nvidiaCDIHookPath, links).(Hook)
+	return append(hooks, hook), nil
+}
+
+// getLinksForMount maps the path to created links if any.
+func (d additionalSymlinks) getLinksForMount(path string) []string {
+	dir, filename := filepath.Split(path)
+	switch {
+	case d.isDriverLibrary("libcuda.so", filename):
+		// XXX Many applications wrongly assume that libcuda.so exists (e.g. with dlopen).
+		// create libcuda.so -> libcuda.so.1 symlink
+		link := fmt.Sprintf("%s::%s", "libcuda.so.1", filepath.Join(dir, "libcuda.so"))
+		return []string{link}
+	case d.isDriverLibrary("libGLX_nvidia.so", filename):
+		// XXX GLVND requires this symlink for indirect GLX support.
+		// create libGLX_indirect.so.0 -> libGLX_nvidia.so.VERSION symlink
+		link := fmt.Sprintf("%s::%s", filename, filepath.Join(dir, "libGLX_indirect.so.0"))
+		return []string{link}
+	case d.isDriverLibrary("libnvidia-opticalflow.so", filename):
+		// XXX Fix missing symlink for libnvidia-opticalflow.so.
+		// create libnvidia-opticalflow.so -> libnvidia-opticalflow.so.1 symlink
+		link := fmt.Sprintf("%s::%s", "libnvidia-opticalflow.so.1", filepath.Join(dir, "libnvidia-opticalflow.so"))
+		return []string{link}
+	}
+	return nil
+}
+
+// isDriverLibrary checks whether the specified filename is a specific driver library.
+func (d additionalSymlinks) isDriverLibrary(libraryName string, filename string) bool {
+	pattern := libraryName + "." + d.version
+	match, _ := filepath.Match(pattern, filename)
+	return match
+}

internal/discover/symlinks_test.go

@@ -0,0 +1,330 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWithWithDriverDotSoSymlinks(t *testing.T) {
+	testCases := []struct {
+		description          string
+		discover             Discover
+		version              string
+		expectedDevices      []Device
+		expectedDevicesError error
+		expectedHooks        []Hook
+		expectedHooksError   error
+		expectedMounts       []Mount
+		expectedMountsError  error
+	}{
+		{
+			description: "empty discoverer remains empty",
+			discover:    None{},
+		},
+		{
+			description: "non-matching discoverer remains unchanged",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					devices := []Device{
+						{
+							Path: "/dev/dev1",
+						},
+					}
+					return devices, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					hooks := []Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/path/to/a/hook",
+							Args:      []string{"hook", "arg1", "arg2"},
+						},
+					}
+					return hooks, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libnotcuda.so.1.2.3",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			expectedDevices: []Device{
+				{
+					Path: "/dev/dev1",
+				},
+			},
+			expectedHooks: []Hook{
+				{
+					Lifecycle: "prestart",
+					Path:      "/path/to/a/hook",
+					Args:      []string{"hook", "arg1", "arg2"},
+				},
+			},
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libnotcuda.so.1.2.3",
+				},
+			},
+		},
+		{
+			description: "libcuda.so.RM_VERSION is matched",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					return nil, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					return nil, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libcuda.so.1.2.3",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			version: "1.2.3",
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libcuda.so.1.2.3",
+				},
+			},
+			expectedHooks: []Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "/path/to/nvidia-cdi-hook",
+					Args:      []string{"nvidia-cdi-hook", "create-symlinks", "--link", "libcuda.so.1::/usr/lib/libcuda.so"},
+				},
+			},
+		},
+		{
+			description: "libcuda.so.RM_VERSION is matched by pattern",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					return nil, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					return nil, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libcuda.so.1.2.3",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			version: "",
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libcuda.so.1.2.3",
+				},
+			},
+			expectedHooks: []Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "/path/to/nvidia-cdi-hook",
+					Args:      []string{"nvidia-cdi-hook", "create-symlinks", "--link", "libcuda.so.1::/usr/lib/libcuda.so"},
+				},
+			},
+		},
+		{
+			description: "beta libcuda.so.RM_VERSION is matched",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					return nil, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					return nil, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libcuda.so.1.2",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libcuda.so.1.2",
+				},
+			},
+			expectedHooks: []Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "/path/to/nvidia-cdi-hook",
+					Args:      []string{"nvidia-cdi-hook", "create-symlinks", "--link", "libcuda.so.1::/usr/lib/libcuda.so"},
+				},
+			},
+		},
+		{
+			description: "non-matching libcuda.so.RM_VERSION is ignored",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					return nil, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					return nil, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libcuda.so.1.2.3",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			version: "4.5.6",
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libcuda.so.1.2.3",
+				},
+			},
+		},
+		{
+			description: "hooks are extended",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					return nil, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					hooks := []Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/path/to/a/hook",
+							Args:      []string{"hook", "arg1", "arg2"},
+						},
+					}
+					return hooks, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libcuda.so.1.2.3",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			version: "1.2.3",
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libcuda.so.1.2.3",
+				},
+			},
+			expectedHooks: []Hook{
+				{
+					Lifecycle: "prestart",
+					Path:      "/path/to/a/hook",
+					Args:      []string{"hook", "arg1", "arg2"},
+				},
+				{
+					Lifecycle: "createContainer",
+					Path:      "/path/to/nvidia-cdi-hook",
+					Args:      []string{"nvidia-cdi-hook", "create-symlinks", "--link", "libcuda.so.1::/usr/lib/libcuda.so"},
+				},
+			},
+		},
+		{
+			description: "all driver so symlinks are matched",
+			discover: &DiscoverMock{
+				DevicesFunc: func() ([]Device, error) {
+					return nil, nil
+				},
+				HooksFunc: func() ([]Hook, error) {
+					return nil, nil
+				},
+				MountsFunc: func() ([]Mount, error) {
+					mounts := []Mount{
+						{
+							Path: "/usr/lib/libcuda.so.1.2.3",
+						},
+						{
+							Path: "/usr/lib/libGLX_nvidia.so.1.2.3",
+						},
+						{
+							Path: "/usr/lib/libnvidia-opticalflow.so.1.2.3",
+						},
+						{
+							Path: "/usr/lib/libanother.so.1.2.3",
+						},
+					}
+					return mounts, nil
+				},
+			},
+			expectedMounts: []Mount{
+				{
+					Path: "/usr/lib/libcuda.so.1.2.3",
+				},
+				{
+					Path: "/usr/lib/libGLX_nvidia.so.1.2.3",
+				},
+				{
+					Path: "/usr/lib/libnvidia-opticalflow.so.1.2.3",
+				},
+				{
+					Path: "/usr/lib/libanother.so.1.2.3",
+				},
+			},
+			expectedHooks: []Hook{
+				{
+					Lifecycle: "createContainer",
+					Path:      "/path/to/nvidia-cdi-hook",
+					Args: []string{
+						"nvidia-cdi-hook", "create-symlinks",
+						"--link", "libcuda.so.1::/usr/lib/libcuda.so",
+						"--link", "libGLX_nvidia.so.1.2.3::/usr/lib/libGLX_indirect.so.0",
+						"--link", "libnvidia-opticalflow.so.1::/usr/lib/libnvidia-opticalflow.so",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			d := WithDriverDotSoSymlinks(
+				tc.discover,
+				tc.version,
+				"/path/to/nvidia-cdi-hook",
+			)
+
+			devices, err := d.Devices()
+			require.ErrorIs(t, err, tc.expectedDevicesError)
+			require.EqualValues(t, tc.expectedDevices, devices)
+
+			hooks, err := d.Hooks()
+			require.ErrorIs(t, err, tc.expectedHooksError)
+			require.EqualValues(t, tc.expectedHooks, hooks)
+
+			mounts, err := d.Mounts()
+			require.ErrorIs(t, err, tc.expectedMountsError)
+			require.EqualValues(t, tc.expectedMounts, mounts)
+		})
+	}
+}

internal/ldcache/ldcache.go

@@ -22,15 +22,12 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"syscall"
 	"unsafe"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 )
 
 const ldcachePath = "/etc/ld.so.cache"
@@ -82,10 +79,9 @@ type entry2 struct {
 
 // LDCache represents the interface for performing lookups into the LDCache
 //
-//go:generate moq -out ldcache_mock.go . LDCache
+//go:generate moq -rm -out ldcache_mock.go . LDCache
 type LDCache interface {
 	List() ([]string, []string)
-	Lookup(...string) ([]string, []string)
 }
 
 type ldcache struct {
@@ -105,14 +101,7 @@ func New(logger logger.Interface, root string) (LDCache, error) {
 
 	logger.Debugf("Opening ld.conf at %v", path)
 	f, err := os.Open(path)
-	if os.IsNotExist(err) {
-		logger.Warningf("Could not find ld.so.cache at %v; creating empty cache", path)
-		e := &empty{
-			logger: logger,
-			path:   path,
-		}
-		return e, nil
-	} else if err != nil {
+	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
@@ -196,7 +185,7 @@ type entry struct {
 }
 
 // getEntries returns the entires of the ldcache in a go-friendly struct.
-func (c *ldcache) getEntries(selected func(string) bool) []entry {
+func (c *ldcache) getEntries() []entry {
 	var entries []entry
 	for _, e := range c.entries {
 		bits := 0
@@ -223,9 +212,6 @@ func (c *ldcache) getEntries(selected func(string) bool) []entry {
 			c.logger.Debugf("Skipping invalid lib")
 			continue
 		}
-		if !selected(lib) {
-			continue
-		}
 		value := bytesToString(c.libs[e.Value:])
 		if value == "" {
 			c.logger.Debugf("Skipping invalid value for lib %v", lib)
@@ -236,51 +222,19 @@ func (c *ldcache) getEntries(selected func(string) bool) []entry {
 			bits:    bits,
 			value:   value,
 		}
-
 		entries = append(entries, e)
 	}
-
 	return entries
 }
 
 // List creates a list of libraries in the ldcache.
 // The 32-bit and 64-bit libraries are returned separately.
 func (c *ldcache) List() ([]string, []string) {
-	all := func(s string) bool { return true }
-
-	return c.resolveSelected(all)
-}
-
-// Lookup searches the ldcache for the specified prefixes.
-// The 32-bit and 64-bit libraries matching the prefixes are returned.
-func (c *ldcache) Lookup(libPrefixes ...string) ([]string, []string) {
-	c.logger.Debugf("Looking up %v in cache", libPrefixes)
-
-	// We define a functor to check whether a given library name matches any of the prefixes
-	matchesAnyPrefix := func(s string) bool {
-		for _, p := range libPrefixes {
-			if strings.HasPrefix(s, p) {
-				return true
-			}
-		}
-		return false
-	}
-
-	return c.resolveSelected(matchesAnyPrefix)
-}
-
-// resolveSelected process the entries in the LDCach based on the supplied filter and returns the resolved paths.
-// The paths are separated by bittage.
-func (c *ldcache) resolveSelected(selected func(string) bool) ([]string, []string) {
 	paths := make(map[int][]string)
 	processed := make(map[string]bool)
 
-	for _, e := range c.getEntries(selected) {
-		path, err := c.resolve(e.value)
-		if err != nil {
-			c.logger.Debugf("Could not resolve entry: %v", err)
-			continue
-		}
+	for _, e := range c.getEntries() {
+		path := filepath.Join(c.root, e.value)
 		if processed[path] {
 			continue
 		}
@@ -291,29 +245,6 @@ func (c *ldcache) resolveSelected(selected func(string) bool) ([]string, []strin
 	return paths[32], paths[64]
 }
 
-// resolve resolves the specified ldcache entry based on the value being processed.
-// The input is the name of the entry in the cache.
-func (c *ldcache) resolve(target string) (string, error) {
-	name := filepath.Join(c.root, target)
-
-	c.logger.Debugf("checking %v", name)
-
-	link, err := symlinks.Resolve(name)
-	if err != nil {
-		return "", fmt.Errorf("failed to resolve symlink: %v", err)
-	}
-	if link == name {
-		return name, nil
-	}
-
-	// We return absolute paths for all targets
-	if !filepath.IsAbs(link) || strings.HasPrefix(link, ".") {
-		link = filepath.Join(filepath.Dir(target), link)
-	}
-
-	return c.resolve(link)
-}
-
 // bytesToString converts a byte slice to a string.
 // This assumes that the byte slice is null-terminated
 func bytesToString(value []byte) string {

internal/ldcache/ldcache_mock.go

@@ -20,9 +20,6 @@ var _ LDCache = &LDCacheMock{}
 //			ListFunc: func() ([]string, []string) {
 //				panic("mock out the List method")
 //			},
-//			LookupFunc: func(strings ...string) ([]string, []string) {
-//				panic("mock out the Lookup method")
-//			},
 //		}
 //
 //		// use mockedLDCache in code that requires LDCache
@@ -33,22 +30,13 @@ type LDCacheMock struct {
 	// ListFunc mocks the List method.
 	ListFunc func() ([]string, []string)
 
-	// LookupFunc mocks the Lookup method.
-	LookupFunc func(strings ...string) ([]string, []string)
-
 	// calls tracks calls to the methods.
 	calls struct {
 		// List holds details about calls to the List method.
 		List []struct {
 		}
-		// Lookup holds details about calls to the Lookup method.
-		Lookup []struct {
-			// Strings is the strings argument value.
-			Strings []string
-		}
 	}
-	lockList   sync.RWMutex
-	lockLookup sync.RWMutex
+	lockList sync.RWMutex
 }
 
 // List calls ListFunc.
@@ -77,35 +65,3 @@ func (mock *LDCacheMock) ListCalls() []struct {
 	mock.lockList.RUnlock()
 	return calls
 }
-
-// Lookup calls LookupFunc.
-func (mock *LDCacheMock) Lookup(strings ...string) ([]string, []string) {
-	if mock.LookupFunc == nil {
-		panic("LDCacheMock.LookupFunc: method is nil but LDCache.Lookup was just called")
-	}
-	callInfo := struct {
-		Strings []string
-	}{
-		Strings: strings,
-	}
-	mock.lockLookup.Lock()
-	mock.calls.Lookup = append(mock.calls.Lookup, callInfo)
-	mock.lockLookup.Unlock()
-	return mock.LookupFunc(strings...)
-}
-
-// LookupCalls gets all the calls that were made to Lookup.
-// Check the length with:
-//
-//	len(mockedLDCache.LookupCalls())
-func (mock *LDCacheMock) LookupCalls() []struct {
-	Strings []string
-} {
-	var calls []struct {
-		Strings []string
-	}
-	mock.lockLookup.RLock()
-	calls = mock.calls.Lookup
-	mock.lockLookup.RUnlock()
-	return calls
-}

internal/lookup/ldcache.go

@@ -0,0 +1,118 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+)
+
+type ldcacheLocator struct {
+	*builder
+	resolvesTo map[string]string
+}
+
+var _ Locator = (*ldcacheLocator)(nil)
+
+func NewLdcacheLocator(opts ...Option) Locator {
+	b := newBuilder(opts...)
+
+	cache, err := ldcache.New(b.logger, b.root)
+	if err != nil {
+		b.logger.Warningf("Failed to load ldcache: %v", err)
+		if b.isOptional {
+			return &null{}
+		}
+		return &notFound{}
+	}
+
+	chain := NewSymlinkChainLocator(WithOptional(true))
+
+	resolvesTo := make(map[string]string)
+	_, libs64 := cache.List()
+	for _, library := range libs64 {
+		if _, processed := resolvesTo[library]; processed {
+			continue
+		}
+		candidates, err := chain.Locate(library)
+		if err != nil {
+			b.logger.Errorf("error processing library %s from ldcache: %v", library, err)
+			continue
+		}
+
+		if len(candidates) == 0 {
+			resolvesTo[library] = library
+			continue
+		}
+
+		// candidates represents a symlink chain.
+		// The first element represents the start of the chain and the last
+		// element the final target.
+		target := candidates[len(candidates)-1]
+		for _, candidate := range candidates {
+			resolvesTo[candidate] = target
+		}
+	}
+
+	return &ldcacheLocator{
+		builder:    b,
+		resolvesTo: resolvesTo,
+	}
+}
+
+// Locate finds the specified libraryname.
+// If the input is a library name, the ldcache is searched otherwise the
+// provided path is resolved as a symlink.
+func (l ldcacheLocator) Locate(libname string) ([]string, error) {
+	var matcher func(string, string) bool
+
+	if filepath.IsAbs(libname) {
+		matcher = func(p string, c string) bool {
+			m, _ := filepath.Match(filepath.Join(l.root, p), c)
+			return m
+		}
+	} else {
+		matcher = func(p string, c string) bool {
+			m, _ := filepath.Match(p, filepath.Base(c))
+			return m
+		}
+	}
+
+	var matches []string
+	seen := make(map[string]bool)
+	for name, target := range l.resolvesTo {
+		if !matcher(libname, name) {
+			continue
+		}
+		if seen[target] {
+			continue
+		}
+		seen[target] = true
+		matches = append(matches, target)
+	}
+
+	slices.Sort(matches)
+
+	if len(matches) == 0 && !l.isOptional {
+		return nil, fmt.Errorf("%s: %w", libname, ErrNotFound)
+	}
+
+	return matches, nil
+}

internal/lookup/ldcache_test.go

@@ -0,0 +1,77 @@
+package lookup
+
+import (
+	"path/filepath"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestLDCacheLookup(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	testCases := []struct {
+		rootFs        string
+		inputs        []string
+		expected      string
+		expectedError error
+	}{
+		{
+			rootFs:        "rootfs-empty",
+			inputs:        []string{"libcuda.so.1", "libcuda.so.*", "libcuda.so.*.*", "libcuda.so.999.88.77"},
+			expectedError: ErrNotFound,
+		},
+		{
+			rootFs: "rootfs-1",
+			inputs: []string{
+				"libcuda.so.1",
+				"libcuda.so.*",
+				"libcuda.so.*.*",
+				"libcuda.so.999.88.77",
+				"/lib/x86_64-linux-gnu/libcuda.so.1",
+				"/lib/x86_64-linux-gnu/libcuda.so.*",
+				"/lib/x86_64-linux-gnu/libcuda.so.*.*",
+				"/lib/x86_64-linux-gnu/libcuda.so.999.88.77",
+			},
+			expected: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77",
+		},
+		{
+			rootFs: "rootfs-2",
+			inputs: []string{
+				"libcuda.so.1",
+				"libcuda.so.*",
+				"libcuda.so.*.*",
+				"libcuda.so.999.88.77",
+				"/var/lib/nvidia/lib64/libcuda.so.1",
+				"/var/lib/nvidia/lib64/libcuda.so.*",
+				"/var/lib/nvidia/lib64/libcuda.so.*.*",
+				"/var/lib/nvidia/lib64/libcuda.so.999.88.77",
+			},
+			expected: "/var/lib/nvidia/lib64/libcuda.so.999.88.77",
+		},
+	}
+
+	for _, tc := range testCases {
+		for _, input := range tc.inputs {
+			t.Run(tc.rootFs+" "+input, func(t *testing.T) {
+				rootfs := filepath.Join(moduleRoot, "testdata", "lookup", tc.rootFs)
+				l := NewLdcacheLocator(
+					WithLogger(logger),
+					WithRoot(rootfs),
+				)
+
+				candidates, err := l.Locate(input)
+				require.ErrorIs(t, err, tc.expectedError)
+				if tc.expectedError == nil {
+					require.Equal(t, []string{filepath.Join(rootfs, tc.expected)}, candidates)
+				}
+			})
+		}
+	}
+}

internal/lookup/library.go

@@ -16,20 +16,6 @@
 
 package lookup
 
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-)
-
-type ldcacheLocator struct {
-	logger logger.Interface
-	cache  ldcache.LDCache
-}
-
-var _ Locator = (*ldcacheLocator)(nil)
-
 // NewLibraryLocator creates a library locator using the specified options.
 func NewLibraryLocator(opts ...Option) Locator {
 	b := newBuilder(opts...)
@@ -63,39 +49,7 @@ func NewLibraryLocator(opts ...Option) Locator {
 
 	l := First(
 		symlinkLocator,
-		newLdcacheLocator(opts...),
+		NewLdcacheLocator(opts...),
 	)
 	return l
 }
-
-func newLdcacheLocator(opts ...Option) Locator {
-	b := newBuilder(opts...)
-
-	cache, err := ldcache.New(b.logger, b.root)
-	if err != nil {
-		// If we failed to open the LDCache, we default to a symlink locator.
-		b.logger.Warningf("Failed to load ldcache: %v", err)
-		return nil
-	}
-
-	return &ldcacheLocator{
-		logger: b.logger,
-		cache:  cache,
-	}
-}
-
-// Locate finds the specified libraryname.
-// If the input is a library name, the ldcache is searched otherwise the
-// provided path is resolved as a symlink.
-func (l ldcacheLocator) Locate(libname string) ([]string, error) {
-	paths32, paths64 := l.cache.Lookup(libname)
-	if len(paths32) > 0 {
-		l.logger.Warningf("Ignoring 32-bit libraries for %v: %v", libname, paths32)
-	}
-
-	if len(paths64) == 0 {
-		return nil, fmt.Errorf("64-bit library %v: %w", libname, ErrNotFound)
-	}
-
-	return paths64, nil
-}

internal/lookup/library_test.go

@@ -24,82 +24,8 @@ import (
 
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
 )
 
-func TestLDCacheLocator(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testDir := t.TempDir()
-	symlinkDir := filepath.Join(testDir, "/lib/symlink")
-	require.NoError(t, os.MkdirAll(symlinkDir, 0755))
-
-	versionLib := filepath.Join(symlinkDir, "libcuda.so.1.2.3")
-	soLink := filepath.Join(symlinkDir, "libcuda.so")
-	sonameLink := filepath.Join(symlinkDir, "libcuda.so.1")
-
-	_, err := os.Create(versionLib)
-	require.NoError(t, err)
-	require.NoError(t, os.Symlink(versionLib, sonameLink))
-	require.NoError(t, os.Symlink(sonameLink, soLink))
-
-	lut := newLdcacheLocator(
-		WithLogger(logger),
-		WithRoot(testDir),
-	)
-
-	testCases := []struct {
-		description   string
-		libname       string
-		ldcacheMap    map[string]string
-		expected      []string
-		expectedError error
-	}{
-		{
-			description: "lib only resolves in LDCache",
-			libname:     "libcuda.so",
-			ldcacheMap: map[string]string{
-				"libcuda.so": "/lib/from/ldcache/libcuda.so.4.5.6",
-			},
-			expected: []string{"/lib/from/ldcache/libcuda.so.4.5.6"},
-		},
-		{
-			description:   "lib only not in LDCache returns error",
-			libname:       "libnotcuda.so",
-			expectedError: ErrNotFound,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			// We override the LDCache with a mock implementation
-			l := lut.(*ldcacheLocator)
-			l.cache = &ldcache.LDCacheMock{
-				LookupFunc: func(strings ...string) ([]string, []string) {
-					var result []string
-					for _, s := range strings {
-						if v, ok := tc.ldcacheMap[s]; ok {
-							result = append(result, v)
-						}
-					}
-					return nil, result
-				},
-			}
-
-			candidates, err := lut.Locate(tc.libname)
-			require.ErrorIs(t, err, tc.expectedError)
-
-			var cleanedCandidates []string
-			for _, c := range candidates {
-				// On MacOS /var and /tmp symlink to /private/var and /private/tmp which is included in the resolved path.
-				cleanedCandidates = append(cleanedCandidates, strings.TrimPrefix(c, "/private"))
-			}
-			require.EqualValues(t, tc.expected, cleanedCandidates)
-		})
-	}
-}
-
 func TestLibraryLocator(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 

internal/lookup/null.go

@@ -1,5 +1,5 @@
 /**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+# Copyright 2024 NVIDIA CORPORATION
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,24 +14,23 @@
 # limitations under the License.
 **/
 
-package ldcache
+package lookup
 
-import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+import "fmt"
 
-type empty struct {
-	logger logger.Interface
-	path   string
+// A null locator always returns an empty response.
+type null struct {
 }
 
-var _ LDCache = (*empty)(nil)
-
-// List always returns nil for an empty ldcache
-func (e *empty) List() ([]string, []string) {
+// Locate always returns empty for a null locator.
+func (l *null) Locate(string) ([]string, error) {
 	return nil, nil
 }
 
-// Lookup logs a debug message and returns nil for an empty ldcache
-func (e *empty) Lookup(prefixes ...string) ([]string, []string) {
-	e.logger.Debugf("Calling Lookup(%v) on empty ldcache: %v", prefixes, e.path)
-	return nil, nil
+// A notFound locator always returns an ErrNotFound error.
+type notFound struct {
+}
+
+func (l *notFound) Locate(s string) ([]string, error) {
+	return nil, fmt.Errorf("%s: %w", s, ErrNotFound)
 }

internal/lookup/root/root_test.go

@@ -0,0 +1,81 @@
+/**
+# Copyright 2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"path/filepath"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestDriverLibrariesLocate(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	testCases := []struct {
+		rootFs        string
+		inputs        []string
+		expected      string
+		expectedError error
+	}{
+		{
+			rootFs:        "rootfs-empty",
+			inputs:        []string{"libcuda.so.1", "libcuda.so.*", "libcuda.so.*.*", "libcuda.so.999.88.77"},
+			expectedError: lookup.ErrNotFound,
+		},
+		{
+			rootFs:   "rootfs-no-cache-lib64",
+			inputs:   []string{"libcuda.so.1", "libcuda.so.*", "libcuda.so.*.*", "libcuda.so.999.88.77"},
+			expected: "/usr/lib64/libcuda.so.999.88.77",
+		},
+		{
+			rootFs:   "rootfs-1",
+			inputs:   []string{"libcuda.so.1", "libcuda.so.*", "libcuda.so.*.*", "libcuda.so.999.88.77"},
+			expected: "/lib/x86_64-linux-gnu/libcuda.so.999.88.77",
+		},
+		{
+			rootFs:   "rootfs-2",
+			inputs:   []string{"libcuda.so.1", "libcuda.so.*", "libcuda.so.*.*", "libcuda.so.999.88.77"},
+			expected: "/var/lib/nvidia/lib64/libcuda.so.999.88.77",
+		},
+	}
+
+	for _, tc := range testCases {
+		for _, input := range tc.inputs {
+			t.Run(tc.rootFs+input, func(t *testing.T) {
+				rootfs := filepath.Join(moduleRoot, "testdata", "lookup", tc.rootFs)
+				driver := New(
+					WithLogger(logger),
+					WithDriverRoot(rootfs),
+				)
+
+				candidates, err := driver.Libraries().Locate(input)
+				require.ErrorIs(t, err, tc.expectedError)
+				if tc.expectedError == nil {
+					require.Equal(t, []string{filepath.Join(rootfs, tc.expected)}, candidates)
+				}
+			})
+		}
+	}
+}

internal/lookup/symlinks.go

@@ -62,6 +62,7 @@ func (p symlinkChain) Locate(pattern string) ([]string, error) {
 		return candidates, nil
 	}
 
+	var filenames []string
 	found := make(map[string]bool)
 	for len(candidates) > 0 {
 		candidate := candidates[0]
@@ -70,6 +71,7 @@ func (p symlinkChain) Locate(pattern string) ([]string, error) {
 			continue
 		}
 		found[candidate] = true
+		filenames = append(filenames, candidate)
 
 		target, err := symlinks.Resolve(candidate)
 		if err != nil {
@@ -88,11 +90,6 @@ func (p symlinkChain) Locate(pattern string) ([]string, error) {
 			candidates = append(candidates, target)
 		}
 	}
-
-	var filenames []string
-	for f := range found {
-		filenames = append(filenames, f)
-	}
 	return filenames, nil
 }
 

internal/lookup/symlinks/symlink.go

@@ -25,7 +25,7 @@ import (
 func Resolve(filename string) (string, error) {
 	info, err := os.Lstat(filename)
 	if err != nil {
-		return filename, fmt.Errorf("failed to get file info: %v", info)
+		return filename, fmt.Errorf("failed to get file info: %w", err)
 	}
 	if info.Mode()&os.ModeSymlink == 0 {
 		return filename, nil
@@ -33,3 +33,18 @@ func Resolve(filename string) (string, error) {
 
 	return os.Readlink(filename)
 }
+
+// ForceCreate creates a specified symlink.
+// If a file (or empty directory) exists at the path it is removed.
+func ForceCreate(target string, link string) error {
+	_, err := os.Lstat(link)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to get file info: %w", err)
+	}
+	if !os.IsNotExist(err) {
+		if err := os.Remove(link); err != nil {
+			return fmt.Errorf("failed to remove existing file: %w", err)
+		}
+	}
+	return os.Symlink(target, link)
+}

internal/modifier/cdi.go

@@ -90,11 +90,9 @@ func getDevicesFromSpec(logger logger.Interface, ociSpec oci.Spec, cfg *config.C
 		}
 	}
 
-	envDevices := container.DevicesFromEnvvars(visibleDevicesEnvvar)
-
 	var devices []string
 	seen := make(map[string]bool)
-	for _, name := range envDevices.List() {
+	for _, name := range container.VisibleDevicesFromEnvVar() {
 		if !parser.IsQualifiedName(name) {
 			name = fmt.Sprintf("%s=%s", cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.DefaultKind, name)
 		}

internal/modifier/csv.go

@@ -30,23 +30,16 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
 )
 
-const (
-	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
-	visibleDevicesVoid   = "void"
-
-	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
-)
-
 // NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
 // The modifications are defined by CSV MountSpecs.
-func NewCSVModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
+func NewCSVModifier(logger logger.Interface, cfg *config.Config, container image.CUDA) (oci.SpecModifier, error) {
+	if devices := container.VisibleDevicesFromEnvVar(); len(devices) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}
 	logger.Infof("Constructing modifier from config: %+v", *cfg)
 
-	if err := checkRequirements(logger, image); err != nil {
+	if err := checkRequirements(logger, container); err != nil {
 		return nil, fmt.Errorf("requirements not met: %v", err)
 	}
 
@@ -55,7 +48,7 @@ func NewCSVModifier(logger logger.Interface, cfg *config.Config, image image.CUD
 		return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
 	}
 
-	if image.Getenv(nvidiaRequireJetpackEnvvar) != "csv-mounts=all" {
+	if container.Getenv(image.EnvVarNvidiaRequireJetpack) != "csv-mounts=all" {
 		csvFiles = csv.BaseFilesOnly(csvFiles)
 	}
 

internal/modifier/gated.go

@@ -36,7 +36,7 @@ import (
 //
 // If not devices are selected, no changes are made.
 func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
+	if devices := image.VisibleDevicesFromEnvVar(); len(devices) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}
@@ -46,7 +46,7 @@ func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image
 	driverRoot := cfg.NVIDIAContainerCLIConfig.Root
 	devRoot := cfg.NVIDIAContainerCLIConfig.Root
 
-	if cfg.Features.IsEnabled(config.FeatureGDS, image) {
+	if image.Getenv("NVIDIA_GDS") == "enabled" {
 		d, err := discover.NewGDSDiscoverer(logger, driverRoot, devRoot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to construct discoverer for GDS devices: %w", err)
@@ -54,7 +54,7 @@ func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image
 		discoverers = append(discoverers, d)
 	}
 
-	if cfg.Features.IsEnabled(config.FeatureMOFED, image) {
+	if image.Getenv("NVIDIA_MOFED") == "enabled" {
 		d, err := discover.NewMOFEDDiscoverer(logger, devRoot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to construct discoverer for MOFED devices: %w", err)
@@ -62,7 +62,7 @@ func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image
 		discoverers = append(discoverers, d)
 	}
 
-	if cfg.Features.IsEnabled(config.FeatureNVSWITCH, image) {
+	if image.Getenv("NVIDIA_NVSWITCH") == "enabled" {
 		d, err := discover.NewNvSwitchDiscoverer(logger, devRoot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to construct discoverer for NVSWITCH devices: %w", err)
@@ -70,7 +70,7 @@ func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image
 		discoverers = append(discoverers, d)
 	}
 
-	if cfg.Features.IsEnabled(config.FeatureGDRCopy, image) {
+	if image.Getenv("NVIDIA_GDRCOPY") == "enabled" {
 		d, err := discover.NewGDRCopyDiscoverer(logger, devRoot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to construct discoverer for GDRCopy devices: %w", err)

internal/modifier/graphics.go

@@ -29,8 +29,8 @@ import (
 
 // NewGraphicsModifier constructs a modifier that injects graphics-related modifications into an OCI runtime specification.
 // The value of the NVIDIA_DRIVER_CAPABILITIES environment variable is checked to determine if this modification should be made.
-func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, image image.CUDA, driver *root.Driver) (oci.SpecModifier, error) {
-	if required, reason := requiresGraphicsModifier(image); !required {
+func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerImage image.CUDA, driver *root.Driver) (oci.SpecModifier, error) {
+	if required, reason := requiresGraphicsModifier(containerImage); !required {
 		logger.Infof("No graphics modifier required: %v", reason)
 		return nil, nil
 	}
@@ -50,7 +50,7 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, image imag
 	devRoot := driver.Root
 	drmNodes, err := discover.NewDRMNodesDiscoverer(
 		logger,
-		image.DevicesFromEnvvars(visibleDevicesEnvvar),
+		containerImage.DevicesFromEnvvars(image.EnvVarNvidiaVisibleDevices),
 		devRoot,
 		nvidiaCDIHookPath,
 	)
@@ -67,7 +67,7 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, image imag
 
 // requiresGraphicsModifier determines whether a graphics modifier is required.
 func requiresGraphicsModifier(cudaImage image.CUDA) (bool, string) {
-	if devices := cudaImage.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
+	if devices := cudaImage.VisibleDevicesFromEnvVar(); len(devices) == 0 {
 		return false, "no devices requested"
 	}
 

internal/modifier/list.go

@@ -22,14 +22,12 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 )
 
-type list struct {
-	modifiers []oci.SpecModifier
-}
+type List []oci.SpecModifier
 
 // Merge merges a set of OCI specification modifiers as a list.
 // This can be used to compose modifiers.
 func Merge(modifiers ...oci.SpecModifier) oci.SpecModifier {
-	var filteredModifiers []oci.SpecModifier
+	var filteredModifiers List
 	for _, m := range modifiers {
 		if m == nil {
 			continue
@@ -37,19 +35,19 @@ func Merge(modifiers ...oci.SpecModifier) oci.SpecModifier {
 		filteredModifiers = append(filteredModifiers, m)
 	}
 
-	return list{
-		modifiers: filteredModifiers,
-	}
+	return filteredModifiers
 }
 
 // Modify applies a list of modifiers in sequence and returns on any errors encountered.
-func (m list) Modify(spec *specs.Spec) error {
-	for _, mm := range m.modifiers {
+func (m List) Modify(spec *specs.Spec) error {
+	for _, mm := range m {
+		if mm == nil {
+			continue
+		}
 		err := mm.Modify(spec)
 		if err != nil {
 			return err
 		}
 	}
-
 	return nil
 }

internal/nvsandboxutils/api.go

@@ -0,0 +1,45 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+// libraryOptions hold the parameters than can be set by a LibraryOption
+type libraryOptions struct {
+	path  string
+	flags int
+}
+
+// LibraryOption represents a functional option to configure the underlying nvsandboxutils library
+type LibraryOption func(*libraryOptions)
+
+// WithLibraryPath provides an option to set the library name to be used by the nvsandboxutils library.
+func WithLibraryPath(path string) LibraryOption {
+	return func(o *libraryOptions) {
+		o.path = path
+	}
+}
+
+// SetLibraryOptions applies the specified options to the nvsandboxutils library.
+// If this is called when a library is already loaded, an error is raised.
+func SetLibraryOptions(opts ...LibraryOption) error {
+	libnvsandboxutils.Lock()
+	defer libnvsandboxutils.Unlock()
+	if libnvsandboxutils.refcount != 0 {
+		return errLibraryAlreadyLoaded
+	}
+	libnvsandboxutils.init(opts...)
+	return nil
+}

internal/nvsandboxutils/cgo_helpers.h

@@ -0,0 +1,25 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.
+// Code generated by https://git.io/c-for-go. DO NOT EDIT.
+
+#include "nvsandboxutils.h"
+#include <stdlib.h>
+#pragma once
+
+#define __CGOGEN 1
+

internal/nvsandboxutils/cgo_helpers_static.go

@@ -0,0 +1,38 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+var cgoAllocsUnknown = new(struct{})
+
+func clen(n []byte) int {
+	for i := 0; i < len(n); i++ {
+		if n[i] == 0 {
+			return i
+		}
+	}
+	return len(n)
+}
+
+// Creates an int8 array of fixed input length to store the Go string.
+// TODO: Add error check if input string has a length greater than INPUT_LENGTH
+func convertStringToFixedArray(str string) [INPUT_LENGTH]int8 {
+	var output [INPUT_LENGTH]int8
+	for i, s := range str {
+		output[i] = int8(s)
+	}
+	return output
+}

internal/nvsandboxutils/const.go

@@ -0,0 +1,156 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.
+// Code generated by https://git.io/c-for-go. DO NOT EDIT.
+
+package nvsandboxutils
+
+/*
+#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup
+#include "nvsandboxutils.h"
+#include <stdlib.h>
+#include "cgo_helpers.h"
+*/
+import "C"
+
+const (
+	// INPUT_LENGTH as defined in nvsandboxutils/nvsandboxutils.h
+	INPUT_LENGTH = 256
+	// MAX_FILE_PATH as defined in nvsandboxutils/nvsandboxutils.h
+	MAX_FILE_PATH = 256
+	// MAX_NAME_LENGTH as defined in nvsandboxutils/nvsandboxutils.h
+	MAX_NAME_LENGTH = 256
+)
+
+// Ret as declared in nvsandboxutils/nvsandboxutils.h
+type Ret int32
+
+// Ret enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	SUCCESS                     Ret = iota
+	ERROR_UNINITIALIZED         Ret = 1
+	ERROR_NOT_SUPPORTED         Ret = 2
+	ERROR_INVALID_ARG           Ret = 3
+	ERROR_INSUFFICIENT_SIZE     Ret = 4
+	ERROR_VERSION_NOT_SUPPORTED Ret = 5
+	ERROR_LIBRARY_LOAD          Ret = 6
+	ERROR_FUNCTION_NOT_FOUND    Ret = 7
+	ERROR_DEVICE_NOT_FOUND      Ret = 8
+	ERROR_NVML_LIB_CALL         Ret = 9
+	ERROR_OUT_OF_MEMORY         Ret = 10
+	ERROR_FILEPATH_NOT_FOUND    Ret = 11
+	ERROR_UNKNOWN               Ret = 65535
+)
+
+// LogLevel as declared in nvsandboxutils/nvsandboxutils.h
+type LogLevel int32
+
+// LogLevel enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	LOG_LEVEL_FATAL LogLevel = iota
+	LOG_LEVEL_ERROR LogLevel = 1
+	LOG_LEVEL_WARN  LogLevel = 2
+	LOG_LEVEL_DEBUG LogLevel = 3
+	LOG_LEVEL_INFO  LogLevel = 4
+	LOG_LEVEL_NONE  LogLevel = 65535
+)
+
+// RootfsInputType as declared in nvsandboxutils/nvsandboxutils.h
+type RootfsInputType int32
+
+// RootfsInputType enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	NV_ROOTFS_DEFAULT RootfsInputType = iota
+	NV_ROOTFS_PATH    RootfsInputType = 1
+	NV_ROOTFS_PID     RootfsInputType = 2
+)
+
+// FileType as declared in nvsandboxutils/nvsandboxutils.h
+type FileType int32
+
+// FileType enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	NV_DEV  FileType = iota
+	NV_PROC FileType = 1
+	NV_SYS  FileType = 2
+)
+
+// FileSystemSubType as declared in nvsandboxutils/nvsandboxutils.h
+type FileSystemSubType int32
+
+// FileSystemSubType enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	NV_DEV_NVIDIA                                         FileSystemSubType = iota
+	NV_DEV_DRI_CARD                                       FileSystemSubType = 1
+	NV_DEV_DRI_RENDERD                                    FileSystemSubType = 2
+	NV_DEV_DRI_CARD_SYMLINK                               FileSystemSubType = 3
+	NV_DEV_DRI_RENDERD_SYMLINK                            FileSystemSubType = 4
+	NV_DEV_NVIDIA_UVM                                     FileSystemSubType = 5
+	NV_DEV_NVIDIA_UVM_TOOLS                               FileSystemSubType = 6
+	NV_DEV_NVIDIA_MODESET                                 FileSystemSubType = 7
+	NV_DEV_NVIDIA_CTL                                     FileSystemSubType = 8
+	NV_DEV_GDRDRV                                         FileSystemSubType = 9
+	NV_DEV_NVIDIA_CAPS_NVIDIA_CAP                         FileSystemSubType = 10
+	NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID                   FileSystemSubType = 11
+	NV_PROC_DRIVER_NVIDIA_GPUS                            FileSystemSubType = 12
+	NV_PROC_NVIDIA_PARAMS                                 FileSystemSubType = 13
+	NV_PROC_NVIDIA_CAPS_MIG_MINORS                        FileSystemSubType = 14
+	NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU                FileSystemSubType = 15
+	NV_PROC_DRIVER_NVIDIA_CAPABILITIES                    FileSystemSubType = 16
+	NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS FileSystemSubType = 17
+	NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID                  FileSystemSubType = 18
+	NV_SYS_MODULE_NVIDIA_DRIVER                           FileSystemSubType = 19
+	NV_NUM_SUBTYPE                                        FileSystemSubType = 20
+)
+
+// FileModule as declared in nvsandboxutils/nvsandboxutils.h
+type FileModule int32
+
+// FileModule enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	NV_GPU                   FileModule = iota
+	NV_MIG                   FileModule = 1
+	NV_DRIVER_NVIDIA         FileModule = 2
+	NV_DRIVER_NVIDIA_UVM     FileModule = 3
+	NV_DRIVER_NVIDIA_MODESET FileModule = 4
+	NV_DRIVER_GDRDRV         FileModule = 5
+	NV_SYSTEM                FileModule = 6
+)
+
+// FileFlag as declared in nvsandboxutils/nvsandboxutils.h
+type FileFlag int32
+
+// FileFlag enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	NV_FILE_FLAG_HINT        FileFlag = 1
+	NV_FILE_FLAG_MASKOUT     FileFlag = 2
+	NV_FILE_FLAG_CONTENT     FileFlag = 4
+	NV_FILE_FLAG_DEPRECTATED FileFlag = 8
+	NV_FILE_FLAG_CANDIDATES  FileFlag = 16
+)
+
+// GpuInputType as declared in nvsandboxutils/nvsandboxutils.h
+type GpuInputType int32
+
+// GpuInputType enumeration from nvsandboxutils/nvsandboxutils.h
+const (
+	NV_GPU_INPUT_GPU_UUID  GpuInputType = iota
+	NV_GPU_INPUT_MIG_UUID  GpuInputType = 1
+	NV_GPU_INPUT_PCI_ID    GpuInputType = 2
+	NV_GPU_INPUT_PCI_INDEX GpuInputType = 3
+)

internal/nvsandboxutils/doc.go

@@ -0,0 +1,23 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.
+// Code generated by https://git.io/c-for-go. DO NOT EDIT.
+
+/*
+Package NVSANDBOXUTILS bindings
+*/
+package nvsandboxutils

internal/nvsandboxutils/dynamicLibrary_mock.go

@@ -0,0 +1,157 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package nvsandboxutils
+
+import (
+	"sync"
+)
+
+// Ensure, that dynamicLibraryMock does implement dynamicLibrary.
+// If this is not the case, regenerate this file with moq.
+var _ dynamicLibrary = &dynamicLibraryMock{}
+
+// dynamicLibraryMock is a mock implementation of dynamicLibrary.
+//
+//	func TestSomethingThatUsesdynamicLibrary(t *testing.T) {
+//
+//		// make and configure a mocked dynamicLibrary
+//		mockeddynamicLibrary := &dynamicLibraryMock{
+//			CloseFunc: func() error {
+//				panic("mock out the Close method")
+//			},
+//			LookupFunc: func(s string) error {
+//				panic("mock out the Lookup method")
+//			},
+//			OpenFunc: func() error {
+//				panic("mock out the Open method")
+//			},
+//		}
+//
+//		// use mockeddynamicLibrary in code that requires dynamicLibrary
+//		// and then make assertions.
+//
+//	}
+type dynamicLibraryMock struct {
+	// CloseFunc mocks the Close method.
+	CloseFunc func() error
+
+	// LookupFunc mocks the Lookup method.
+	LookupFunc func(s string) error
+
+	// OpenFunc mocks the Open method.
+	OpenFunc func() error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Close holds details about calls to the Close method.
+		Close []struct {
+		}
+		// Lookup holds details about calls to the Lookup method.
+		Lookup []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Open holds details about calls to the Open method.
+		Open []struct {
+		}
+	}
+	lockClose  sync.RWMutex
+	lockLookup sync.RWMutex
+	lockOpen   sync.RWMutex
+}
+
+// Close calls CloseFunc.
+func (mock *dynamicLibraryMock) Close() error {
+	callInfo := struct {
+	}{}
+	mock.lockClose.Lock()
+	mock.calls.Close = append(mock.calls.Close, callInfo)
+	mock.lockClose.Unlock()
+	if mock.CloseFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.CloseFunc()
+}
+
+// CloseCalls gets all the calls that were made to Close.
+// Check the length with:
+//
+//	len(mockeddynamicLibrary.CloseCalls())
+func (mock *dynamicLibraryMock) CloseCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockClose.RLock()
+	calls = mock.calls.Close
+	mock.lockClose.RUnlock()
+	return calls
+}
+
+// Lookup calls LookupFunc.
+func (mock *dynamicLibraryMock) Lookup(s string) error {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockLookup.Lock()
+	mock.calls.Lookup = append(mock.calls.Lookup, callInfo)
+	mock.lockLookup.Unlock()
+	if mock.LookupFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.LookupFunc(s)
+}
+
+// LookupCalls gets all the calls that were made to Lookup.
+// Check the length with:
+//
+//	len(mockeddynamicLibrary.LookupCalls())
+func (mock *dynamicLibraryMock) LookupCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockLookup.RLock()
+	calls = mock.calls.Lookup
+	mock.lockLookup.RUnlock()
+	return calls
+}
+
+// Open calls OpenFunc.
+func (mock *dynamicLibraryMock) Open() error {
+	callInfo := struct {
+	}{}
+	mock.lockOpen.Lock()
+	mock.calls.Open = append(mock.calls.Open, callInfo)
+	mock.lockOpen.Unlock()
+	if mock.OpenFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.OpenFunc()
+}
+
+// OpenCalls gets all the calls that were made to Open.
+// Check the length with:
+//
+//	len(mockeddynamicLibrary.OpenCalls())
+func (mock *dynamicLibraryMock) OpenCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockOpen.RLock()
+	calls = mock.calls.Open
+	mock.lockOpen.RUnlock()
+	return calls
+}

internal/nvsandboxutils/gen/generate-bindings.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file generates bindings for nvsandboxutils by calling c-for-go.
+
+set -x -e
+
+PWD=$(pwd)
+GEN_DIR="$PWD/gen"
+PKG_DIR="$PWD"
+GEN_BINDINGS_DIR="$GEN_DIR/nvsandboxutils"
+PKG_BINDINGS_DIR="$PKG_DIR"
+
+SOURCES=$(find "$GEN_BINDINGS_DIR" -type f)
+
+mkdir -p "$PKG_BINDINGS_DIR"
+
+cp "$GEN_BINDINGS_DIR/nvsandboxutils.h" "$PKG_BINDINGS_DIR/nvsandboxutils.h"
+spatch --in-place --very-quiet --sp-file "$GEN_BINDINGS_DIR/anonymous_structs.cocci" "$PKG_BINDINGS_DIR/nvsandboxutils.h" > /dev/null
+
+echo "Generating the bindings..."
+c-for-go -out "$PKG_DIR/.." "$GEN_BINDINGS_DIR/nvsandboxutils.yml"
+cd "$PKG_BINDINGS_DIR"
+go tool cgo -godefs types.go > types_gen.go
+go fmt types_gen.go
+cd - > /dev/null
+rm -rf "$PKG_BINDINGS_DIR/cgo_helpers.go" "$PKG_BINDINGS_DIR/types.go" "$PKG_BINDINGS_DIR/_obj"
+go run "$GEN_BINDINGS_DIR/generateapi.go" --sourceDir "$PKG_BINDINGS_DIR" --output "$PKG_BINDINGS_DIR/zz_generated.api.go"
+# go fmt "$PKG_BINDINGS_DIR"
+
+SED_SEARCH_STRING='// WARNING: This file has automatically been generated on'
+SED_REPLACE_STRING='// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.'
+grep -l -R "$SED_SEARCH_STRING" "$PKG_DIR" | grep -v "/gen/" | xargs sed -i -E "s#$SED_SEARCH_STRING.*\$#$SED_REPLACE_STRING#g"
+
+SED_SEARCH_STRING='// (.*) nvsandboxutils/nvsandboxutils.h:[0-9]+'
+SED_REPLACE_STRING='// \1 nvsandboxutils/nvsandboxutils.h'
+grep -l -RE "$SED_SEARCH_STRING" "$PKG_DIR" | grep -v "/gen/" | xargs sed -i -E "s#$SED_SEARCH_STRING\$#$SED_REPLACE_STRING#g"
+

internal/nvsandboxutils/gen/nvsandboxutils/anonymous_structs.cocci

@@ -0,0 +1,100 @@
+@patch@
+type WRAPPER_TYPE;
+field list FIELDS;
+identifier V;
+expression E;
+fresh identifier ST = "nvSandboxUtilsGenerated_struct___";
+fresh identifier TEMP_VAR = "nvSandboxUtilsGenerated_variable___" ## V;
+@@
+
+++ struct ST {
+++    WRAPPER_TYPE TEMP_VAR;
+++    FIELDS
+++ };
++
+
+WRAPPER_TYPE
+{
+    ...
+(
+-    struct {
+-       FIELDS
+-   } V[E];
++   struct ST V[E];
+
+|
+
+-    struct {
+-       FIELDS
+-   } V;
++   struct ST V;
+)
+    ...
+};
+
+@capture@
+type WRAPPER_TYPE;
+identifier TEMP_VAR;
+identifier ST =~ "^nvSandboxUtilsGenerated_struct___";
+@@
+
+struct ST {
+  WRAPPER_TYPE TEMP_VAR;
+  ...
+};
+
+@script:python concat@
+WRAPPER_TYPE << capture.WRAPPER_TYPE;
+TEMP_VAR << capture.TEMP_VAR;
+ST << capture.ST;
+T;
+@@
+
+def removePrefix(string, prefix):
+    if string.startswith(prefix):
+        return string[len(prefix):]
+    return string
+
+def removeSuffix(string, suffix):
+    if string.endswith(suffix):
+        return string[:-len(suffix)]
+    return string
+
+WRAPPER_TYPE = removeSuffix(WRAPPER_TYPE, "_t")
+TEMP_VAR = removePrefix(TEMP_VAR, "nvSandboxUtilsGenerated_variable___")
+coccinelle.T = cocci.make_type(WRAPPER_TYPE + TEMP_VAR[0].upper() + TEMP_VAR[1:] + "_t")
+
+@add_typedef@
+identifier capture.ST;
+type concat.T;
+type WRAPPER_TYPE;
+identifier TEMP_VAR;
+@@
+
+- struct ST {
++ typedef struct {
+- WRAPPER_TYPE TEMP_VAR;
+  ...
+- };
++ } T;
+
+@update@
+identifier capture.ST;
+type concat.T;
+identifier V;
+expression E;
+type WRAPPER_TYPE;
+@@
+
+WRAPPER_TYPE
+{
+    ...
+(
+-   struct ST V[E];
++   T V[E];
+|
+-   struct ST V;
++   T V;
+)
+    ...
+};

internal/nvsandboxutils/gen/nvsandboxutils/generateapi.go

@@ -0,0 +1,389 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strings"
+	"unicode"
+)
+
+type GeneratableInterfacePoperties struct {
+	Type                      string
+	Interface                 string
+	Exclude                   []string
+	PackageMethodsAliasedFrom string
+}
+
+var GeneratableInterfaces = []GeneratableInterfacePoperties{
+	{
+		Type:                      "library",
+		Interface:                 "Interface",
+		PackageMethodsAliasedFrom: "libnvsandboxutils",
+	},
+}
+
+func main() {
+	sourceDir := flag.String("sourceDir", "", "Path to the source directory for all go files")
+	output := flag.String("output", "", "Path to the output file (default: stdout)")
+	flag.Parse()
+
+	// Check if required flags are provided
+	if *sourceDir == "" {
+		flag.Usage()
+		return
+	}
+
+	writer, closer, err := getWriter(*output)
+	if err != nil {
+		fmt.Printf("Error: %v", err)
+		return
+	}
+	defer func() {
+		_ = closer()
+	}()
+
+	header, err := generateHeader()
+	if err != nil {
+		fmt.Printf("Error: %v", err)
+		return
+	}
+	fmt.Fprint(writer, header)
+
+	for i, p := range GeneratableInterfaces {
+		if p.PackageMethodsAliasedFrom != "" {
+			comment, err := generatePackageMethodsComment(p)
+			if err != nil {
+				fmt.Printf("Error: %v", err)
+				return
+			}
+			fmt.Fprint(writer, comment)
+
+			output, err := generatePackageMethods(*sourceDir, p)
+			if err != nil {
+				fmt.Printf("Error: %v", err)
+				return
+			}
+			fmt.Fprintf(writer, "%s\n", output)
+		}
+
+		comment, err := generateInterfaceComment(p)
+		if err != nil {
+			fmt.Printf("Error: %v", err)
+			return
+		}
+		fmt.Fprint(writer, comment)
+
+		output, err := generateInterface(*sourceDir, p)
+		if err != nil {
+			fmt.Printf("Error: %v", err)
+			return
+		}
+		fmt.Fprint(writer, output)
+
+		if i < (len(GeneratableInterfaces) - 1) {
+			fmt.Fprint(writer, "\n")
+		}
+	}
+}
+
+func getWriter(outputFile string) (io.Writer, func() error, error) {
+	if outputFile == "" {
+		return os.Stdout, func() error { return nil }, nil
+	}
+
+	file, err := os.Create(outputFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return file, file.Close, nil
+}
+
+func generateHeader() (string, error) {
+	lines := []string{
+		"/**",
+		"# Copyright 2024 NVIDIA CORPORATION",
+		"#",
+		"# Licensed under the Apache License, Version 2.0 (the \"License\");",
+		"# you may not use this file except in compliance with the License.",
+		"# You may obtain a copy of the License at",
+		"#",
+		"#     http://www.apache.org/licenses/LICENSE-2.0",
+		"#",
+		"# Unless required by applicable law or agreed to in writing, software",
+		"# distributed under the License is distributed on an \"AS IS\" BASIS,",
+		"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.",
+		"# See the License for the specific language governing permissions and",
+		"# limitations under the License.",
+		"**/",
+		"",
+		"// Generated Code; DO NOT EDIT.",
+		"",
+		"package nvsandboxutils",
+		"",
+		"",
+	}
+	return strings.Join(lines, "\n"), nil
+}
+
+func generatePackageMethodsComment(input GeneratableInterfacePoperties) (string, error) {
+	commentFmt := []string{
+		"// The variables below represent package level methods from the %s type.",
+	}
+
+	var signature strings.Builder
+	comment := strings.Join(commentFmt, "\n")
+	comment = fmt.Sprintf(comment, input.Type)
+	signature.WriteString(fmt.Sprintf("%s\n", comment))
+	return signature.String(), nil
+}
+
+func generateInterfaceComment(input GeneratableInterfacePoperties) (string, error) {
+	commentFmt := []string{
+		"// %s represents the interface for the %s type.",
+		"//",
+		"//go:generate moq -out mock/%s.go -pkg mock . %s:%s",
+	}
+
+	var signature strings.Builder
+	comment := strings.Join(commentFmt, "\n")
+	comment = fmt.Sprintf(comment, input.Interface, input.Type, strings.ToLower(input.Interface), input.Interface, input.Interface)
+	signature.WriteString(fmt.Sprintf("%s\n", comment))
+	return signature.String(), nil
+}
+
+func generatePackageMethods(sourceDir string, input GeneratableInterfacePoperties) (string, error) {
+	var signature strings.Builder
+
+	signature.WriteString("var (\n")
+
+	methods, err := extractMethodsFromPackage(sourceDir, input)
+	if err != nil {
+		return "", err
+	}
+
+	for _, method := range methods {
+		name := method.Name.Name
+		formatted := fmt.Sprintf("\t%s = %s.%s\n", name, input.PackageMethodsAliasedFrom, name)
+		signature.WriteString(formatted)
+	}
+
+	signature.WriteString(")\n")
+
+	return signature.String(), nil
+}
+
+func generateInterface(sourceDir string, input GeneratableInterfacePoperties) (string, error) {
+	var signature strings.Builder
+
+	signature.WriteString(fmt.Sprintf("type %s interface {\n", input.Interface))
+
+	methods, err := extractMethodsFromPackage(sourceDir, input)
+	if err != nil {
+		return "", err
+	}
+
+	for _, method := range methods {
+		formatted := fmt.Sprintf("\t%s\n", formatMethodSignature(method))
+		signature.WriteString(formatted)
+	}
+
+	signature.WriteString("}\n")
+
+	return signature.String(), nil
+}
+
+func getGoFiles(sourceDir string) (map[string][]byte, error) {
+	gofiles := make(map[string][]byte)
+
+	err := filepath.WalkDir(sourceDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() || filepath.Ext(path) != ".go" {
+			return nil
+		}
+
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		gofiles[path] = content
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("walking %s: %w", sourceDir, err)
+	}
+
+	return gofiles, nil
+}
+
+func extractMethodsFromPackage(sourceDir string, input GeneratableInterfacePoperties) ([]*ast.FuncDecl, error) {
+	gofiles, err := getGoFiles(sourceDir)
+	if err != nil {
+		return nil, err
+	}
+
+	var methods []*ast.FuncDecl
+	for file, content := range gofiles {
+		m, err := extractMethods(file, content, input)
+		if err != nil {
+			return nil, err
+		}
+		methods = append(methods, m...)
+	}
+
+	sort.Slice(methods, func(i, j int) bool {
+		return methods[i].Name.Name < methods[j].Name.Name
+	})
+
+	return methods, nil
+}
+
+func extractMethods(sourceFile string, sourceContent []byte, input GeneratableInterfacePoperties) ([]*ast.FuncDecl, error) {
+	// Parse source file
+	fset := token.NewFileSet()
+	node, err := parser.ParseFile(fset, sourceFile, sourceContent, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	// Traverse AST to find type declarations and associated methods
+	var methods []*ast.FuncDecl
+	for _, decl := range node.Decls {
+		funcDecl, ok := decl.(*ast.FuncDecl)
+		if !ok {
+			continue
+		}
+
+		// Check if the function is a method associated with the specified type
+		if receiverType := funcDecl.Recv; receiverType != nil {
+			var ident *ast.Ident
+
+			for _, field := range receiverType.List {
+				switch fieldType := field.Type.(type) {
+				case *ast.Ident:
+					ident = fieldType
+				case *ast.StarExpr:
+					// Update ident if it's a *ast.StarExpr
+					if newIdent, ok := fieldType.X.(*ast.Ident); ok {
+						// If the inner type is an *ast.Ident, update ident
+						ident = newIdent
+					}
+				}
+
+				// No identifier found
+				if ident == nil {
+					continue
+				}
+
+				// Identifier is not the one we are looking for
+				if ident.Name != input.Type {
+					continue
+				}
+
+				// Ignore non-public methods
+				if !isPublic(funcDecl.Name.Name) {
+					continue
+				}
+
+				// Ignore method in the exclude list
+				if slices.Contains(input.Exclude, funcDecl.Name.Name) {
+					continue
+				}
+
+				methods = append(methods, funcDecl)
+			}
+		}
+	}
+
+	return methods, nil
+}
+
+func formatMethodSignature(decl *ast.FuncDecl) string {
+	var signature strings.Builder
+
+	// Write method name
+	signature.WriteString(decl.Name.Name)
+	signature.WriteString("(")
+
+	// Write parameters
+	if decl.Type.Params != nil {
+		for i, param := range decl.Type.Params.List {
+			if i > 0 {
+				signature.WriteString(", ")
+			}
+			signature.WriteString(formatFieldList(param))
+		}
+	}
+
+	signature.WriteString(")")
+
+	// Write return types
+	if decl.Type.Results != nil {
+		signature.WriteString(" ")
+		if len(decl.Type.Results.List) > 1 {
+			signature.WriteString("(")
+		}
+		for i, result := range decl.Type.Results.List {
+			if i > 0 {
+				signature.WriteString(", ")
+			}
+			signature.WriteString(formatFieldList(result))
+		}
+		if len(decl.Type.Results.List) > 1 {
+			signature.WriteString(")")
+		}
+	}
+
+	return signature.String()
+}
+
+func formatFieldList(field *ast.Field) string {
+	var builder strings.Builder
+	switch fieldType := field.Type.(type) {
+	case *ast.Ident:
+		builder.WriteString(fieldType.Name)
+	case *ast.ArrayType:
+		builder.WriteString("[]")
+		builder.WriteString(formatFieldList(&ast.Field{Type: fieldType.Elt}))
+	case *ast.StarExpr:
+		builder.WriteString("*")
+		builder.WriteString(formatFieldList(&ast.Field{Type: fieldType.X}))
+	}
+	return builder.String()
+}
+
+func isPublic(name string) bool {
+	if len(name) == 0 {
+		return false
+	}
+	return unicode.IsUpper([]rune(name)[0])
+}

internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.h

@@ -0,0 +1,298 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __NVSANDBOXUTILS_H__
+#define __NVSANDBOXUTILS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define INPUT_LENGTH 256
+#define MAX_FILE_PATH 256
+#define MAX_NAME_LENGTH 256
+
+/***************************************************************************************************/
+/** @defgroup enums Enumerations
+ *  @{
+ */
+/***************************************************************************************************/
+
+/**
+ * Return types
+ */
+typedef enum
+{
+    NVSANDBOXUTILS_SUCCESS                              = 0,         //!< The operation was successful
+    NVSANDBOXUTILS_ERROR_UNINITIALIZED                  = 1,         //!< The library wasn't successfully initialized
+    NVSANDBOXUTILS_ERROR_NOT_SUPPORTED                  = 2,         //!< The requested operation is not supported on target device
+    NVSANDBOXUTILS_ERROR_INVALID_ARG                    = 3,         //!< A supplied argument is invalid
+    NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE              = 4,         //!< A supplied argument is not large enough
+    NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED          = 5,         //!< Requested library version is not supported
+    NVSANDBOXUTILS_ERROR_LIBRARY_LOAD                   = 6,         //!< The library load failed
+    NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND             = 7,         //!< Called function was not found
+    NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND               = 8,         //!< Target device was not found
+    NVSANDBOXUTILS_ERROR_NVML_LIB_CALL                  = 9,         //!< NVML library call failed
+    NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY                  = 10,        //!< There is insufficient memory
+    NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND             = 11,        //!< A supplied file path was not found
+    NVSANDBOXUTILS_ERROR_UNKNOWN                        = 0xFFFF,    //!< Unknown error occurred
+} nvSandboxUtilsRet_t;
+
+/**
+ * Return if there is an error 
+ */
+#define RETURN_ON_SANDBOX_ERROR(result) \
+    if ((result) != NVSANDBOXUTILS_SUCCESS) { \
+        NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \
+        return result; \
+    }
+
+/**
+ * Log levels
+ */
+typedef enum
+{
+    NVSANDBOXUTILS_LOG_LEVEL_FATAL         = 0,                       //!< Log fatal errors
+    NVSANDBOXUTILS_LOG_LEVEL_ERROR         = 1,                       //!< Log all errors
+    NVSANDBOXUTILS_LOG_LEVEL_WARN          = 2,                       //!< Log all warnings
+    NVSANDBOXUTILS_LOG_LEVEL_DEBUG         = 3,                       //!< Log all debug messages
+    NVSANDBOXUTILS_LOG_LEVEL_INFO          = 4,                       //!< Log all info messages
+    NVSANDBOXUTILS_LOG_LEVEL_NONE          = 0xFFFF,                  //!< Log none
+} nvSandboxUtilsLogLevel_t;
+
+/**
+ * Input rootfs to help access files inside the driver container
+ */ 
+typedef enum
+{
+    NV_ROOTFS_DEFAULT,                                               //!< Default no rootfs 
+    NV_ROOTFS_PATH,                                                  //!< /run/nvidia/driver
+    NV_ROOTFS_PID,                                                   //!< /proc/PID/mountinfo
+} nvSandboxUtilsRootfsInputType_t;
+
+/**
+ * File type
+ */
+typedef enum
+{
+    NV_DEV,                                                          //!< /dev file system
+    NV_PROC,                                                         //!< /proc file system
+    NV_SYS,                                                          //!< /sys file system
+} nvSandboxUtilsFileType_t;
+
+/**
+ * File subtype  
+ */
+typedef enum
+{
+    NV_DEV_NVIDIA,                                                   //!< /dev/nvidia0
+    NV_DEV_DRI_CARD,                                                 //!< /dev/dri/card1
+    NV_DEV_DRI_RENDERD,                                              //!< /dev/dri/renderD128
+    NV_DEV_DRI_CARD_SYMLINK,                                         //!< /dev/dri/by-path/pci-0000:41:00.0-card
+    NV_DEV_DRI_RENDERD_SYMLINK,                                      //!< /dev/dri/by-path/pci-0000:41:00.0-render
+    NV_DEV_NVIDIA_UVM,                                               //!< /dev/nvidia-uvm
+    NV_DEV_NVIDIA_UVM_TOOLS,                                         //!< /dev/nvidia-uvm-tools
+    NV_DEV_NVIDIA_MODESET,                                           //!< /dev/nvidia-uvm-modeset
+    NV_DEV_NVIDIA_CTL,                                               //!< /dev/nvidiactl
+    NV_DEV_GDRDRV,                                                   //!< /dev/gdrdrv
+    NV_DEV_NVIDIA_CAPS_NVIDIA_CAP,                                   //!< /dev/nvidia-caps/nvidia-cap22
+    NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID,                             //!< /proc/driver/nvidia/gpus/0000:2d:00.0
+    NV_PROC_DRIVER_NVIDIA_GPUS,                                      //!< /proc/driver/nvidia/gpus (for mask out)
+    NV_PROC_NVIDIA_PARAMS,                                           //!< /proc/driver/nvidia/params
+    NV_PROC_NVIDIA_CAPS_MIG_MINORS,                                  //!< /proc/driver/nvidia-caps/mig-minors
+    NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU,                          //!< /proc/driver/nvidia/capabilities/gpu0
+    NV_PROC_DRIVER_NVIDIA_CAPABILITIES,                              //!< /proc/driver/nvidia/capabilities (for mask out)
+    NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS,           //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access
+    NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID,                            //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0
+    NV_SYS_MODULE_NVIDIA_DRIVER,                                     //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out)
+    NV_NUM_SUBTYPE, // always at the end.
+} nvSandboxUtilsFileSystemSubType_t;
+
+/**
+ * File module
+ */
+typedef enum
+{
+    NV_GPU,                                                           //!< Target device
+    NV_MIG,                                                           //!< Target device- MIG
+    NV_DRIVER_NVIDIA,                                                 //!< NVIDIA kernel driver
+    NV_DRIVER_NVIDIA_UVM,                                             //!< NVIDIA kernel driver-UVM
+    NV_DRIVER_NVIDIA_MODESET,                                         //!< NVIDIA kernel driver-modeset
+    NV_DRIVER_GDRDRV,                                                 //!< GDRDRV driver
+    NV_SYSTEM,                                                        //!< System module
+} nvSandboxUtilsFileModule_t;
+
+/**
+ * Flag to provide additional details about the file
+ */
+typedef enum
+{
+    NV_FILE_FLAG_HINT            = (1 << 0),                         //!< Default no hint
+    NV_FILE_FLAG_MASKOUT         = (1 << 1),                         //!< For /proc/driver/nvidia/gpus
+    NV_FILE_FLAG_CONTENT         = (1 << 2),                         //!< For /proc/driver/nvidia/params
+                                                                     //!< For SYMLINK
+                                                                     //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file
+    NV_FILE_FLAG_DEPRECTATED     = (1 << 3),                         //!< For all the FIRMWARE GSP file
+    NV_FILE_FLAG_CANDIDATES      = (1 << 4),                         //!< For libcuda.so
+} nvSandboxUtilsFileFlag_t;
+
+/**
+ * Input type of the target device
+ */
+typedef enum
+{
+    NV_GPU_INPUT_GPU_UUID,                                           //!< GPU UUID
+    NV_GPU_INPUT_MIG_UUID,                                           //!< MIG UUID
+    NV_GPU_INPUT_PCI_ID,                                             //!< PCIe DBDF ID
+    NV_GPU_INPUT_PCI_INDEX,                                          //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF)
+} nvSandboxUtilsGpuInputType_t;
+
+/** @} */
+
+/***************************************************************************************************/
+/** @defgroup dataTypes Structures and Unions
+ *  @{
+ */
+/***************************************************************************************************/
+
+/**
+ * Initalization input v1
+ */
+typedef struct
+{
+    unsigned int version;                                            //!< Version for the structure
+    nvSandboxUtilsRootfsInputType_t type;                            //!< One of \p nvSandboxUtilsRootfsInputType_t
+    char value[INPUT_LENGTH];                                        //!< String representation of input
+} nvSandboxUtilsInitInput_v1_t;
+
+typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t;
+
+/**
+ * File system information 
+ */
+typedef struct nvSandboxUtilsGpuFileInfo_v1_t
+{
+    struct nvSandboxUtilsGpuFileInfo_v1_t *next;                     //!< Pointer to the next node in the linked list
+    nvSandboxUtilsFileType_t fileType;                               //!< One of \p nvSandboxUtilsFileType_t
+    nvSandboxUtilsFileSystemSubType_t fileSubType;                   //!< One of \p nvSandboxUtilsFileSystemSubType_t
+    nvSandboxUtilsFileModule_t module;                               //!< One of \p nvSandboxUtilsFileModule_t
+    nvSandboxUtilsFileFlag_t flags;                                  //!< One of \p nvSandboxUtilsFileFlag_t
+    char *filePath;                                                  //!< Relative file path to rootfs
+}nvSandboxUtilsGpuFileInfo_v1_t;
+
+/**
+ * GPU resource request v1
+ */
+typedef struct
+{
+     unsigned int version;                                           //!< Version for the structure
+     nvSandboxUtilsGpuInputType_t inputType;                         //!< One of \p nvSandboxUtilsGpuInputType_t
+     char input[INPUT_LENGTH];                                       //!< String representation of input
+     nvSandboxUtilsGpuFileInfo_v1_t *files;                          //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t
+} nvSandboxUtilsGpuRes_v1_t;
+
+typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t;
+
+/** @} */
+
+/***************************************************************************************************/
+/** @defgroup funcs Functions
+ *  @{
+ */
+/***************************************************************************************************/
+
+/* *************************************************
+ * Initialize library
+ * *************************************************
+ */
+/**
+ * Prepare library resources before library API can be used.
+ * This initialization will not fail if one of the initialization prerequisites fails.
+ * @param           input  Reference to the called-supplied input struct that has initialization fields  
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p input->value isn't a valid rootfs path
+ * @returns         @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED  if \p input->version isn't supported by the library
+ * @returns         @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND     if any of the required file paths are not found during initialization
+ * @returns         @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY          if there is insufficient system memory during initialization
+ * @returns         @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD           on any error during loading the library
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input);
+
+/* *************************************************
+ * Shutdown library
+ * *************************************************
+ */
+/**
+ * Clean up library resources created by init call
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void);
+
+/* *************************************************
+ * Get NVIDIA RM driver version
+ * *************************************************
+ */
+/**
+ * Get NVIDIA RM driver version
+ * @param           version  Reference to caller-supplied buffer to return driver version string
+ * @param           length   The maximum allowed length of the string returned in \p version
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p version is NULL
+ * @returns         @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL          on any error during driver version query from NVML
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length);
+
+/* *************************************************
+ * Get /dev, /proc, /sys file system information
+ * *************************************************
+ */
+/**
+ * Get /dev, /proc, /sys file system information
+ * @param           request  Reference to caller-supplied request struct to return the file system information 
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p request->input doesn't match any device
+ * @returns         @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED  if \p request->version isn't supported by the library
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request);
+
+/* *************************************************
+ * Get content of given file path
+ * *************************************************
+ */
+/**
+ * Get file content of input file path
+ * @param           filePath     Reference to the file path
+ * @param           content      Reference to the caller-supplied buffer to return the file content
+ * @param           contentSize  Reference to the maximum allowed size of content. It is updated to the actual size of the content on return
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p filePath or \p content is NULL
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE      if \p contentSize is too small
+ * @returns         @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND     on an error while obtaining the content for the file path
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize);
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+#endif // __NVSANDBOXUTILS_H__

internal/nvsandboxutils/gen/nvsandboxutils/nvsandboxutils.yml

@@ -0,0 +1,66 @@
+# Copyright (c) 2024, NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+GENERATOR:
+  PackageName: nvsandboxutils
+  PackageDescription: "Package NVSANDBOXUTILS bindings"
+  PackageLicense: |-
+    Copyright (c) 2024, NVIDIA CORPORATION
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+  Includes: ["nvsandboxutils.h"]
+  FlagGroups:
+    - {name: "LDFLAGS", traits: ["linux"], flags: ["-Wl,--export-dynamic","-Wl,--unresolved-symbols=ignore-in-object-files"]}
+    - {name: "LDFLAGS", traits: ["darwin"], flags: ["-Wl,-undefined,dynamic_lookup"]}
+PARSER:
+  SourcesPaths: ["nvsandboxutils.h"]
+TRANSLATOR:
+  ConstRules:
+    defines: eval
+    enum: eval
+  PtrTips:
+    function:
+      - {target: "^nvSandboxUtils", default: "sref"}
+  MemTips:
+    - {target: "^nvSandboxUtils", default: "raw"}
+  Rules:
+    const:
+      - {action: accept, from: "^NVSANDBOXUTILS_"}
+      - {action: accept, from: "^nvSandboxUtils"}
+      - {action: replace, from: "^NVSANDBOXUTILS_"}
+      - {action: replace, from: "^nvSandboxUtils"}
+      - {action: accept, from: "^NV"}
+      - {action: accept, from: "^MAX"}
+      - {action: accept, from: "^INPUT"}
+      - {action: replace, from: "_t$"}
+      - {transform: export}
+    type:
+      - {action: accept, from: "^nvSandboxUtils"}
+      - {action: replace, from: "^nvSandboxUtils"}
+      - {action: replace, from: "_t$"}
+      - {transform: export}
+    function:
+      - {action: accept, from: "^nvSandboxUtils"}
+      - {transform: unexport}

internal/nvsandboxutils/gen/update-bindings.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file allows for the nvsandboxutils bindings to be updated using the tooling
+# implemented in https://github.com/NVIDIA/go-nvml.
+# To run this:
+#   cd internal/nvsandboxutils
+#   ./update-bindings.sh
+
+set -e
+
+BUILDIMAGE=bindings
+
+docker build \
+    --build-arg GOLANG_VERSION=1.22.1 \
+    --build-arg C_FOR_GO_TAG=8eeee8c3b71f9c3c90c4a73db54ed08b0bba971d \
+    -t ${BUILDIMAGE} \
+    -f docker/Dockerfile.devel \
+    https://github.com/NVIDIA/go-nvml.git
+
+
+docker run --rm -ti \
+		-e GOCACHE=/tmp/.cache/go \
+		-e GOMODCACHE=/tmp/.cache/gomod \
+    -v $(pwd):/nvsandboxutils \
+    -w /nvsandboxutils \
+    -u $(id -u):$(id -g) \
+    ${BUILDIMAGE} \
+    ./gen/generate-bindings.sh

internal/nvsandboxutils/gpu-resources.go

@@ -0,0 +1,67 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+import (
+	"strings"
+	"unsafe"
+)
+
+import "C"
+
+type GpuResource struct {
+	Version uint32
+}
+
+type GpuFileInfo struct {
+	Path    string
+	Type    FileType
+	SubType FileSystemSubType
+	Module  FileModule
+	Flags   FileFlag
+}
+
+func (l *library) GetGpuResource(uuid string) ([]GpuFileInfo, Ret) {
+	deviceType := NV_GPU_INPUT_GPU_UUID
+	if strings.HasPrefix(uuid, "MIG-") {
+		deviceType = NV_GPU_INPUT_MIG_UUID
+	}
+
+	request := GpuRes{
+		Version:   1,
+		InputType: uint32(deviceType),
+		Input:     convertStringToFixedArray(uuid),
+	}
+
+	ret := nvSandboxUtilsGetGpuResource(&request)
+	if ret != SUCCESS {
+		return nil, ret
+	}
+
+	var fileInfos []GpuFileInfo
+	for fileInfo := request.Files; fileInfo != nil; fileInfo = fileInfo.Next {
+		fi := GpuFileInfo{
+			Path:    C.GoString((*C.char)(unsafe.Pointer(fileInfo.FilePath))),
+			Type:    FileType(fileInfo.FileType),
+			SubType: FileSystemSubType(fileInfo.FileSubType),
+			Module:  FileModule(fileInfo.Module),
+			Flags:   FileFlag(fileInfo.Flags),
+		}
+		fileInfos = append(fileInfos, fi)
+	}
+	return fileInfos, SUCCESS
+}

internal/nvsandboxutils/impl.go

@@ -0,0 +1,64 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+import "C"
+
+func (l *library) Init(path string) Ret {
+	if err := l.load(); err != nil {
+		return ERROR_LIBRARY_LOAD
+	}
+
+	input := InitInput{
+		Version: 1,
+		Type:    uint32(NV_ROOTFS_PATH),
+		Value:   convertStringToFixedArray(path),
+	}
+
+	return nvSandboxUtilsInit(&input)
+}
+
+func (l *library) Shutdown() Ret {
+	ret := nvSandboxUtilsShutdown()
+	if ret != SUCCESS {
+		return ret
+	}
+
+	err := l.close()
+	if err != nil {
+		return ERROR_UNKNOWN
+	}
+
+	return ret
+}
+
+// TODO: Is this length specified in the header file?
+const VERSION_LENGTH = 100
+
+func (l *library) GetDriverVersion() (string, Ret) {
+	Version := make([]byte, VERSION_LENGTH)
+	ret := nvSandboxUtilsGetDriverVersion(&Version[0], VERSION_LENGTH)
+	return string(Version[:clen(Version)]), ret
+}
+
+func (l *library) GetFileContent(path string) (string, Ret) {
+	Content := make([]byte, MAX_FILE_PATH)
+	FilePath := []byte(path + string(byte(0)))
+	Size := uint32(MAX_FILE_PATH)
+	ret := nvSandboxUtilsGetFileContent(&FilePath[0], &Content[0], &Size)
+	return string(Content[:clen(Content)]), ret
+}

internal/nvsandboxutils/lib.go

@@ -0,0 +1,156 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+const (
+	defaultNvSandboxUtilsLibraryName      = "libnvidia-sandboxutils.so.1"
+	defaultNvSandboxUtilsLibraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+var errLibraryNotLoaded = errors.New("library not loaded")
+var errLibraryAlreadyLoaded = errors.New("library already loaded")
+
+// dynamicLibrary is an interface for abstacting the underlying library.
+// This also allows for mocking and testing.
+
+//go:generate moq -rm -stub -out dynamicLibrary_mock.go . dynamicLibrary
+type dynamicLibrary interface {
+	Lookup(string) error
+	Open() error
+	Close() error
+}
+
+// library represents an nvsandboxutils library.
+// This includes a reference to the underlying DynamicLibrary
+type library struct {
+	sync.Mutex
+	path     string
+	refcount refcount
+	dl       dynamicLibrary
+}
+
+// libnvsandboxutils is a global instance of the nvsandboxutils library.
+var libnvsandboxutils = newLibrary()
+
+func New(opts ...LibraryOption) Interface {
+	return newLibrary(opts...)
+}
+
+func newLibrary(opts ...LibraryOption) *library {
+	l := &library{}
+	l.init(opts...)
+	return l
+}
+
+func (l *library) init(opts ...LibraryOption) {
+	o := libraryOptions{}
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	if o.path == "" {
+		o.path = defaultNvSandboxUtilsLibraryName
+	}
+	if o.flags == 0 {
+		o.flags = defaultNvSandboxUtilsLibraryLoadFlags
+	}
+
+	l.path = o.path
+	l.dl = dl.New(o.path, o.flags)
+}
+
+// LookupSymbol checks whether the specified library symbol exists in the library.
+// Note that this requires that the library be loaded.
+func (l *library) LookupSymbol(name string) error {
+	if l == nil || l.refcount == 0 {
+		return fmt.Errorf("error looking up %s: %w", name, errLibraryNotLoaded)
+	}
+	return l.dl.Lookup(name)
+}
+
+// load initializes the library and updates the versioned symbols.
+// Multiple calls to an already loaded library will return without error.
+func (l *library) load() (rerr error) {
+	l.Lock()
+	defer l.Unlock()
+
+	defer func() { l.refcount.IncOnNoError(rerr) }()
+	if l.refcount > 0 {
+		return nil
+	}
+
+	if err := l.dl.Open(); err != nil {
+		return fmt.Errorf("error opening %s: %w", l.path, err)
+	}
+
+	// Update the errorStringFunc to point to nvsandboxutils.ErrorString
+	errorStringFunc = nvsanboxutilsErrorString
+
+	// Update all versioned symbols
+	l.updateVersionedSymbols()
+
+	return nil
+}
+
+// close the underlying library and ensure that the global pointer to the
+// library is set to nil to ensure that subsequent calls to open will reinitialize it.
+// Multiple calls to an already closed nvsandboxutils library will return without error.
+func (l *library) close() (rerr error) {
+	l.Lock()
+	defer l.Unlock()
+
+	defer func() { l.refcount.DecOnNoError(rerr) }()
+	if l.refcount != 1 {
+		return nil
+	}
+
+	if err := l.dl.Close(); err != nil {
+		return fmt.Errorf("error closing %s: %w", l.path, err)
+	}
+
+	// Update the errorStringFunc to point to defaultErrorStringFunc
+	errorStringFunc = defaultErrorStringFunc
+
+	return nil
+}
+
+// Default all versioned APIs to v1 (to infer the types)
+var (
+// Insert default versions for APIs here.
+// Example:
+// nvsandboxUtilsFunction = nvsandboxUtilsFunction_v1
+)
+
+// updateVersionedSymbols checks for versioned symbols in the loaded dynamic library.
+// If newer versioned symbols exist, these replace the default `v1` symbols initialized above.
+// When new versioned symbols are added, these would have to be initialized above and have
+// corresponding checks and subsequent assignments added below.
+func (l *library) updateVersionedSymbols() {
+	// Example:
+	// err := l.dl.Lookup("nvsandboxUtilsFunction_v2")
+	// if err == nil {
+	// 	nvsandboxUtilsFunction = nvsandboxUtilsFunction_v2
+	// }
+}

internal/nvsandboxutils/lib_test.go

@@ -0,0 +1,245 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func newTestLibrary(dl dynamicLibrary) *library {
+	return &library{dl: dl}
+}
+
+func TestLookupFromDefault(t *testing.T) {
+	errClose := errors.New("close error")
+	errOpen := errors.New("open error")
+	errLookup := errors.New("lookup error")
+
+	testCases := []struct {
+		description          string
+		dl                   dynamicLibrary
+		skipLoadLibrary      bool
+		expectedLoadError    error
+		expectedLookupErrror error
+		expectedCloseError   error
+	}{
+		{
+			description:          "library not loaded yields error",
+			dl:                   &dynamicLibraryMock{},
+			skipLoadLibrary:      true,
+			expectedLookupErrror: errLibraryNotLoaded,
+		},
+		{
+			description: "open error is returned",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return errOpen
+				},
+			},
+
+			expectedLoadError:    errOpen,
+			expectedLookupErrror: errLibraryNotLoaded,
+		},
+		{
+			description: "lookup error is returned",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return nil
+				},
+				LookupFunc: func(s string) error {
+					return fmt.Errorf("%w: %s", errLookup, s)
+				},
+				CloseFunc: func() error {
+					return nil
+				},
+			},
+
+			expectedLookupErrror: errLookup,
+		},
+		{
+			description: "lookup succeeds",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return nil
+				},
+				LookupFunc: func(s string) error {
+					return nil
+				},
+				CloseFunc: func() error {
+					return nil
+				},
+			},
+		},
+		{
+			description: "lookup succeeds",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return nil
+				},
+				LookupFunc: func(s string) error {
+					return nil
+				},
+				CloseFunc: func() error {
+					return nil
+				},
+			},
+		},
+		{
+			description: "close error is returned",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return nil
+				},
+				LookupFunc: func(s string) error {
+					return nil
+				},
+				CloseFunc: func() error {
+					return errClose
+				},
+			},
+			expectedCloseError: errClose,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			l := newTestLibrary(tc.dl)
+			if !tc.skipLoadLibrary {
+				require.ErrorIs(t, l.load(), tc.expectedLoadError)
+			}
+			require.ErrorIs(t, l.LookupSymbol("symbol"), tc.expectedLookupErrror)
+			require.ErrorIs(t, l.close(), tc.expectedCloseError)
+			if tc.expectedCloseError == nil {
+				require.Equal(t, 0, int(l.refcount))
+			} else {
+				require.Equal(t, 1, int(l.refcount))
+			}
+		})
+	}
+}
+
+func TestLoadAndCloseNesting(t *testing.T) {
+	dl := &dynamicLibraryMock{
+		OpenFunc: func() error {
+			return nil
+		},
+		CloseFunc: func() error {
+			return nil
+		},
+	}
+
+	l := newTestLibrary(dl)
+
+	// When calling close before opening the library nothing happens.
+	require.Equal(t, 0, len(dl.calls.Close))
+	require.Nil(t, l.close())
+	require.Equal(t, 0, len(dl.calls.Close))
+
+	// When calling load twice, the library was only opened once
+	require.Equal(t, 0, len(dl.calls.Open))
+	require.Nil(t, l.load())
+	require.Equal(t, 1, len(dl.calls.Open))
+	require.Nil(t, l.load())
+	require.Equal(t, 1, len(dl.calls.Open))
+
+	// Only after calling close twice, was the library closed
+	require.Equal(t, 0, len(dl.calls.Close))
+	require.Nil(t, l.close())
+	require.Equal(t, 0, len(dl.calls.Close))
+	require.Nil(t, l.close())
+	require.Equal(t, 1, len(dl.calls.Close))
+
+	// Calling close again doesn't attempt to close the library again
+	require.Nil(t, l.close())
+	require.Equal(t, 1, len(dl.calls.Close))
+}
+
+func TestLoadAndCloseWithErrors(t *testing.T) {
+	testCases := []struct {
+		description           string
+		dl                    dynamicLibrary
+		expectedLoadRefcount  refcount
+		expectedCloseRefcount refcount
+	}{
+		{
+			description: "regular flow",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return nil
+				},
+				CloseFunc: func() error {
+					return nil
+				},
+			},
+			expectedLoadRefcount:  1,
+			expectedCloseRefcount: 0,
+		},
+		{
+			description: "open error",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return errors.New("")
+				},
+				CloseFunc: func() error {
+					return nil
+				},
+			},
+			expectedLoadRefcount:  0,
+			expectedCloseRefcount: 0,
+		},
+		{
+			description: "close error",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return nil
+				},
+				CloseFunc: func() error {
+					return errors.New("")
+				},
+			},
+			expectedLoadRefcount:  1,
+			expectedCloseRefcount: 1,
+		},
+		{
+			description: "open and close error",
+			dl: &dynamicLibraryMock{
+				OpenFunc: func() error {
+					return errors.New("")
+				},
+				CloseFunc: func() error {
+					return errors.New("")
+				},
+			},
+			expectedLoadRefcount:  0,
+			expectedCloseRefcount: 0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			l := newTestLibrary(tc.dl)
+			_ = l.load()
+			require.Equal(t, tc.expectedLoadRefcount, l.refcount)
+			_ = l.close()
+			require.Equal(t, tc.expectedCloseRefcount, l.refcount)
+		})
+	}
+}

internal/nvsandboxutils/mock/interface.go

@@ -0,0 +1,325 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package mock
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils"
+	"sync"
+)
+
+// Ensure, that Interface does implement nvsandboxutils.Interface.
+// If this is not the case, regenerate this file with moq.
+var _ nvsandboxutils.Interface = &Interface{}
+
+// Interface is a mock implementation of nvsandboxutils.Interface.
+//
+//	func TestSomethingThatUsesInterface(t *testing.T) {
+//
+//		// make and configure a mocked nvsandboxutils.Interface
+//		mockedInterface := &Interface{
+//			ErrorStringFunc: func(ret nvsandboxutils.Ret) string {
+//				panic("mock out the ErrorString method")
+//			},
+//			GetDriverVersionFunc: func() (string, nvsandboxutils.Ret) {
+//				panic("mock out the GetDriverVersion method")
+//			},
+//			GetFileContentFunc: func(s string) (string, nvsandboxutils.Ret) {
+//				panic("mock out the GetFileContent method")
+//			},
+//			GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) {
+//				panic("mock out the GetGpuResource method")
+//			},
+//			InitFunc: func(s string) nvsandboxutils.Ret {
+//				panic("mock out the Init method")
+//			},
+//			LookupSymbolFunc: func(s string) error {
+//				panic("mock out the LookupSymbol method")
+//			},
+//			ShutdownFunc: func() nvsandboxutils.Ret {
+//				panic("mock out the Shutdown method")
+//			},
+//		}
+//
+//		// use mockedInterface in code that requires nvsandboxutils.Interface
+//		// and then make assertions.
+//
+//	}
+type Interface struct {
+	// ErrorStringFunc mocks the ErrorString method.
+	ErrorStringFunc func(ret nvsandboxutils.Ret) string
+
+	// GetDriverVersionFunc mocks the GetDriverVersion method.
+	GetDriverVersionFunc func() (string, nvsandboxutils.Ret)
+
+	// GetFileContentFunc mocks the GetFileContent method.
+	GetFileContentFunc func(s string) (string, nvsandboxutils.Ret)
+
+	// GetGpuResourceFunc mocks the GetGpuResource method.
+	GetGpuResourceFunc func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret)
+
+	// InitFunc mocks the Init method.
+	InitFunc func(s string) nvsandboxutils.Ret
+
+	// LookupSymbolFunc mocks the LookupSymbol method.
+	LookupSymbolFunc func(s string) error
+
+	// ShutdownFunc mocks the Shutdown method.
+	ShutdownFunc func() nvsandboxutils.Ret
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// ErrorString holds details about calls to the ErrorString method.
+		ErrorString []struct {
+			// Ret is the ret argument value.
+			Ret nvsandboxutils.Ret
+		}
+		// GetDriverVersion holds details about calls to the GetDriverVersion method.
+		GetDriverVersion []struct {
+		}
+		// GetFileContent holds details about calls to the GetFileContent method.
+		GetFileContent []struct {
+			// S is the s argument value.
+			S string
+		}
+		// GetGpuResource holds details about calls to the GetGpuResource method.
+		GetGpuResource []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Init holds details about calls to the Init method.
+		Init []struct {
+			// S is the s argument value.
+			S string
+		}
+		// LookupSymbol holds details about calls to the LookupSymbol method.
+		LookupSymbol []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Shutdown holds details about calls to the Shutdown method.
+		Shutdown []struct {
+		}
+	}
+	lockErrorString      sync.RWMutex
+	lockGetDriverVersion sync.RWMutex
+	lockGetFileContent   sync.RWMutex
+	lockGetGpuResource   sync.RWMutex
+	lockInit             sync.RWMutex
+	lockLookupSymbol     sync.RWMutex
+	lockShutdown         sync.RWMutex
+}
+
+// ErrorString calls ErrorStringFunc.
+func (mock *Interface) ErrorString(ret nvsandboxutils.Ret) string {
+	if mock.ErrorStringFunc == nil {
+		panic("Interface.ErrorStringFunc: method is nil but Interface.ErrorString was just called")
+	}
+	callInfo := struct {
+		Ret nvsandboxutils.Ret
+	}{
+		Ret: ret,
+	}
+	mock.lockErrorString.Lock()
+	mock.calls.ErrorString = append(mock.calls.ErrorString, callInfo)
+	mock.lockErrorString.Unlock()
+	return mock.ErrorStringFunc(ret)
+}
+
+// ErrorStringCalls gets all the calls that were made to ErrorString.
+// Check the length with:
+//
+//	len(mockedInterface.ErrorStringCalls())
+func (mock *Interface) ErrorStringCalls() []struct {
+	Ret nvsandboxutils.Ret
+} {
+	var calls []struct {
+		Ret nvsandboxutils.Ret
+	}
+	mock.lockErrorString.RLock()
+	calls = mock.calls.ErrorString
+	mock.lockErrorString.RUnlock()
+	return calls
+}
+
+// GetDriverVersion calls GetDriverVersionFunc.
+func (mock *Interface) GetDriverVersion() (string, nvsandboxutils.Ret) {
+	if mock.GetDriverVersionFunc == nil {
+		panic("Interface.GetDriverVersionFunc: method is nil but Interface.GetDriverVersion was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockGetDriverVersion.Lock()
+	mock.calls.GetDriverVersion = append(mock.calls.GetDriverVersion, callInfo)
+	mock.lockGetDriverVersion.Unlock()
+	return mock.GetDriverVersionFunc()
+}
+
+// GetDriverVersionCalls gets all the calls that were made to GetDriverVersion.
+// Check the length with:
+//
+//	len(mockedInterface.GetDriverVersionCalls())
+func (mock *Interface) GetDriverVersionCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockGetDriverVersion.RLock()
+	calls = mock.calls.GetDriverVersion
+	mock.lockGetDriverVersion.RUnlock()
+	return calls
+}
+
+// GetFileContent calls GetFileContentFunc.
+func (mock *Interface) GetFileContent(s string) (string, nvsandboxutils.Ret) {
+	if mock.GetFileContentFunc == nil {
+		panic("Interface.GetFileContentFunc: method is nil but Interface.GetFileContent was just called")
+	}
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockGetFileContent.Lock()
+	mock.calls.GetFileContent = append(mock.calls.GetFileContent, callInfo)
+	mock.lockGetFileContent.Unlock()
+	return mock.GetFileContentFunc(s)
+}
+
+// GetFileContentCalls gets all the calls that were made to GetFileContent.
+// Check the length with:
+//
+//	len(mockedInterface.GetFileContentCalls())
+func (mock *Interface) GetFileContentCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockGetFileContent.RLock()
+	calls = mock.calls.GetFileContent
+	mock.lockGetFileContent.RUnlock()
+	return calls
+}
+
+// GetGpuResource calls GetGpuResourceFunc.
+func (mock *Interface) GetGpuResource(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) {
+	if mock.GetGpuResourceFunc == nil {
+		panic("Interface.GetGpuResourceFunc: method is nil but Interface.GetGpuResource was just called")
+	}
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockGetGpuResource.Lock()
+	mock.calls.GetGpuResource = append(mock.calls.GetGpuResource, callInfo)
+	mock.lockGetGpuResource.Unlock()
+	return mock.GetGpuResourceFunc(s)
+}
+
+// GetGpuResourceCalls gets all the calls that were made to GetGpuResource.
+// Check the length with:
+//
+//	len(mockedInterface.GetGpuResourceCalls())
+func (mock *Interface) GetGpuResourceCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockGetGpuResource.RLock()
+	calls = mock.calls.GetGpuResource
+	mock.lockGetGpuResource.RUnlock()
+	return calls
+}
+
+// Init calls InitFunc.
+func (mock *Interface) Init(s string) nvsandboxutils.Ret {
+	if mock.InitFunc == nil {
+		panic("Interface.InitFunc: method is nil but Interface.Init was just called")
+	}
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockInit.Lock()
+	mock.calls.Init = append(mock.calls.Init, callInfo)
+	mock.lockInit.Unlock()
+	return mock.InitFunc(s)
+}
+
+// InitCalls gets all the calls that were made to Init.
+// Check the length with:
+//
+//	len(mockedInterface.InitCalls())
+func (mock *Interface) InitCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockInit.RLock()
+	calls = mock.calls.Init
+	mock.lockInit.RUnlock()
+	return calls
+}
+
+// LookupSymbol calls LookupSymbolFunc.
+func (mock *Interface) LookupSymbol(s string) error {
+	if mock.LookupSymbolFunc == nil {
+		panic("Interface.LookupSymbolFunc: method is nil but Interface.LookupSymbol was just called")
+	}
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockLookupSymbol.Lock()
+	mock.calls.LookupSymbol = append(mock.calls.LookupSymbol, callInfo)
+	mock.lockLookupSymbol.Unlock()
+	return mock.LookupSymbolFunc(s)
+}
+
+// LookupSymbolCalls gets all the calls that were made to LookupSymbol.
+// Check the length with:
+//
+//	len(mockedInterface.LookupSymbolCalls())
+func (mock *Interface) LookupSymbolCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockLookupSymbol.RLock()
+	calls = mock.calls.LookupSymbol
+	mock.lockLookupSymbol.RUnlock()
+	return calls
+}
+
+// Shutdown calls ShutdownFunc.
+func (mock *Interface) Shutdown() nvsandboxutils.Ret {
+	if mock.ShutdownFunc == nil {
+		panic("Interface.ShutdownFunc: method is nil but Interface.Shutdown was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockShutdown.Lock()
+	mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo)
+	mock.lockShutdown.Unlock()
+	return mock.ShutdownFunc()
+}
+
+// ShutdownCalls gets all the calls that were made to Shutdown.
+// Check the length with:
+//
+//	len(mockedInterface.ShutdownCalls())
+func (mock *Interface) ShutdownCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockShutdown.RLock()
+	calls = mock.calls.Shutdown
+	mock.lockShutdown.RUnlock()
+	return calls
+}

internal/nvsandboxutils/nvsandboxutils.go

@@ -0,0 +1,72 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+// WARNING: THIS FILE WAS AUTOMATICALLY GENERATED.
+// Code generated by https://git.io/c-for-go. DO NOT EDIT.
+
+package nvsandboxutils
+
+/*
+#cgo linux LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup
+#include "nvsandboxutils.h"
+#include <stdlib.h>
+#include "cgo_helpers.h"
+*/
+import "C"
+import "unsafe"
+
+// nvSandboxUtilsInit function as declared in nvsandboxutils/nvsandboxutils.h
+func nvSandboxUtilsInit(Input *InitInput) Ret {
+	cInput, _ := (*C.nvSandboxUtilsInitInput_t)(unsafe.Pointer(Input)), cgoAllocsUnknown
+	__ret := C.nvSandboxUtilsInit(cInput)
+	__v := (Ret)(__ret)
+	return __v
+}
+
+// nvSandboxUtilsShutdown function as declared in nvsandboxutils/nvsandboxutils.h
+func nvSandboxUtilsShutdown() Ret {
+	__ret := C.nvSandboxUtilsShutdown()
+	__v := (Ret)(__ret)
+	return __v
+}
+
+// nvSandboxUtilsGetDriverVersion function as declared in nvsandboxutils/nvsandboxutils.h
+func nvSandboxUtilsGetDriverVersion(Version *byte, Length uint32) Ret {
+	cVersion, _ := (*C.char)(unsafe.Pointer(Version)), cgoAllocsUnknown
+	cLength, _ := (C.uint)(Length), cgoAllocsUnknown
+	__ret := C.nvSandboxUtilsGetDriverVersion(cVersion, cLength)
+	__v := (Ret)(__ret)
+	return __v
+}
+
+// nvSandboxUtilsGetGpuResource function as declared in nvsandboxutils/nvsandboxutils.h
+func nvSandboxUtilsGetGpuResource(Request *GpuRes) Ret {
+	cRequest, _ := (*C.nvSandboxUtilsGpuRes_t)(unsafe.Pointer(Request)), cgoAllocsUnknown
+	__ret := C.nvSandboxUtilsGetGpuResource(cRequest)
+	__v := (Ret)(__ret)
+	return __v
+}
+
+// nvSandboxUtilsGetFileContent function as declared in nvsandboxutils/nvsandboxutils.h
+func nvSandboxUtilsGetFileContent(FilePath *byte, Content *byte, ContentSize *uint32) Ret {
+	cFilePath, _ := (*C.char)(unsafe.Pointer(FilePath)), cgoAllocsUnknown
+	cContent, _ := (*C.char)(unsafe.Pointer(Content)), cgoAllocsUnknown
+	cContentSize, _ := (*C.uint)(unsafe.Pointer(ContentSize)), cgoAllocsUnknown
+	__ret := C.nvSandboxUtilsGetFileContent(cFilePath, cContent, cContentSize)
+	__v := (Ret)(__ret)
+	return __v
+}

internal/nvsandboxutils/nvsandboxutils.h

@@ -0,0 +1,298 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __NVSANDBOXUTILS_H__
+#define __NVSANDBOXUTILS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define INPUT_LENGTH 256
+#define MAX_FILE_PATH 256
+#define MAX_NAME_LENGTH 256
+
+/***************************************************************************************************/
+/** @defgroup enums Enumerations
+ *  @{
+ */
+/***************************************************************************************************/
+
+/**
+ * Return types
+ */
+typedef enum
+{
+    NVSANDBOXUTILS_SUCCESS                              = 0,         //!< The operation was successful
+    NVSANDBOXUTILS_ERROR_UNINITIALIZED                  = 1,         //!< The library wasn't successfully initialized
+    NVSANDBOXUTILS_ERROR_NOT_SUPPORTED                  = 2,         //!< The requested operation is not supported on target device
+    NVSANDBOXUTILS_ERROR_INVALID_ARG                    = 3,         //!< A supplied argument is invalid
+    NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE              = 4,         //!< A supplied argument is not large enough
+    NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED          = 5,         //!< Requested library version is not supported
+    NVSANDBOXUTILS_ERROR_LIBRARY_LOAD                   = 6,         //!< The library load failed
+    NVSANDBOXUTILS_ERROR_FUNCTION_NOT_FOUND             = 7,         //!< Called function was not found
+    NVSANDBOXUTILS_ERROR_DEVICE_NOT_FOUND               = 8,         //!< Target device was not found
+    NVSANDBOXUTILS_ERROR_NVML_LIB_CALL                  = 9,         //!< NVML library call failed
+    NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY                  = 10,        //!< There is insufficient memory
+    NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND             = 11,        //!< A supplied file path was not found
+    NVSANDBOXUTILS_ERROR_UNKNOWN                        = 0xFFFF,    //!< Unknown error occurred
+} nvSandboxUtilsRet_t;
+
+/**
+ * Return if there is an error
+ */
+#define RETURN_ON_SANDBOX_ERROR(result) \
+    if ((result) != NVSANDBOXUTILS_SUCCESS) { \
+        NVSANDBOXUTILS_ERROR_MSG("%s %d result=%d", __func__, __LINE__, result); \
+        return result; \
+    }
+
+/**
+ * Log levels
+ */
+typedef enum
+{
+    NVSANDBOXUTILS_LOG_LEVEL_FATAL         = 0,                       //!< Log fatal errors
+    NVSANDBOXUTILS_LOG_LEVEL_ERROR         = 1,                       //!< Log all errors
+    NVSANDBOXUTILS_LOG_LEVEL_WARN          = 2,                       //!< Log all warnings
+    NVSANDBOXUTILS_LOG_LEVEL_DEBUG         = 3,                       //!< Log all debug messages
+    NVSANDBOXUTILS_LOG_LEVEL_INFO          = 4,                       //!< Log all info messages
+    NVSANDBOXUTILS_LOG_LEVEL_NONE          = 0xFFFF,                  //!< Log none
+} nvSandboxUtilsLogLevel_t;
+
+/**
+ * Input rootfs to help access files inside the driver container
+ */
+typedef enum
+{
+    NV_ROOTFS_DEFAULT,                                               //!< Default no rootfs
+    NV_ROOTFS_PATH,                                                  //!< /run/nvidia/driver
+    NV_ROOTFS_PID,                                                   //!< /proc/PID/mountinfo
+} nvSandboxUtilsRootfsInputType_t;
+
+/**
+ * File type
+ */
+typedef enum
+{
+    NV_DEV,                                                          //!< /dev file system
+    NV_PROC,                                                         //!< /proc file system
+    NV_SYS,                                                          //!< /sys file system
+} nvSandboxUtilsFileType_t;
+
+/**
+ * File subtype
+ */
+typedef enum
+{
+    NV_DEV_NVIDIA,                                                   //!< /dev/nvidia0
+    NV_DEV_DRI_CARD,                                                 //!< /dev/dri/card1
+    NV_DEV_DRI_RENDERD,                                              //!< /dev/dri/renderD128
+    NV_DEV_DRI_CARD_SYMLINK,                                         //!< /dev/dri/by-path/pci-0000:41:00.0-card
+    NV_DEV_DRI_RENDERD_SYMLINK,                                      //!< /dev/dri/by-path/pci-0000:41:00.0-render
+    NV_DEV_NVIDIA_UVM,                                               //!< /dev/nvidia-uvm
+    NV_DEV_NVIDIA_UVM_TOOLS,                                         //!< /dev/nvidia-uvm-tools
+    NV_DEV_NVIDIA_MODESET,                                           //!< /dev/nvidia-uvm-modeset
+    NV_DEV_NVIDIA_CTL,                                               //!< /dev/nvidiactl
+    NV_DEV_GDRDRV,                                                   //!< /dev/gdrdrv
+    NV_DEV_NVIDIA_CAPS_NVIDIA_CAP,                                   //!< /dev/nvidia-caps/nvidia-cap22
+    NV_PROC_DRIVER_NVIDIA_GPUS_PCIBUSID,                             //!< /proc/driver/nvidia/gpus/0000:2d:00.0
+    NV_PROC_DRIVER_NVIDIA_GPUS,                                      //!< /proc/driver/nvidia/gpus (for mask out)
+    NV_PROC_NVIDIA_PARAMS,                                           //!< /proc/driver/nvidia/params
+    NV_PROC_NVIDIA_CAPS_MIG_MINORS,                                  //!< /proc/driver/nvidia-caps/mig-minors
+    NV_PROC_DRIVER_NVIDIA_CAPABILITIES_GPU,                          //!< /proc/driver/nvidia/capabilities/gpu0
+    NV_PROC_DRIVER_NVIDIA_CAPABILITIES,                              //!< /proc/driver/nvidia/capabilities (for mask out)
+    NV_PROC_DRIVER_NVIDIA_CAPABILITIIES_GPU_MIG_CI_ACCESS,           //!< proc/driver/nvidia/capabilities/gpu0/mig/gi2/ci0/access
+    NV_SYS_MODULE_NVIDIA_DRIVER_PCIBUSID,                            //!< /sys/module/nvidia/drivers/pci:nvidia/0000:2d:00.0
+    NV_SYS_MODULE_NVIDIA_DRIVER,                                     //!< /sys/module/nvidia/drivers/pci:nvidia (for mask out)
+    NV_NUM_SUBTYPE, // always at the end.
+} nvSandboxUtilsFileSystemSubType_t;
+
+/**
+ * File module
+ */
+typedef enum
+{
+    NV_GPU,                                                           //!< Target device
+    NV_MIG,                                                           //!< Target device- MIG
+    NV_DRIVER_NVIDIA,                                                 //!< NVIDIA kernel driver
+    NV_DRIVER_NVIDIA_UVM,                                             //!< NVIDIA kernel driver-UVM
+    NV_DRIVER_NVIDIA_MODESET,                                         //!< NVIDIA kernel driver-modeset
+    NV_DRIVER_GDRDRV,                                                 //!< GDRDRV driver
+    NV_SYSTEM,                                                        //!< System module
+} nvSandboxUtilsFileModule_t;
+
+/**
+ * Flag to provide additional details about the file
+ */
+typedef enum
+{
+    NV_FILE_FLAG_HINT            = (1 << 0),                         //!< Default no hint
+    NV_FILE_FLAG_MASKOUT         = (1 << 1),                         //!< For /proc/driver/nvidia/gpus
+    NV_FILE_FLAG_CONTENT         = (1 << 2),                         //!< For /proc/driver/nvidia/params
+                                                                     //!< For SYMLINK
+                                                                     //!< Use \p nvSandboxUtilsGetFileContent to get name of the linked file
+    NV_FILE_FLAG_DEPRECTATED     = (1 << 3),                         //!< For all the FIRMWARE GSP file
+    NV_FILE_FLAG_CANDIDATES      = (1 << 4),                         //!< For libcuda.so
+} nvSandboxUtilsFileFlag_t;
+
+/**
+ * Input type of the target device
+ */
+typedef enum
+{
+    NV_GPU_INPUT_GPU_UUID,                                           //!< GPU UUID
+    NV_GPU_INPUT_MIG_UUID,                                           //!< MIG UUID
+    NV_GPU_INPUT_PCI_ID,                                             //!< PCIe DBDF ID
+    NV_GPU_INPUT_PCI_INDEX,                                          //!< PCIe bus order (0 points to the GPU that has lowest PCIe BDF)
+} nvSandboxUtilsGpuInputType_t;
+
+/** @} */
+
+/***************************************************************************************************/
+/** @defgroup dataTypes Structures and Unions
+ *  @{
+ */
+/***************************************************************************************************/
+
+/**
+ * Initalization input v1
+ */
+typedef struct
+{
+    unsigned int version;                                            //!< Version for the structure
+    nvSandboxUtilsRootfsInputType_t type;                            //!< One of \p nvSandboxUtilsRootfsInputType_t
+    char value[INPUT_LENGTH];                                        //!< String representation of input
+} nvSandboxUtilsInitInput_v1_t;
+
+typedef nvSandboxUtilsInitInput_v1_t nvSandboxUtilsInitInput_t;
+
+/**
+ * File system information
+ */
+typedef struct nvSandboxUtilsGpuFileInfo_v1_t
+{
+    struct nvSandboxUtilsGpuFileInfo_v1_t *next;                     //!< Pointer to the next node in the linked list
+    nvSandboxUtilsFileType_t fileType;                               //!< One of \p nvSandboxUtilsFileType_t
+    nvSandboxUtilsFileSystemSubType_t fileSubType;                   //!< One of \p nvSandboxUtilsFileSystemSubType_t
+    nvSandboxUtilsFileModule_t module;                               //!< One of \p nvSandboxUtilsFileModule_t
+    nvSandboxUtilsFileFlag_t flags;                                  //!< One of \p nvSandboxUtilsFileFlag_t
+    char *filePath;                                                  //!< Relative file path to rootfs
+}nvSandboxUtilsGpuFileInfo_v1_t;
+
+/**
+ * GPU resource request v1
+ */
+typedef struct
+{
+     unsigned int version;                                           //!< Version for the structure
+     nvSandboxUtilsGpuInputType_t inputType;                         //!< One of \p nvSandboxUtilsGpuInputType_t
+     char input[INPUT_LENGTH];                                       //!< String representation of input
+     nvSandboxUtilsGpuFileInfo_v1_t *files;                          //!< Linked list of \ref nvSandboxUtilsGpuFileInfo_v1_t
+} nvSandboxUtilsGpuRes_v1_t;
+
+typedef nvSandboxUtilsGpuRes_v1_t nvSandboxUtilsGpuRes_t;
+
+/** @} */
+
+/***************************************************************************************************/
+/** @defgroup funcs Functions
+ *  @{
+ */
+/***************************************************************************************************/
+
+/* *************************************************
+ * Initialize library
+ * *************************************************
+ */
+/**
+ * Prepare library resources before library API can be used.
+ * This initialization will not fail if one of the initialization prerequisites fails.
+ * @param           input  Reference to the called-supplied input struct that has initialization fields
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p input->value isn't a valid rootfs path
+ * @returns         @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED  if \p input->version isn't supported by the library
+ * @returns         @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND     if any of the required file paths are not found during initialization
+ * @returns         @ref NVSANDBOXUTILS_ERROR_OUT_OF_MEMORY          if there is insufficient system memory during initialization
+ * @returns         @ref NVSANDBOXUTILS_ERROR_LIBRARY_LOAD           on any error during loading the library
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsInit(nvSandboxUtilsInitInput_t *input);
+
+/* *************************************************
+ * Shutdown library
+ * *************************************************
+ */
+/**
+ * Clean up library resources created by init call
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsShutdown(void);
+
+/* *************************************************
+ * Get NVIDIA RM driver version
+ * *************************************************
+ */
+/**
+ * Get NVIDIA RM driver version
+ * @param           version  Reference to caller-supplied buffer to return driver version string
+ * @param           length   The maximum allowed length of the string returned in \p version
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p version is NULL
+ * @returns         @ref NVSANDBOXUTILS_ERROR_NVML_LIB_CALL          on any error during driver version query from NVML
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsGetDriverVersion(char *version, unsigned int length);
+
+/* *************************************************
+ * Get /dev, /proc, /sys file system information
+ * *************************************************
+ */
+/**
+ * Get /dev, /proc, /sys file system information
+ * @param           request  Reference to caller-supplied request struct to return the file system information
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p request->input doesn't match any device
+ * @returns         @ref NVSANDBOXUTILS_ERROR_VERSION_NOT_SUPPORTED  if \p request->version isn't supported by the library
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsGetGpuResource(nvSandboxUtilsGpuRes_t *request);
+
+/* *************************************************
+ * Get content of given file path
+ * *************************************************
+ */
+/**
+ * Get file content of input file path
+ * @param           filePath     Reference to the file path
+ * @param           content      Reference to the caller-supplied buffer to return the file content
+ * @param           contentSize  Reference to the maximum allowed size of content. It is updated to the actual size of the content on return
+ *
+ * @returns         @ref NVSANDBOXUTILS_SUCCESS                      on success
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INVALID_ARG            if \p filePath or \p content is NULL
+ * @returns         @ref NVSANDBOXUTILS_ERROR_INSUFFICIENT_SIZE      if \p contentSize is too small
+ * @returns         @ref NVSANDBOXUTILS_ERROR_FILEPATH_NOT_FOUND     on an error while obtaining the content for the file path
+ */
+nvSandboxUtilsRet_t nvSandboxUtilsGetFileContent(char *filePath, char *content, unsigned int *contentSize);
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+#endif // __NVSANDBOXUTILS_H__

internal/nvsandboxutils/refcount.go

@@ -0,0 +1,31 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+type refcount int
+
+func (r *refcount) IncOnNoError(err error) {
+	if err == nil {
+		(*r)++
+	}
+}
+
+func (r *refcount) DecOnNoError(err error) {
+	if err == nil && (*r) > 0 {
+		(*r)--
+	}
+}

internal/nvsandboxutils/refcount_test.go

@@ -0,0 +1,139 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRefcount(t *testing.T) {
+	testCases := []struct {
+		description      string
+		workload         func(r *refcount)
+		expectedRefcount refcount
+	}{
+		{
+			description:      "No inc or dec",
+			workload:         func(r *refcount) {},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Single inc, no error",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+			},
+			expectedRefcount: refcount(1),
+		},
+		{
+			description: "Single inc, with error",
+			workload: func(r *refcount) {
+				r.IncOnNoError(errors.New(""))
+			},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Double inc, no error",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.IncOnNoError(nil)
+			},
+			expectedRefcount: refcount(2),
+		},
+		{
+			description: "Double inc, one with error",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.IncOnNoError(errors.New(""))
+			},
+			expectedRefcount: refcount(1),
+		},
+		{
+			description: "Single dec, no error",
+			workload: func(r *refcount) {
+				r.DecOnNoError(nil)
+			},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Single dec, with error",
+			workload: func(r *refcount) {
+				r.DecOnNoError(errors.New(""))
+			},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Single inc, single dec, no errors",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.DecOnNoError(nil)
+			},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Double inc, Double dec, no errors",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.IncOnNoError(nil)
+				r.DecOnNoError(nil)
+				r.DecOnNoError(nil)
+			},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Double inc, Double dec, one inc error",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.IncOnNoError(errors.New(""))
+				r.DecOnNoError(nil)
+				r.DecOnNoError(nil)
+			},
+			expectedRefcount: refcount(0),
+		},
+		{
+			description: "Double inc, Double dec, one dec error",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.IncOnNoError(nil)
+				r.DecOnNoError(nil)
+				r.DecOnNoError(errors.New(""))
+			},
+			expectedRefcount: refcount(1),
+		},
+		{
+			description: "Double inc, Tripple dec, one dec error early on",
+			workload: func(r *refcount) {
+				r.IncOnNoError(nil)
+				r.IncOnNoError(nil)
+				r.DecOnNoError(errors.New(""))
+				r.DecOnNoError(nil)
+				r.DecOnNoError(nil)
+			},
+			expectedRefcount: refcount(0),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			var r refcount
+			tc.workload(&r)
+			require.Equal(t, tc.expectedRefcount, r)
+		})
+	}
+}

internal/nvsandboxutils/return.go

@@ -0,0 +1,74 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvsandboxutils
+
+import (
+	"fmt"
+)
+
+// nvsandboxutils.ErrorString()
+func (l *library) ErrorString(r Ret) string {
+	return r.Error()
+}
+
+// String returns the string representation of a Ret.
+func (r Ret) String() string {
+	return r.Error()
+}
+
+// Error returns the string representation of a Ret.
+func (r Ret) Error() string {
+	return errorStringFunc(r)
+}
+
+// Assigned to nvsandboxutils.ErrorString if the system nvsandboxutils library is in use.
+var errorStringFunc = defaultErrorStringFunc
+
+// nvsanboxutilsErrorString is an alias for the default error string function.
+var nvsanboxutilsErrorString = defaultErrorStringFunc
+
+// defaultErrorStringFunc provides a basic nvsandboxutils.ErrorString implementation.
+// This allows the nvsandboxutils.ErrorString function to be used even if the nvsandboxutils library
+// is not loaded.
+var defaultErrorStringFunc = func(r Ret) string {
+	switch r {
+	case SUCCESS:
+		return "SUCCESS"
+	case ERROR_UNINITIALIZED:
+		return "ERROR_UNINITIALIZED"
+	case ERROR_NOT_SUPPORTED:
+		return "ERROR_NOT_SUPPORTED"
+	case ERROR_INVALID_ARG:
+		return "ERROR_INVALID_ARG"
+	case ERROR_INSUFFICIENT_SIZE:
+		return "ERROR_INSUFFICIENT_SIZE"
+	case ERROR_VERSION_NOT_SUPPORTED:
+		return "ERROR_VERSION_NOT_SUPPORTED"
+	case ERROR_LIBRARY_LOAD:
+		return "ERROR_LIBRARY_LOAD"
+	case ERROR_FUNCTION_NOT_FOUND:
+		return "ERROR_FUNCTION_NOT_FOUND"
+	case ERROR_DEVICE_NOT_FOUND:
+		return "ERROR_DEVICE_NOT_FOUND"
+	case ERROR_NVML_LIB_CALL:
+		return "ERROR_NVML_LIB_CALL"
+	case ERROR_UNKNOWN:
+		return "ERROR_UNKNOWN"
+	default:
+		return fmt.Sprintf("unknown return value: %d", r)
+	}
+}

internal/nvsandboxutils/types_gen.go

@@ -0,0 +1,39 @@
+// Code generated by cmd/cgo -godefs; DO NOT EDIT.
+// cgo -godefs types.go
+
+package nvsandboxutils
+
+type InitInput_v1 struct {
+	Version uint32
+	Type    uint32
+	Value   [256]int8
+}
+
+type InitInput struct {
+	Version uint32
+	Type    uint32
+	Value   [256]int8
+}
+
+type GpuFileInfo_v1 struct {
+	Next        *GpuFileInfo_v1
+	FileType    uint32
+	FileSubType uint32
+	Module      uint32
+	Flags       uint32
+	FilePath    *int8
+}
+
+type GpuRes_v1 struct {
+	Version   uint32
+	InputType uint32
+	Input     [256]int8
+	Files     *GpuFileInfo_v1
+}
+
+type GpuRes struct {
+	Version   uint32
+	InputType uint32
+	Input     [256]int8
+	Files     *GpuFileInfo_v1
+}

internal/nvsandboxutils/zz_generated.api.go

@@ -0,0 +1,43 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+// Generated Code; DO NOT EDIT.
+
+package nvsandboxutils
+
+// The variables below represent package level methods from the library type.
+var (
+	ErrorString = libnvsandboxutils.ErrorString
+	GetDriverVersion = libnvsandboxutils.GetDriverVersion
+	GetFileContent = libnvsandboxutils.GetFileContent
+	GetGpuResource = libnvsandboxutils.GetGpuResource
+	Init = libnvsandboxutils.Init
+	LookupSymbol = libnvsandboxutils.LookupSymbol
+	Shutdown = libnvsandboxutils.Shutdown
+)
+
+// Interface represents the interface for the library type.
+//
+//go:generate moq -out mock/interface.go -pkg mock . Interface:Interface
+type Interface interface {
+	ErrorString(Ret) string
+	GetDriverVersion() (string, Ret)
+	GetFileContent(string) (string, Ret)
+	GetGpuResource(string) ([]GpuFileInfo, Ret)
+	Init(string) Ret
+	LookupSymbol(string) error
+	Shutdown() Ret
+}

internal/platform-support/dgpu/dgpu.go

@@ -17,6 +17,8 @@
 package dgpu
 
 import (
+	"errors"
+
 	"github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
@@ -25,22 +27,78 @@ import (
 )
 
 // NewForDevice creates a discoverer for the specified Device.
+// nvsandboxutils is used for discovery if specified, otherwise NVML is used.
 func NewForDevice(d device.Device, opts ...Option) (discover.Discover, error) {
 	o := new(opts...)
 
-	return o.newNvmlDGPUDiscoverer(&toRequiredInfo{d})
+	var discoverers []discover.Discover
+	var errs error
+	nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(d)
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvsandboxutilsDiscoverer != nil {
+		discoverers = append(discoverers, nvsandboxutilsDiscoverer)
+	}
+
+	nvmlDiscoverer, err := o.newNvmlDGPUDiscoverer(&toRequiredInfo{d})
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvmlDiscoverer != nil {
+		discoverers = append(discoverers, nvmlDiscoverer)
+	}
+
+	if len(discoverers) == 0 {
+		return nil, errs
+	}
+
+	return discover.WithCache(
+		discover.FirstValid(
+			discoverers...,
+		),
+	), nil
 }
 
-// NewForDevice creates a discoverer for the specified device and its associated MIG device.
+// NewForMigDevice creates a discoverer for the specified device and its associated MIG device.
+// nvsandboxutils is used for discovery if specified, otherwise NVML is used.
 func NewForMigDevice(d device.Device, mig device.MigDevice, opts ...Option) (discover.Discover, error) {
 	o := new(opts...)
+	o.isMigDevice = true
 
-	return o.newNvmlMigDiscoverer(
+	var discoverers []discover.Discover
+	var errs error
+	nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(mig)
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvsandboxutilsDiscoverer != nil {
+		discoverers = append(discoverers, nvsandboxutilsDiscoverer)
+	}
+
+	nvmlDiscoverer, err := o.newNvmlMigDiscoverer(
 		&toRequiredMigInfo{
 			MigDevice: mig,
 			parent:    &toRequiredInfo{d},
 		},
 	)
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvmlDiscoverer != nil {
+		discoverers = append(discoverers, nvmlDiscoverer)
+	}
+
+	if len(discoverers) == 0 {
+		return nil, errs
+	}
+
+	return discover.WithCache(
+		discover.FirstValid(
+			discoverers...,
+		),
+	), nil
+
 }
 
 func new(opts ...Option) *options {

internal/platform-support/dgpu/nvml_test.go

@@ -139,7 +139,7 @@ func TestNewNvmlMIGDiscoverer(t *testing.T) {
 			},
 			expectedDevices: nil,
 			expectedMounts:  nil,
-			expectedHooks:   []discover.Hook{},
+			expectedHooks:   nil,
 		},
 	}
 	for _, tc := range testCases {

internal/platform-support/dgpu/nvsandboxutils.go

@@ -0,0 +1,131 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dgpu
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils"
+)
+
+type nvsandboxutilsDGPU struct {
+	lib               nvsandboxutils.Interface
+	uuid              string
+	devRoot           string
+	isMig             bool
+	nvidiaCDIHookPath string
+	deviceLinks       []string
+}
+
+var _ discover.Discover = (*nvsandboxutilsDGPU)(nil)
+
+type UUIDer interface {
+	GetUUID() (string, nvml.Return)
+}
+
+func (o *options) newNvsandboxutilsDGPUDiscoverer(d UUIDer) (discover.Discover, error) {
+	if o.nvsandboxutilslib == nil {
+		return nil, nil
+	}
+
+	uuid, nvmlRet := d.GetUUID()
+	if nvmlRet != nvml.SUCCESS {
+		return nil, fmt.Errorf("failed to get device UUID: %w", nvmlRet)
+	}
+
+	nvd := nvsandboxutilsDGPU{
+		lib:               o.nvsandboxutilslib,
+		uuid:              uuid,
+		devRoot:           strings.TrimSuffix(filepath.Clean(o.devRoot), "/dev"),
+		isMig:             o.isMigDevice,
+		nvidiaCDIHookPath: o.nvidiaCDIHookPath,
+	}
+
+	return &nvd, nil
+}
+
+func (d *nvsandboxutilsDGPU) Devices() ([]discover.Device, error) {
+	gpuFileInfos, ret := d.lib.GetGpuResource(d.uuid)
+	if ret != nvsandboxutils.SUCCESS {
+		return nil, fmt.Errorf("failed to get GPU resource: %w", ret)
+	}
+
+	var devices []discover.Device
+	for _, info := range gpuFileInfos {
+		switch {
+		case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD:
+			if d.isMig {
+				continue
+			}
+			fallthrough
+		case info.SubType == nvsandboxutils.NV_DEV_NVIDIA, info.SubType == nvsandboxutils.NV_DEV_NVIDIA_CAPS_NVIDIA_CAP:
+			containerPath := info.Path
+			if d.devRoot != "/" {
+				containerPath = strings.TrimPrefix(containerPath, d.devRoot)
+			}
+
+			// TODO: Extend discover.Device with additional information.
+			device := discover.Device{
+				HostPath: info.Path,
+				Path:     containerPath,
+			}
+			devices = append(devices, device)
+		case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD_SYMLINK, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD_SYMLINK:
+			if d.isMig {
+				continue
+			}
+			if info.Flags == nvsandboxutils.NV_FILE_FLAG_CONTENT {
+				targetPath, ret := d.lib.GetFileContent(info.Path)
+				if ret != nvsandboxutils.SUCCESS {
+					return nil, fmt.Errorf("failed to get symlink: %w", ret)
+				}
+				d.deviceLinks = append(d.deviceLinks, fmt.Sprintf("%v::%v", targetPath, info.Path))
+			}
+		}
+	}
+
+	return devices, nil
+}
+
+// Hooks returns a hook to create the by-path symlinks for the discovered devices.
+func (d *nvsandboxutilsDGPU) Hooks() ([]discover.Hook, error) {
+	if len(d.deviceLinks) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range d.deviceLinks {
+		args = append(args, "--link", l)
+	}
+
+	hook := discover.CreateNvidiaCDIHook(
+		d.nvidiaCDIHookPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []discover.Hook{hook}, nil
+}
+
+func (d *nvsandboxutilsDGPU) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}

internal/platform-support/dgpu/nvsandboxutils_test.go

@@ -0,0 +1,174 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dgpu
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	mocknvml "github.com/NVIDIA/go-nvml/pkg/nvml/mock"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils"
+	mocknvsandboxutils "github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils/mock"
+)
+
+func TestNewNvsandboxutilsDGPUDiscoverer(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	nvmllib := &mocknvml.Interface{}
+	devicelib := device.New(
+		nvmllib,
+	)
+
+	testCases := []struct {
+		description     string
+		devRoot         string
+		device          nvml.Device
+		nvsandboxutils  nvsandboxutils.Interface
+		expectedError   error
+		expectedDevices []discover.Device
+		expectedHooks   []discover.Hook
+		expectedMounts  []discover.Mount
+	}{
+		{
+			description: "detects host devices",
+			device: &mocknvml.Device{
+				GetUUIDFunc: func() (string, nvml.Return) {
+					return "GPU-1234", nvml.SUCCESS
+				},
+			},
+			nvsandboxutils: &mocknvsandboxutils.Interface{
+				GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) {
+					infos := []nvsandboxutils.GpuFileInfo{
+						{
+							Path: "/dev/nvidia0",
+							Type: nvsandboxutils.NV_DEV,
+						},
+						{
+							Path: "/dev/nvidiactl",
+							Type: nvsandboxutils.NV_DEV,
+						},
+						{
+							Path: "/dev/nvidia-uvm",
+							Type: nvsandboxutils.NV_DEV,
+						},
+						{
+							Path: "/dev/nvidia-uvm-tools",
+							Type: nvsandboxutils.NV_DEV,
+						},
+					}
+					return infos, nvsandboxutils.SUCCESS
+				},
+			},
+			expectedDevices: []discover.Device{
+				{
+					Path:     "/dev/nvidia0",
+					HostPath: "/dev/nvidia0",
+				},
+				{
+					Path:     "/dev/nvidiactl",
+					HostPath: "/dev/nvidiactl",
+				},
+				{
+					Path:     "/dev/nvidia-uvm",
+					HostPath: "/dev/nvidia-uvm",
+				},
+				{
+					Path:     "/dev/nvidia-uvm-tools",
+					HostPath: "/dev/nvidia-uvm-tools",
+				},
+			},
+		},
+		{
+			description: "detects container devices",
+			devRoot:     "/some/root",
+			device: &mocknvml.Device{
+				GetUUIDFunc: func() (string, nvml.Return) {
+					return "GPU-1234", nvml.SUCCESS
+				},
+			},
+			nvsandboxutils: &mocknvsandboxutils.Interface{
+				GetGpuResourceFunc: func(s string) ([]nvsandboxutils.GpuFileInfo, nvsandboxutils.Ret) {
+					infos := []nvsandboxutils.GpuFileInfo{
+						{
+							Path: "/some/root/dev/nvidia0",
+							Type: nvsandboxutils.NV_DEV,
+						},
+						{
+							Path: "/some/root/dev/nvidiactl",
+							Type: nvsandboxutils.NV_DEV,
+						},
+						{
+							Path: "/some/root/dev/nvidia-uvm",
+							Type: nvsandboxutils.NV_DEV,
+						},
+						{
+							Path: "/some/root/dev/nvidia-uvm-tools",
+							Type: nvsandboxutils.NV_DEV,
+						},
+					}
+					return infos, nvsandboxutils.SUCCESS
+				},
+			},
+			expectedDevices: []discover.Device{
+				{
+					Path:     "/dev/nvidia0",
+					HostPath: "/some/root/dev/nvidia0",
+				},
+				{
+					Path:     "/dev/nvidiactl",
+					HostPath: "/some/root/dev/nvidiactl",
+				},
+				{
+					Path:     "/dev/nvidia-uvm",
+					HostPath: "/some/root/dev/nvidia-uvm",
+				},
+				{
+					Path:     "/dev/nvidia-uvm-tools",
+					HostPath: "/some/root/dev/nvidia-uvm-tools",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			o := &options{
+				logger:            logger,
+				devRoot:           tc.devRoot,
+				nvsandboxutilslib: tc.nvsandboxutils,
+			}
+
+			device, err := devicelib.NewDevice(tc.device)
+			require.NoError(t, err)
+
+			d, err := o.newNvsandboxutilsDGPUDiscoverer(device)
+			require.ErrorIs(t, err, tc.expectedError)
+
+			devices, _ := d.Devices()
+			require.EqualValues(t, tc.expectedDevices, devices)
+			hooks, _ := d.Hooks()
+			require.EqualValues(t, tc.expectedHooks, hooks)
+			mounts, _ := d.Mounts()
+			require.EqualValues(t, tc.expectedMounts, mounts)
+		})
+	}
+}

internal/platform-support/dgpu/options.go

@@ -19,6 +19,7 @@ package dgpu
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils"
 )
 
 type options struct {
@@ -26,10 +27,13 @@ type options struct {
 	devRoot           string
 	nvidiaCDIHookPath string
 
+	isMigDevice bool
 	// migCaps stores the MIG capabilities for the system.
 	// If MIG is not available, this is nil.
 	migCaps      nvcaps.MigCaps
 	migCapsError error
+
+	nvsandboxutilslib nvsandboxutils.Interface
 }
 
 type Option func(*options)
@@ -61,3 +65,10 @@ func WithMIGCaps(migCaps nvcaps.MigCaps) Option {
 		l.migCaps = migCaps
 	}
 }
+
+// WithNvsandboxuitilsLib sets the nvsandboxutils library implementation.
+func WithNvsandboxuitilsLib(nvsandboxutilslib nvsandboxutils.Interface) Option {
+	return func(l *options) {
+		l.nvsandboxutilslib = nvsandboxutilslib
+	}
+}

internal/platform-support/tegra/csv.go

@@ -49,14 +49,20 @@ func (o tegraOptions) newDiscovererFromCSVFiles() (discover.Discover, error) {
 		targetsByType[csv.MountSpecDir],
 	)
 
-	// Libraries and symlinks use the same locator.
-	libraries := discover.NewMounts(
-		o.logger,
-		o.symlinkLocator,
-		o.driverRoot,
-		targetsByType[csv.MountSpecLib],
+	// We create a discoverer for mounted libraries and add additional .so
+	// symlinks for the driver.
+	libraries := discover.WithDriverDotSoSymlinks(
+		discover.NewMounts(
+			o.logger,
+			o.symlinkLocator,
+			o.driverRoot,
+			targetsByType[csv.MountSpecLib],
+		),
+		"",
+		o.nvidiaCDIHookPath,
 	)
 
+	// We process the explicitly requested symlinks.
 	symlinkTargets := o.ignorePatterns.Apply(targetsByType[csv.MountSpecSym]...)
 	o.logger.Debugf("Filtered symlink targets: %v", symlinkTargets)
 	symlinks := discover.NewMounts(
@@ -65,7 +71,7 @@ func (o tegraOptions) newDiscovererFromCSVFiles() (discover.Discover, error) {
 		o.driverRoot,
 		symlinkTargets,
 	)
-	createSymlinks := o.createCSVSymlinkHooks(symlinkTargets, libraries)
+	createSymlinks := o.createCSVSymlinkHooks(symlinkTargets)
 
 	d := discover.Merge(
 		devices,

internal/platform-support/tegra/symlinks.go

@@ -18,8 +18,6 @@ package tegra
 
 import (
 	"fmt"
-	"path/filepath"
-	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
@@ -31,7 +29,6 @@ type symlinkHook struct {
 	logger            logger.Interface
 	nvidiaCDIHookPath string
 	targets           []string
-	mountsFrom        discover.Discover
 
 	// The following can be overridden for testing
 	symlinkChainLocator lookup.Locator
@@ -39,12 +36,11 @@ type symlinkHook struct {
 }
 
 // createCSVSymlinkHooks creates a discoverer for a hook that creates required symlinks in the container
-func (o tegraOptions) createCSVSymlinkHooks(targets []string, mounts discover.Discover) discover.Discover {
+func (o tegraOptions) createCSVSymlinkHooks(targets []string) discover.Discover {
 	return symlinkHook{
 		logger:              o.logger,
 		nvidiaCDIHookPath:   o.nvidiaCDIHookPath,
 		targets:             targets,
-		mountsFrom:          mounts,
 		symlinkChainLocator: o.symlinkChainLocator,
 		resolveSymlink:      o.resolveSymlink,
 	}
@@ -52,62 +48,12 @@ func (o tegraOptions) createCSVSymlinkHooks(targets []string, mounts discover.Di
 
 // Hooks returns a hook to create the symlinks from the required CSV files
 func (d symlinkHook) Hooks() ([]discover.Hook, error) {
-	specificLinks, err := d.getSpecificLinks()
-	if err != nil {
-		return nil, fmt.Errorf("failed to determine specific links: %v", err)
-	}
-
-	csvSymlinks := d.getCSVFileSymlinks()
-
 	return discover.CreateCreateSymlinkHook(
 		d.nvidiaCDIHookPath,
-		append(csvSymlinks, specificLinks...),
+		d.getCSVFileSymlinks(),
 	).Hooks()
 }
 
-// getSpecificLinks returns the required specic links that need to be created
-func (d symlinkHook) getSpecificLinks() ([]string, error) {
-	mounts, err := d.mountsFrom.Mounts()
-	if err != nil {
-		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
-	}
-
-	linkProcessed := make(map[string]bool)
-	var links []string
-	for _, m := range mounts {
-		var target string
-		var link string
-
-		lib := filepath.Base(m.Path)
-
-		switch {
-		case strings.HasPrefix(lib, "libcuda.so"):
-			// XXX Many applications wrongly assume that libcuda.so exists (e.g. with dlopen).
-			target = "libcuda.so.1"
-			link = "libcuda.so"
-		case strings.HasPrefix(lib, "libGLX_nvidia.so"):
-			// XXX GLVND requires this symlink for indirect GLX support.
-			target = lib
-			link = "libGLX_indirect.so.0"
-		case strings.HasPrefix(lib, "libnvidia-opticalflow.so"):
-			// XXX Fix missing symlink for libnvidia-opticalflow.so.
-			target = "libnvidia-opticalflow.so.1"
-			link = "libnvidia-opticalflow.so"
-		default:
-			continue
-		}
-		if linkProcessed[link] {
-			continue
-		}
-		linkProcessed[link] = true
-
-		linkPath := filepath.Join(filepath.Dir(m.Path), link)
-		links = append(links, fmt.Sprintf("%v::%v", target, linkPath))
-	}
-
-	return links, nil
-}
-
 // getSymlinkCandidates returns a list of symlinks that are candidates for being created.
 func (d symlinkHook) getSymlinkCandidates() []string {
 	var candidates []string

internal/runtime/runtime_factory.go

@@ -79,26 +79,27 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp
 	if err != nil {
 		return nil, err
 	}
-	// For CDI mode we make no additional modifications.
-	if mode == "cdi" {
-		return modeModifier, nil
+
+	var modifiers modifier.List
+	for _, modifierType := range supportedModifierTypes(mode) {
+		switch modifierType {
+		case "mode":
+			modifiers = append(modifiers, modeModifier)
+		case "graphics":
+			graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, image, driver)
+			if err != nil {
+				return nil, err
+			}
+			modifiers = append(modifiers, graphicsModifier)
+		case "feature-gated":
+			featureGatedModifier, err := modifier.NewFeatureGatedModifier(logger, cfg, image)
+			if err != nil {
+				return nil, err
+			}
+			modifiers = append(modifiers, featureGatedModifier)
+		}
 	}
 
-	graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, image, driver)
-	if err != nil {
-		return nil, err
-	}
-
-	featureModifier, err := modifier.NewFeatureGatedModifier(logger, cfg, image)
-	if err != nil {
-		return nil, err
-	}
-
-	modifiers := modifier.Merge(
-		modeModifier,
-		graphicsModifier,
-		featureModifier,
-	)
 	return modifiers, nil
 }
 
@@ -114,3 +115,17 @@ func newModeModifier(logger logger.Interface, mode string, cfg *config.Config, o
 
 	return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)
 }
+
+// supportedModifierTypes returns the modifiers supported for a specific runtime mode.
+func supportedModifierTypes(mode string) []string {
+	switch mode {
+	case "cdi":
+		// For CDI mode we make no additional modifications.
+		return []string{"mode"}
+	case "csv":
+		// For CSV mode we support mode and feature-gated modification.
+		return []string{"mode", "feature-gated"}
+	default:
+		return []string{"mode", "graphics", "feature-gated"}
+	}
+}

pkg/config/engine/api.go

@@ -18,9 +18,16 @@ package engine
 
 // Interface defines the API for a runtime config updater.
 type Interface interface {
+	AddRuntime(string, string, bool) error
 	DefaultRuntime() string
-	AddRuntime(string, string, bool, ...map[string]interface{}) error
-	Set(string, interface{})
+	GetRuntimeConfig(string) (RuntimeConfig, error)
 	RemoveRuntime(string) error
 	Save(string) (int64, error)
+	Set(string, interface{})
+	String() string
+}
+
+// RuntimeConfig defines the interface to query container runtime handler configuration
+type RuntimeConfig interface {
+	GetBinaryPath() string
 }

pkg/config/engine/containerd/config.go

@@ -0,0 +1,144 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+	config := *c.Tree
+
+	config.Set("version", c.Version)
+
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
+	for _, r := range runtimeNamesForConfig {
+		options := config.GetSubtreeByPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", r})
+		if options == nil {
+			continue
+		}
+		c.Logger.Debugf("using options from runtime %v: %v", r, options)
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name}, options.Copy())
+		break
+	}
+
+	if config.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name}) == nil {
+		c.Logger.Warningf("could not infer options from runtimes %v; using defaults", runtimeNamesForConfig)
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := c.getRuntimeAnnotations([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name, "options", "BinaryName"}, path)
+
+	if setAsDefault {
+		config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}, name)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
+	if c == nil || c.Tree == nil {
+		return nil, nil
+	}
+
+	config := *c.Tree
+	if !config.HasPath(path) {
+		return nil, nil
+	}
+	annotationsI, ok := config.GetPath(path).([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("invalid annotations: %v", annotationsI)
+	}
+
+	var annotations []string
+	for _, annotation := range annotationsI {
+		a, ok := annotation.(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid annotation: %v", annotation)
+		}
+		annotations = append(annotations, a)
+	}
+
+	return annotations, nil
+}
+
+// Set sets the specified containerd option.
+func (c *Config) Set(key string, value interface{}) {
+	config := *c.Tree
+	config.SetPath([]string{"plugins", c.CRIRuntimePluginName, key}, value)
+	*c.Tree = config
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	config.DeletePath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimePath := []string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name}
+	for i := 0; i < len(runtimePath); i++ {
+		if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimePath[:len(runtimePath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}

pkg/config/engine/containerd/config_test.go

@@ -19,20 +19,20 @@ package containerd
 import (
 	"testing"
 
-	"github.com/pelletier/go-toml"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 func TestAddRuntime(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 	testCases := []struct {
-		description     string
-		config          string
-		setAsDefault    bool
-		configOverrides []map[string]interface{}
-		expectedConfig  string
-		expectedError   error
+		description    string
+		config         string
+		setAsDefault   bool
+		expectedConfig string
+		expectedError  error
 	}{
 		{
 			description: "empty config not default runtime",
@@ -46,37 +46,12 @@ func TestAddRuntime(t *testing.T) {
 					privileged_without_host_devices = false
 					runtime_engine = ""
 					runtime_root = ""
-					runtime_type = ""
+					runtime_type = "io.containerd.runc.v2"
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
 						BinaryName = "/usr/bin/test"
 			`,
 			expectedError: nil,
 		},
-		{
-			description: "empty config not default runtime with overrides",
-			configOverrides: []map[string]interface{}{
-				{
-					"options": map[string]interface{}{
-						"SystemdCgroup": true,
-					},
-				},
-			},
-			expectedConfig: `
-			version = 2
-			[plugins]
-			[plugins."io.containerd.grpc.v1.cri"]
-				[plugins."io.containerd.grpc.v1.cri".containerd]
-				[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
-					privileged_without_host_devices = false
-					runtime_engine = ""
-					runtime_root = ""
-					runtime_type = ""
-					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
-						BinaryName = "/usr/bin/test"
-						SystemdCgroup = true
-			`,
-		},
 		{
 			description: "options from runc are imported",
 			config: `
@@ -162,7 +137,7 @@ func TestAddRuntime(t *testing.T) {
 				`,
 		},
 		{
-			description: "options from runc take precedence over default runtime",
+			description: "options from the default runtime take precedence over runc",
 			config: `
 			version = 2
 			[plugins]
@@ -211,11 +186,73 @@ func TestAddRuntime(t *testing.T) {
 						BinaryName = "/usr/bin/default"
 						SystemdCgroup = false
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
+					privileged_without_host_devices = false
+					runtime_engine = "defaultengine"
+					runtime_root = "defaultroot"
+					runtime_type = "defaulttype"
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
+						BinaryName = "/usr/bin/test"
+						SystemdCgroup = false
+				`,
+		},
+		{
+			description: "empty v3 spec is supported",
+			config: `
+			version = 3
+			`,
+			expectedConfig: `
+			version = 3
+			[plugins]
+			[plugins."io.containerd.cri.v1.runtime"]
+				[plugins."io.containerd.cri.v1.runtime".containerd]
+				[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test]
+					privileged_without_host_devices = false
+					runtime_engine = ""
+					runtime_root = ""
+					runtime_type = "io.containerd.runc.v2"
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test.options]
+						BinaryName = "/usr/bin/test"
+			`,
+			expectedError: nil,
+		},
+		{
+			description: "v3 spec is supported",
+			config: `
+			version = 3
+			[plugins]
+			[plugins."io.containerd.cri.v1.runtime"]
+				[plugins."io.containerd.cri.v1.runtime".containerd]
+				[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
 					privileged_without_host_devices = true
 					runtime_engine = "engine"
 					runtime_root = "root"
 					runtime_type = "type"
-					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
+						BinaryName = "/usr/bin/runc"
+						SystemdCgroup = true
+			`,
+			expectedConfig: `
+			version = 3
+			[plugins]
+			[plugins."io.containerd.cri.v1.runtime"]
+				[plugins."io.containerd.cri.v1.runtime".containerd]
+				[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
+					privileged_without_host_devices = true
+					runtime_engine = "engine"
+					runtime_root = "root"
+					runtime_type = "type"
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
+						BinaryName = "/usr/bin/runc"
+						SystemdCgroup = true
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test]
+					privileged_without_host_devices = true
+					runtime_engine = "engine"
+					runtime_root = "root"
+					runtime_type = "type"
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test.options]
 						BinaryName = "/usr/bin/test"
 						SystemdCgroup = true
 				`,
@@ -224,20 +261,115 @@ func TestAddRuntime(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			config, err := toml.Load(tc.config)
-			require.NoError(t, err)
 			expectedConfig, err := toml.Load(tc.expectedConfig)
 			require.NoError(t, err)
 
-			c := &Config{
-				Logger: logger,
-				Tree:   config,
-			}
-
-			err = c.AddRuntime("test", "/usr/bin/test", tc.setAsDefault, tc.configOverrides...)
+			c, err := New(
+				WithLogger(logger),
+				WithConfigSource(toml.FromString(tc.config)),
+			)
 			require.NoError(t, err)
 
-			require.EqualValues(t, expectedConfig.String(), config.String())
+			err = c.AddRuntime("test", "/usr/bin/test", tc.setAsDefault)
+			require.NoError(t, err)
+
+			require.EqualValues(t, expectedConfig.String(), c.String())
+		})
+	}
+}
+
+func TestGetRuntimeConfig(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	config := `
+	version = 2
+	[plugins]
+	[plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      ignore_blockio_not_enabled_errors = false
+      ignore_rdt_not_enabled_errors = false
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        base_runtime_spec = ""
+        cni_conf_dir = ""
+        cni_max_conf_num = 0
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        privileged_without_host_devices_all_devices_allowed = false
+        runtime_engine = ""
+        runtime_path = ""
+        runtime_root = ""
+        runtime_type = ""
+        sandbox_mode = ""
+        snapshotter = ""
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          base_runtime_spec = ""
+          cni_conf_dir = ""
+          cni_max_conf_num = 0
+          container_annotations = []
+          pod_annotations = []
+          privileged_without_host_devices = false
+          privileged_without_host_devices_all_devices_allowed = false
+          runtime_engine = ""
+          runtime_path = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+          sandbox_mode = "podsandbox"
+          snapshotter = ""
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
+            IoUid = 0
+            NoNewKeyring = false
+            NoPivotRoot = false
+            Root = ""
+            ShimCgroup = ""
+            SystemdCgroup = false
+`
+	testCases := []struct {
+		description   string
+		runtime       string
+		expected      string
+		expectedError error
+	}{
+		{
+			description:   "valid runtime config, existing runtime",
+			runtime:       "runc",
+			expected:      "/usr/bin/runc",
+			expectedError: nil,
+		},
+		{
+			description:   "valid runtime config, non-existing runtime",
+			runtime:       "some-other-runtime",
+			expected:      "",
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+
+			c, err := New(
+				WithLogger(logger),
+				WithConfigSource(toml.FromString(config)),
+			)
+			require.NoError(t, err)
+
+			rc, err := c.GetRuntimeConfig(tc.runtime)
+			require.Equal(t, tc.expectedError, err)
+			require.Equal(t, tc.expected, rc.GetBinaryPath())
 		})
 	}
 }

pkg/config/engine/containerd/config_v1.go

@@ -19,9 +19,8 @@ package containerd
 import (
 	"fmt"
 
-	"github.com/pelletier/go-toml"
-
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 // ConfigV1 represents a version 1 containerd config
@@ -30,7 +29,7 @@ type ConfigV1 Config
 var _ engine.Interface = (*ConfigV1)(nil)
 
 // AddRuntime adds a runtime to the containerd config
-func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool, configOverrides ...map[string]interface{}) error {
+func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
 	if c == nil || c.Tree == nil {
 		return fmt.Errorf("config is nil")
 	}
@@ -39,18 +38,16 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool, confi
 
 	config.Set("version", int64(1))
 
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
 	for _, r := range runtimeNamesForConfig {
-		if options, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", r}).(*toml.Tree); ok {
-			c.Logger.Debugf("using options from runtime %v: %v", r, options.String())
-			options, _ = toml.Load(options.String())
-			config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, options)
-			break
+		options := config.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", r})
+		if options == nil {
+			continue
 		}
+		c.Logger.Debugf("using options from runtime %v: %v", r, options)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, options.Copy())
+		break
+
 	}
 
 	if config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name}) == nil {
@@ -73,28 +70,20 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool, confi
 	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
 	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}, path)
 
-	if setAsDefault && c.UseDefaultRuntimeName {
-		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
-	} else if setAsDefault {
-		// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
-		if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
+	if setAsDefault {
+		if !c.UseLegacyConfig {
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
+		} else {
+			// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
+			if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
+				config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
+				config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
+				config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
+				config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
+			}
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
 		}
-		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
-		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
-
-		defaultRuntimeSubtree := subtreeAtPath(config, "plugins", "cri", "containerd", "default_runtime")
-		if err := defaultRuntimeSubtree.applyOverrides(configOverrides...); err != nil {
-			return fmt.Errorf("failed to apply config overrides to default_runtime: %w", err)
-		}
-	}
-
-	runtimeSubtree := subtreeAtPath(config, "plugins", "cri", "containerd", "runtimes", name)
-	if err := runtimeSubtree.applyOverrides(configOverrides...); err != nil {
-		return fmt.Errorf("failed to apply config overrides: %w", err)
 	}
 
 	*c.Tree = config
@@ -154,14 +143,25 @@ func (c *ConfigV1) RemoveRuntime(name string) error {
 	return nil
 }
 
-// SetOption sets the specified containerd option.
+// Set sets the specified containerd option.
 func (c *ConfigV1) Set(key string, value interface{}) {
 	config := *c.Tree
 	config.SetPath([]string{"plugins", "cri", "containerd", key}, value)
 	*c.Tree = config
 }
 
-// Save wrotes the config to a file
+// Save writes the config to a file
 func (c ConfigV1) Save(path string) (int64, error) {
 	return (Config)(c).Save(path)
 }
+
+func (c *ConfigV1) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", name})
+
+	return &containerdCfgRuntime{
+		tree: runtimeData,
+	}, nil
+}

pkg/config/engine/containerd/config_v1_test.go

@@ -19,20 +19,20 @@ package containerd
 import (
 	"testing"
 
-	"github.com/pelletier/go-toml"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 func TestAddRuntimeV1(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 	testCases := []struct {
-		description     string
-		config          string
-		setAsDefault    bool
-		configOverrides []map[string]interface{}
-		expectedConfig  string
-		expectedError   error
+		description    string
+		config         string
+		setAsDefault   bool
+		expectedConfig string
+		expectedError  error
 	}{
 		{
 			description: "empty config not default runtime",
@@ -53,32 +53,6 @@ func TestAddRuntimeV1(t *testing.T) {
 			`,
 			expectedError: nil,
 		},
-		{
-			description: "empty config not default runtime with overrides",
-			configOverrides: []map[string]interface{}{
-				{
-					"options": map[string]interface{}{
-						"SystemdCgroup": true,
-					},
-				},
-			},
-			expectedConfig: `
-			version = 1
-			[plugins]
-			[plugins.cri]
-				[plugins.cri.containerd]
-				[plugins.cri.containerd.runtimes]
-					[plugins.cri.containerd.runtimes.test]
-					privileged_without_host_devices = false
-					runtime_engine = ""
-					runtime_root = ""
-					runtime_type = ""
-					[plugins.cri.containerd.runtimes.test.options]
-						BinaryName = "/usr/bin/test"
-						Runtime = "/usr/bin/test"
-						SystemdCgroup = true
-			`,
-		},
 		{
 			description: "options from runc are imported",
 			config: `
@@ -164,7 +138,7 @@ func TestAddRuntimeV1(t *testing.T) {
 				`,
 		},
 		{
-			description: "options from runc take precedence over default runtime",
+			description: "options from the default runtime take precedence over runc",
 			config: `
 			[plugins]
 			[plugins.cri]
@@ -212,34 +186,35 @@ func TestAddRuntimeV1(t *testing.T) {
 						BinaryName = "/usr/bin/default"
 						SystemdCgroup = false
 					[plugins.cri.containerd.runtimes.test]
-					privileged_without_host_devices = true
-					runtime_engine = "engine"
-					runtime_root = "root"
-					runtime_type = "type"
+					privileged_without_host_devices = false
+					runtime_engine = "defaultengine"
+					runtime_root = "defaultroot"
+					runtime_type = "defaulttype"
 					[plugins.cri.containerd.runtimes.test.options]
 						BinaryName = "/usr/bin/test"
 						Runtime = "/usr/bin/test"
-						SystemdCgroup = true
+						SystemdCgroup = false
 				`,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			config, err := toml.Load(tc.config)
-			require.NoError(t, err)
 			expectedConfig, err := toml.Load(tc.expectedConfig)
 			require.NoError(t, err)
 
-			c := &ConfigV1{
-				Logger: logger,
-				Tree:   config,
-			}
-
-			err = c.AddRuntime("test", "/usr/bin/test", tc.setAsDefault, tc.configOverrides...)
+			c, err := New(
+				WithLogger(logger),
+				WithConfigSource(toml.FromString(tc.config)),
+				WithUseLegacyConfig(true),
+				WithRuntimeType(""),
+			)
 			require.NoError(t, err)
 
-			require.EqualValues(t, expectedConfig.String(), config.String())
+			err = c.AddRuntime("test", "/usr/bin/test", tc.setAsDefault)
+			require.NoError(t, err)
+
+			require.EqualValues(t, expectedConfig.String(), c.String())
 		})
 	}
 }

pkg/config/engine/containerd/config_v2.go

@@ -1,165 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package containerd
-
-import (
-	"fmt"
-
-	"github.com/pelletier/go-toml"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
-)
-
-// AddRuntime adds a runtime to the containerd config
-func (c *Config) AddRuntime(name string, path string, setAsDefault bool, configOverrides ...map[string]interface{}) error {
-	if c == nil || c.Tree == nil {
-		return fmt.Errorf("config is nil")
-	}
-	config := *c.Tree
-
-	config.Set("version", int64(2))
-
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
-	for _, r := range runtimeNamesForConfig {
-		if options, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", r}).(*toml.Tree); ok {
-			c.Logger.Debugf("using options from runtime %v: %v", r, options.String())
-			options, _ = toml.Load(options.String())
-			config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, options)
-			break
-		}
-	}
-
-	if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
-		c.Logger.Warningf("could not infer options from runtimes %v; using defaults", runtimeNamesForConfig)
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
-	}
-
-	if len(c.ContainerAnnotations) > 0 {
-		annotations, err := c.getRuntimeAnnotations([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"})
-		if err != nil {
-			return err
-		}
-		annotations = append(c.ContainerAnnotations, annotations...)
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
-	}
-
-	config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
-
-	if setAsDefault {
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
-	}
-
-	runtimeSubtree := subtreeAtPath(config, "plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name)
-	if err := runtimeSubtree.applyOverrides(configOverrides...); err != nil {
-		return fmt.Errorf("failed to apply config overrides: %w", err)
-	}
-
-	*c.Tree = config
-	return nil
-}
-
-func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
-	if c == nil || c.Tree == nil {
-		return nil, nil
-	}
-
-	config := *c.Tree
-	if !config.HasPath(path) {
-		return nil, nil
-	}
-	annotationsI, ok := config.GetPath(path).([]interface{})
-	if !ok {
-		return nil, fmt.Errorf("invalid annotations: %v", annotationsI)
-	}
-
-	var annotations []string
-	for _, annotation := range annotationsI {
-		a, ok := annotation.(string)
-		if !ok {
-			return nil, fmt.Errorf("invalid annotation: %v", annotation)
-		}
-		annotations = append(annotations, a)
-	}
-
-	return annotations, nil
-}
-
-// Set sets the specified containerd option.
-func (c *Config) Set(key string, value interface{}) {
-	config := *c.Tree
-	config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", key}, value)
-	*c.Tree = config
-}
-
-// DefaultRuntime returns the default runtime for the cri-o config
-func (c Config) DefaultRuntime() string {
-	if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
-		return runtime
-	}
-	return ""
-}
-
-// RemoveRuntime removes a runtime from the docker config
-func (c *Config) RemoveRuntime(name string) error {
-	if c == nil || c.Tree == nil {
-		return nil
-	}
-
-	config := *c.Tree
-
-	config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
-	if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
-		if runtime == name {
-			config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
-		}
-	}
-
-	runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
-	for i := 0; i < len(runtimePath); i++ {
-		if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
-			if len(runtimes.Keys()) == 0 {
-				config.DeletePath(runtimePath[:len(runtimePath)-i])
-			}
-		}
-	}
-
-	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
-		config.Delete("version")
-	}
-
-	*c.Tree = config
-	return nil
-}
-
-// Save writes the config to the specified path
-func (c Config) Save(path string) (int64, error) {
-	config := c.Tree
-	output, err := config.Marshal()
-	if err != nil {
-		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
-	}
-
-	n, err := engine.Config(path).Write(output)
-	return int64(n), err
-}

pkg/config/engine/containerd/containerd.go

@@ -17,33 +17,159 @@
 package containerd
 
 import (
-	"github.com/pelletier/go-toml"
+	"fmt"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	defaultConfigVersion = 2
+	defaultRuntimeType   = "io.containerd.runc.v2"
 )
 
 // Config represents the containerd config
 type Config struct {
 	*toml.Tree
-	Logger                logger.Interface
-	RuntimeType           string
-	UseDefaultRuntimeName bool
-	ContainerAnnotations  []string
+	Version              int64
+	Logger               logger.Interface
+	RuntimeType          string
+	ContainerAnnotations []string
+	// UseLegacyConfig indicates whether a config file pre v1.3 should be generated.
+	// For version 1 config prior to containerd v1.4 the default runtime was
+	// specified in a containerd.runtimes.default_runtime section.
+	// This was deprecated in v1.4 in favour of containerd.default_runtime_name.
+	// Support for this section has been removed in v2.0.
+	UseLegacyConfig bool
+	// CRIRuntimePluginName represents the fully qualified name of the containerd plugin
+	// for the CRI runtime service. The name of this plugin was changed in v3 of the
+	// containerd configuration file.
+	CRIRuntimePluginName string
 }
 
 var _ engine.Interface = (*Config)(nil)
 
+type containerdCfgRuntime struct {
+	tree *toml.Tree
+}
+
+var _ engine.RuntimeConfig = (*containerdCfgRuntime)(nil)
+
+// GetBinaryPath retrieves the path to the low-level runtime binary for a runtime.
+// If no path is available, the empty string is returned.
+func (c *containerdCfgRuntime) GetBinaryPath() string {
+	if c == nil || c.tree == nil {
+		return ""
+	}
+
+	binPath, _ := c.tree.GetPath([]string{"options", "BinaryName"}).(string)
+	return binPath
+}
+
 // New creates a containerd config with the specified options
 func New(opts ...Option) (engine.Interface, error) {
-	b := &builder{}
+	b := &builder{
+		configVersion: defaultConfigVersion,
+		runtimeType:   defaultRuntimeType,
+	}
 	for _, opt := range opts {
 		opt(b)
 	}
-
 	if b.logger == nil {
 		b.logger = logger.New()
 	}
+	if b.configSource == nil {
+		b.configSource = toml.FromFile(b.path)
+	}
 
-	return b.build()
+	tomlConfig, err := b.configSource.Load()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+
+	configVersion, err := b.parseVersion(tomlConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config version: %w", err)
+	}
+	b.logger.Infof("Using config version %v", configVersion)
+
+	criRuntimePluginName, err := b.criRuntimePluginName(configVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CRI runtime plugin name: %w", err)
+	}
+	b.logger.Infof("Using CRI runtime plugin name %q", criRuntimePluginName)
+
+	cfg := &Config{
+		Tree:                 tomlConfig,
+		Version:              configVersion,
+		CRIRuntimePluginName: criRuntimePluginName,
+		Logger:               b.logger,
+		RuntimeType:          b.runtimeType,
+		UseLegacyConfig:      b.useLegacyConfig,
+		ContainerAnnotations: b.containerAnnotations,
+	}
+
+	switch configVersion {
+	case 1:
+		return (*ConfigV1)(cfg), nil
+	default:
+		return cfg, nil
+	}
+}
+
+// parseVersion returns the version of the config
+func (b *builder) parseVersion(c *toml.Tree) (int64, error) {
+	if c == nil || len(c.Keys()) == 0 {
+		// No config exists, or the config file is empty.
+		if b.useLegacyConfig {
+			// If a legacy config is explicitly requested, we default to a v1 config.
+			return 1, nil
+		}
+		// Use the requested version.
+		return int64(b.configVersion), nil
+	}
+
+	switch v := c.Get("version").(type) {
+	case nil:
+		return 1, nil
+	case int64:
+		return v, nil
+	default:
+		return -1, fmt.Errorf("unsupported type for version field: %v", v)
+	}
+}
+
+func (b *builder) criRuntimePluginName(configVersion int64) (string, error) {
+	switch configVersion {
+	case 1:
+		return "cri", nil
+	case 2:
+		return "io.containerd.grpc.v1.cri", nil
+	default:
+		return "io.containerd.cri.v1.runtime", nil
+	}
+}
+
+func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "runtimes", name})
+	return &containerdCfgRuntime{
+		tree: runtimeData,
+	}, nil
+}
+
+// CommandLineSource returns the CLI-based containerd config loader
+func CommandLineSource(hostRoot string) toml.Loader {
+	return toml.FromCommandLine(chrootIfRequired(hostRoot, "containerd", "config", "dump")...)
+}
+
+func chrootIfRequired(hostRoot string, commandLine ...string) []string {
+	if hostRoot == "" || hostRoot == "/" {
+		return commandLine
+	}
+
+	return append([]string{"chroot", hostRoot}, commandLine...)
 }

pkg/config/engine/containerd/option.go

@@ -17,24 +17,17 @@
 package containerd
 
 import (
-	"fmt"
-	"os"
-
-	"github.com/pelletier/go-toml"
-
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
-)
-
-const (
-	defaultRuntimeType = "io.containerd.runc.v2"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 type builder struct {
 	logger               logger.Interface
+	configSource         toml.Loader
+	configVersion        int
+	useLegacyConfig      bool
 	path                 string
 	runtimeType          string
-	useLegacyConfig      bool
 	containerAnnotations []string
 }
 
@@ -55,6 +48,13 @@ func WithPath(path string) Option {
 	}
 }
 
+// WithConfigSource sets the source for the config.
+func WithConfigSource(configSource toml.Loader) Option {
+	return func(b *builder) {
+		b.configSource = configSource
+	}
+}
+
 // WithRuntimeType sets the runtime type for the config builder
 func WithRuntimeType(runtimeType string) Option {
 	return func(b *builder) {
@@ -62,95 +62,23 @@ func WithRuntimeType(runtimeType string) Option {
 	}
 }
 
-// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
+// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder.
 func WithUseLegacyConfig(useLegacyConfig bool) Option {
 	return func(b *builder) {
 		b.useLegacyConfig = useLegacyConfig
 	}
 }
 
+// WithConfigVersion sets the config version for the config builder
+func WithConfigVersion(configVersion int) Option {
+	return func(b *builder) {
+		b.configVersion = configVersion
+	}
+}
+
 // WithContainerAnnotations sets the container annotations for the config builder
 func WithContainerAnnotations(containerAnnotations ...string) Option {
 	return func(b *builder) {
 		b.containerAnnotations = containerAnnotations
 	}
 }
-
-func (b *builder) build() (engine.Interface, error) {
-	if b.path == "" {
-		return nil, fmt.Errorf("config path is empty")
-	}
-
-	if b.runtimeType == "" {
-		b.runtimeType = defaultRuntimeType
-	}
-
-	config, err := b.loadConfig(b.path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load config: %v", err)
-	}
-	config.Logger = b.logger
-	config.RuntimeType = b.runtimeType
-	config.UseDefaultRuntimeName = !b.useLegacyConfig
-	config.ContainerAnnotations = b.containerAnnotations
-
-	version, err := config.parseVersion(b.useLegacyConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse config version: %v", err)
-	}
-	switch version {
-	case 1:
-		return (*ConfigV1)(config), nil
-	case 2:
-		return config, nil
-	}
-
-	return nil, fmt.Errorf("unsupported config version: %v", version)
-}
-
-// loadConfig loads the containerd config from disk
-func (b *builder) loadConfig(config string) (*Config, error) {
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	if os.IsNotExist(err) {
-		b.logger.Infof("Config file does not exist; using empty config")
-		config = "/dev/null"
-	} else {
-		b.logger.Infof("Loading config from %v", config)
-	}
-
-	tomlConfig, err := toml.LoadFile(config)
-	if err != nil {
-		return nil, err
-	}
-
-	cfg := Config{
-		Tree: tomlConfig,
-	}
-	return &cfg, nil
-}
-
-// parseVersion returns the version of the config
-func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
-	defaultVersion := 2
-	if useLegacyConfig {
-		defaultVersion = 1
-	}
-
-	switch v := c.Get("version").(type) {
-	case nil:
-		switch len(c.Keys()) {
-		case 0: // No config exists, or the config file is empty, use version inferred from containerd
-			return defaultVersion, nil
-		default: // A config file exists, has content, and no version is set
-			return 1, nil
-		}
-	case int64:
-		return int(v), nil
-	default:
-		return -1, fmt.Errorf("unsupported type for version field: %v", v)
-	}
-}

pkg/config/engine/containerd/toml.go

@@ -1,56 +0,0 @@
-/**
-# Copyright 2024 NVIDIA CORPORATION
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package containerd
-
-import (
-	"fmt"
-
-	"github.com/pelletier/go-toml"
-)
-
-// tomlTree is an alias for toml.Tree that allows for extensions.
-type tomlTree toml.Tree
-
-func subtreeAtPath(c toml.Tree, path ...string) *tomlTree {
-	tree := c.GetPath(path).(*toml.Tree)
-	return (*tomlTree)(tree)
-}
-
-func (t *tomlTree) insert(other map[string]interface{}) error {
-
-	for key, value := range other {
-		if insertsubtree, ok := value.(map[string]interface{}); ok {
-			subtree := (*toml.Tree)(t).Get(key).(*toml.Tree)
-			return (*tomlTree)(subtree).insert(insertsubtree)
-		}
-		(*toml.Tree)(t).Set(key, value)
-	}
-	return nil
-}
-
-func (t *tomlTree) applyOverrides(overrides ...map[string]interface{}) error {
-	for _, override := range overrides {
-		subconfig, err := toml.TreeFromMap(override)
-		if err != nil {
-			return fmt.Errorf("invalid toml config: %w", err)
-		}
-		if err := t.insert(subconfig.ToMap()); err != nil {
-			return err
-		}
-	}
-	return nil
-}