Merge pull request #947 from elezar/ensure-libcuda.so-in-ldcache
Some checks are pending
CI Pipeline / code-scanning (push) Waiting to run
CI Pipeline / variables (push) Waiting to run
CI Pipeline / golang (push) Waiting to run
CI Pipeline / image (push) Blocked by required conditions
CI Pipeline / e2e-test (push) Blocked by required conditions

Ensure that libcuda.so is in the ldcache
This commit is contained in:
Evan Lezar
2025-06-24 19:55:51 +02:00
committed by GitHub
16 changed files with 534 additions and 206 deletions

View File

@@ -46,6 +46,9 @@ const (
// An UpdateLDCacheHook is the hook used to update the ldcache in the
// container. This allows injected libraries to be discoverable.
UpdateLDCacheHook = HookName("update-ldcache")
// A CreateSonameSymlinksHook is the hook used to ensure that soname symlinks
// for injected libraries exist in the container.
CreateSonameSymlinksHook = HookName("create-soname-symlinks")
defaultNvidiaCDIHookPath = "/usr/bin/nvidia-cdi-hook"
)

View File

@@ -51,28 +51,22 @@ func (d ldconfig) Hooks() ([]Hook, error) {
return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
}
h := createLDCacheUpdateHook(
d.hookCreator,
d.ldconfigPath,
getLibraryPaths(mounts),
)
return h.Hooks()
}
// createLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
func createLDCacheUpdateHook(hookCreator HookCreator, ldconfig string, libraries []string) *Hook {
var args []string
if ldconfig != "" {
args = append(args, "--ldconfig-path", ldconfig)
if d.ldconfigPath != "" {
args = append(args, "--ldconfig-path", d.ldconfigPath)
}
for _, f := range uniqueFolders(libraries) {
for _, f := range uniqueFolders(getLibraryPaths(mounts)) {
args = append(args, "--folder", f)
}
return hookCreator.Create(UpdateLDCacheHook, args...)
h := Merge(
d.hookCreator.Create(CreateSonameSymlinksHook, args...),
d.hookCreator.Create(UpdateLDCacheHook, args...),
)
return h.Hooks()
}
// getLibraryPaths extracts the library dirs from the specified mounts

View File

@@ -39,11 +39,24 @@ func TestLDCacheUpdateHook(t *testing.T) {
mounts []Mount
mountError error
expectedError error
expectedArgs []string
expectedHooks []Hook
}{
{
description: "empty mounts",
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache"},
description: "empty mounts",
expectedHooks: []Hook{
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks"},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "update-ldcache"},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
},
},
{
description: "mount error",
@@ -66,7 +79,20 @@ func TestLDCacheUpdateHook(t *testing.T) {
Path: "/usr/local/lib/libbar.so",
},
},
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
expectedHooks: []Hook{
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
},
},
{
description: "host paths are ignored",
@@ -76,12 +102,38 @@ func TestLDCacheUpdateHook(t *testing.T) {
Path: "/usr/local/lib/libfoo.so",
},
},
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib"},
expectedHooks: []Hook{
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--folder", "/usr/local/lib"},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib"},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
},
},
{
description: "explicit ldconfig path is passed",
ldconfigPath: testLdconfigPath,
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--ldconfig-path", testLdconfigPath},
expectedHooks: []Hook{
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--ldconfig-path", testLdconfigPath},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
{
Lifecycle: "createContainer",
Path: testNvidiaCDIHookPath,
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--ldconfig-path", testLdconfigPath},
Env: []string{"NVIDIA_CTK_DEBUG=false"},
},
},
},
}
@@ -92,13 +144,6 @@ func TestLDCacheUpdateHook(t *testing.T) {
return tc.mounts, tc.mountError
},
}
expectedHook := Hook{
Path: testNvidiaCDIHookPath,
Args: tc.expectedArgs,
Lifecycle: "createContainer",
Env: []string{"NVIDIA_CTK_DEBUG=false"},
}
d, err := NewLDCacheUpdateHook(logger, mountMock, hookCreator, tc.ldconfigPath)
require.NoError(t, err)
@@ -112,9 +157,7 @@ func TestLDCacheUpdateHook(t *testing.T) {
}
require.NoError(t, err)
require.Len(t, hooks, 1)
require.EqualValues(t, hooks[0], expectedHook)
require.EqualValues(t, tc.expectedHooks, hooks)
devices, err := d.Devices()
require.NoError(t, err)

View File

@@ -0,0 +1,206 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package ldconfig
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
)
const (
// ldsoconfdFilenamePattern specifies the pattern for the filename
// in ld.so.conf.d that includes references to the specified directories.
// The 00-nvcr prefix is chosen to ensure that these libraries have a
// higher precedence than other libraries on the system, but lower than
// the 00-cuda-compat that is included in some containers.
ldsoconfdFilenamePattern = "00-nvcr-*.conf"
)
type Ldconfig struct {
ldconfigPath string
inRoot string
}
// NewRunner creates an exec.Cmd that can be used to run ldconfig.
func NewRunner(id string, ldconfigPath string, containerRoot string, additionalargs ...string) (*exec.Cmd, error) {
args := []string{
id,
strings.TrimPrefix(config.NormalizeLDConfigPath("@"+ldconfigPath), "@"),
containerRoot,
}
args = append(args, additionalargs...)
return createReexecCommand(args)
}
// New creates an Ldconfig struct that is used to perform operations on the
// ldcache and libraries in a particular root (e.g. a container).
func New(ldconfigPath string, inRoot string) (*Ldconfig, error) {
l := &Ldconfig{
ldconfigPath: ldconfigPath,
inRoot: inRoot,
}
if ldconfigPath == "" {
return nil, fmt.Errorf("an ldconfig path must be specified")
}
if inRoot == "" || inRoot == "/" {
return nil, fmt.Errorf("ldconfig must be run in the non-system root")
}
return l, nil
}
// CreateSonameSymlinks uses ldconfig to create the soname symlinks in the
// specified directories.
func (l *Ldconfig) CreateSonameSymlinks(directories ...string) error {
if len(directories) == 0 {
return nil
}
ldconfigPath, err := l.prepareRoot()
if err != nil {
return err
}
args := []string{
filepath.Base(ldconfigPath),
// Explicitly disable updating the LDCache.
"-N",
// Specify -n to only process the specified directories.
"-n",
}
args = append(args, directories...)
return SafeExec(ldconfigPath, args, nil)
}
func (l *Ldconfig) UpdateLDCache(directories ...string) error {
ldconfigPath, err := l.prepareRoot()
if err != nil {
return err
}
args := []string{
filepath.Base(ldconfigPath),
// Explicitly specify using /etc/ld.so.conf since the host's ldconfig may
// be configured to use a different config file by default.
"-f", "/etc/ld.so.conf",
}
if l.ldcacheExists() {
args = append(args, "-C", "/etc/ld.so.cache")
} else {
args = append(args, "-N")
}
// If the ld.so.conf.d directory exists, we create a config file there
// containing the required directories, otherwise we add the specified
// directories to the ldconfig command directly.
if l.ldsoconfdDirectoryExists() {
err := createLdsoconfdFile(ldsoconfdFilenamePattern, directories...)
if err != nil {
return fmt.Errorf("failed to update ld.so.conf.d: %w", err)
}
} else {
args = append(args, directories...)
}
return SafeExec(ldconfigPath, args, nil)
}
func (l *Ldconfig) prepareRoot() (string, error) {
// To prevent leaking the parent proc filesystem, we create a new proc mount
// in the specified root.
if err := mountProc(l.inRoot); err != nil {
return "", fmt.Errorf("error mounting /proc: %w", err)
}
// We mount the host ldconfig before we pivot root since host paths are not
// visible after the pivot root operation.
ldconfigPath, err := mountLdConfig(l.ldconfigPath, l.inRoot)
if err != nil {
return "", fmt.Errorf("error mounting host ldconfig: %w", err)
}
// We pivot to the container root for the new process, this further limits
// access to the host.
if err := pivotRoot(l.inRoot); err != nil {
return "", fmt.Errorf("error running pivot_root: %w", err)
}
return ldconfigPath, nil
}
func (l *Ldconfig) ldcacheExists() bool {
if _, err := os.Stat("/etc/ld.so.cache"); err != nil && os.IsNotExist(err) {
return false
}
return true
}
func (l *Ldconfig) ldsoconfdDirectoryExists() bool {
info, err := os.Stat("/etc/ld.so.conf.d")
if os.IsNotExist(err) {
return false
}
return info.IsDir()
}
// createLdsoconfdFile creates a file at /etc/ld.so.conf.d/.
// The file is created at /etc/ld.so.conf.d/{{ .pattern }} using `CreateTemp` and
// contains the specified directories on each line.
func createLdsoconfdFile(pattern string, dirs ...string) error {
if len(dirs) == 0 {
return nil
}
ldsoconfdDir := "/etc/ld.so.conf.d"
if err := os.MkdirAll(ldsoconfdDir, 0755); err != nil {
return fmt.Errorf("failed to create ld.so.conf.d: %w", err)
}
configFile, err := os.CreateTemp(ldsoconfdDir, pattern)
if err != nil {
return fmt.Errorf("failed to create config file: %w", err)
}
defer func() {
_ = configFile.Close()
}()
added := make(map[string]bool)
for _, dir := range dirs {
if added[dir] {
continue
}
_, err = fmt.Fprintf(configFile, "%s\n", dir)
if err != nil {
return fmt.Errorf("failed to update config file: %w", err)
}
added[dir] = true
}
// The created file needs to be world readable for the cases where the container is run as a non-root user.
if err := configFile.Chmod(0644); err != nil {
return fmt.Errorf("failed to chmod config file: %w", err)
}
return nil
}

View File

@@ -0,0 +1,200 @@
//go:build linux
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package ldconfig
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"syscall"
securejoin "github.com/cyphar/filepath-securejoin"
"github.com/moby/sys/reexec"
"github.com/opencontainers/runc/libcontainer/utils"
"golang.org/x/sys/unix"
)
// pivotRoot will call pivot_root such that rootfs becomes the new root
// filesystem, and everything else is cleaned up.
// This is adapted from the implementation here:
//
// https://github.com/opencontainers/runc/blob/e89a29929c775025419ab0d218a43588b4c12b9a/libcontainer/rootfs_linux.go#L1056-L1113
//
// With the `mount` and `unmount` calls changed to direct unix.Mount and unix.Unmount calls.
func pivotRoot(rootfs string) error {
// While the documentation may claim otherwise, pivot_root(".", ".") is
// actually valid. What this results in is / being the new root but
// /proc/self/cwd being the old root. Since we can play around with the cwd
// with pivot_root this allows us to pivot without creating directories in
// the rootfs. Shout-outs to the LXC developers for giving us this idea.
oldroot, err := unix.Open("/", unix.O_DIRECTORY|unix.O_RDONLY, 0)
if err != nil {
return &os.PathError{Op: "open", Path: "/", Err: err}
}
defer unix.Close(oldroot) //nolint: errcheck
newroot, err := unix.Open(rootfs, unix.O_DIRECTORY|unix.O_RDONLY, 0)
if err != nil {
return &os.PathError{Op: "open", Path: rootfs, Err: err}
}
defer unix.Close(newroot) //nolint: errcheck
// Change to the new root so that the pivot_root actually acts on it.
if err := unix.Fchdir(newroot); err != nil {
return &os.PathError{Op: "fchdir", Path: "fd " + strconv.Itoa(newroot), Err: err}
}
if err := unix.PivotRoot(".", "."); err != nil {
return &os.PathError{Op: "pivot_root", Path: ".", Err: err}
}
// Currently our "." is oldroot (according to the current kernel code).
// However, purely for safety, we will fchdir(oldroot) since there isn't
// really any guarantee from the kernel what /proc/self/cwd will be after a
// pivot_root(2).
if err := unix.Fchdir(oldroot); err != nil {
return &os.PathError{Op: "fchdir", Path: "fd " + strconv.Itoa(oldroot), Err: err}
}
// Make oldroot rslave to make sure our unmounts don't propagate to the
// host (and thus bork the machine). We don't use rprivate because this is
// known to cause issues due to races where we still have a reference to a
// mount while a process in the host namespace are trying to operate on
// something they think has no mounts (devicemapper in particular).
if err := unix.Mount("", ".", "", unix.MS_SLAVE|unix.MS_REC, ""); err != nil {
return err
}
// Perform the unmount. MNT_DETACH allows us to unmount /proc/self/cwd.
if err := unix.Unmount(".", unix.MNT_DETACH); err != nil {
return err
}
// Switch back to our shiny new root.
if err := unix.Chdir("/"); err != nil {
return &os.PathError{Op: "chdir", Path: "/", Err: err}
}
return nil
}
// mountLdConfig mounts the host ldconfig to the mount namespace of the hook.
// We use WithProcfd to perform the mount operations to ensure that the changes
// are persisted across the pivot root.
func mountLdConfig(hostLdconfigPath string, containerRootDirPath string) (string, error) {
hostLdconfigInfo, err := os.Stat(hostLdconfigPath)
if err != nil {
return "", fmt.Errorf("error reading host ldconfig: %w", err)
}
hookScratchDirPath := "/var/run/nvidia-ctk-hook"
ldconfigPath := filepath.Join(hookScratchDirPath, "ldconfig")
if err := utils.MkdirAllInRoot(containerRootDirPath, hookScratchDirPath, 0755); err != nil {
return "", fmt.Errorf("error creating hook scratch folder: %w", err)
}
err = utils.WithProcfd(containerRootDirPath, hookScratchDirPath, func(hookScratchDirFdPath string) error {
return createTmpFs(hookScratchDirFdPath, int(hostLdconfigInfo.Size()))
})
if err != nil {
return "", fmt.Errorf("error creating tmpfs: %w", err)
}
if _, err := createFileInRoot(containerRootDirPath, ldconfigPath, hostLdconfigInfo.Mode()); err != nil {
return "", fmt.Errorf("error creating ldconfig: %w", err)
}
err = utils.WithProcfd(containerRootDirPath, ldconfigPath, func(ldconfigFdPath string) error {
return unix.Mount(hostLdconfigPath, ldconfigFdPath, "", unix.MS_BIND|unix.MS_RDONLY|unix.MS_NODEV|unix.MS_PRIVATE|unix.MS_NOSYMFOLLOW, "")
})
if err != nil {
return "", fmt.Errorf("error bind mounting host ldconfig: %w", err)
}
return ldconfigPath, nil
}
func createFileInRoot(containerRootDirPath string, destinationPath string, mode os.FileMode) (string, error) {
dest, err := securejoin.SecureJoin(containerRootDirPath, destinationPath)
if err != nil {
return "", err
}
// Make the parent directory.
destDir, destBase := filepath.Split(dest)
destDirFd, err := utils.MkdirAllInRootOpen(containerRootDirPath, destDir, 0755)
if err != nil {
return "", fmt.Errorf("error creating parent dir: %w", err)
}
defer destDirFd.Close()
// Make the target file. We want to avoid opening any file that is
// already there because it could be a "bad" file like an invalid
// device or hung tty that might cause a DoS, so we use mknodat.
// destBase does not contain any "/" components, and mknodat does
// not follow trailing symlinks, so we can safely just call mknodat
// here.
if err := unix.Mknodat(int(destDirFd.Fd()), destBase, unix.S_IFREG|uint32(mode), 0); err != nil {
// If we get EEXIST, there was already an inode there and
// we can consider that a success.
if !errors.Is(err, unix.EEXIST) {
return "", fmt.Errorf("error creating empty file: %w", err)
}
}
return dest, nil
}
// mountProc mounts a clean proc filesystem in the new root.
func mountProc(newroot string) error {
target := filepath.Join(newroot, "/proc")
if err := os.MkdirAll(target, 0755); err != nil {
return fmt.Errorf("error creating directory: %w", err)
}
return unix.Mount("proc", target, "proc", 0, "")
}
// createTmpFs creates a tmpfs at the specified location with the specified size.
func createTmpFs(target string, size int) error {
return unix.Mount("tmpfs", target, "tmpfs", 0, fmt.Sprintf("size=%d", size))
}
// createReexecCommand creates a command that can be used to trigger the reexec
// initializer.
// On linux this command runs in new namespaces.
func createReexecCommand(args []string) (*exec.Cmd, error) {
cmd := reexec.Command(args...)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.SysProcAttr = &syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWNS |
syscall.CLONE_NEWUTS |
syscall.CLONE_NEWIPC |
syscall.CLONE_NEWPID |
syscall.CLONE_NEWNET,
}
return cmd, nil
}

View File

@@ -0,0 +1,41 @@
//go:build !linux
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package ldconfig
import (
"fmt"
"os/exec"
)
func pivotRoot(newroot string) error {
return fmt.Errorf("not supported")
}
func mountLdConfig(hostLdconfigPath string, containerRootDirPath string) (string, error) {
return "", fmt.Errorf("not supported")
}
func mountProc(newroot string) error {
return fmt.Errorf("not supported")
}
func createReexecCommand(args []string) (*exec.Cmd, error) {
return nil, fmt.Errorf("not supported")
}

View File

@@ -0,0 +1,58 @@
//go:build linux
/**
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package ldconfig
import (
"fmt"
"os"
"strconv"
"syscall"
"github.com/opencontainers/runc/libcontainer/exeseal"
)
// SafeExec attempts to clone the specified binary (as an memfd, for example) before executing it.
func SafeExec(path string, args []string, envv []string) error {
safeExe, err := cloneBinary(path)
if err != nil {
//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
return syscall.Exec(path, args, envv)
}
defer safeExe.Close()
exePath := "/proc/self/fd/" + strconv.Itoa(int(safeExe.Fd()))
//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
return syscall.Exec(exePath, args, envv)
}
func cloneBinary(path string) (*os.File, error) {
exe, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening current binary: %w", err)
}
defer exe.Close()
stat, err := exe.Stat()
if err != nil {
return nil, fmt.Errorf("checking %v size: %w", path, err)
}
size := stat.Size()
return exeseal.CloneBinary(exe, size, path, os.TempDir())
}

View File

@@ -0,0 +1,28 @@
//go:build !linux
/**
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package ldconfig
import "syscall"
// SafeExec is not implemented on non-linux systems and forwards directly to the
// Exec syscall.
func SafeExec(path string, args []string, envv []string) error {
//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
return syscall.Exec(path, args, envv)
}