Refactor toolkit installer
Some checks failed
CI Pipeline / code-scanning (push) Has been cancelled
CI Pipeline / variables (push) Has been cancelled
CI Pipeline / golang (push) Has been cancelled
CI Pipeline / image (push) Has been cancelled
CI Pipeline / e2e-test (push) Has been cancelled

Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar 2025-03-13 14:51:00 +02:00
parent 30cb8d2122
commit c575025a01
No known key found for this signature in database
17 changed files with 1259 additions and 849 deletions

View File

@ -1,152 +0,0 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package toolkit
import (
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
type executableTarget struct {
dotfileName string
wrapperName string
}
type executable struct {
fileInstaller
source string
target executableTarget
env map[string]string
preLines []string
argLines []string
}
// install installs an executable component of the NVIDIA container toolkit. The source executable
// is copied to a `.real` file and a wapper is created to set up the environment as required.
func (e executable) install(destFolder string) (string, error) {
e.logger.Infof("Installing executable '%v' to %v", e.source, destFolder)
dotfileName := e.dotfileName()
installedDotfileName, err := e.installFileToFolderWithName(destFolder, dotfileName, e.source)
if err != nil {
return "", fmt.Errorf("error installing file '%v' as '%v': %v", e.source, dotfileName, err)
}
e.logger.Infof("Installed '%v'", installedDotfileName)
wrapperFilename, err := e.installWrapper(destFolder, installedDotfileName)
if err != nil {
return "", fmt.Errorf("error wrapping '%v': %v", installedDotfileName, err)
}
e.logger.Infof("Installed wrapper '%v'", wrapperFilename)
return wrapperFilename, nil
}
func (e executable) dotfileName() string {
return e.target.dotfileName
}
func (e executable) wrapperName() string {
return e.target.wrapperName
}
func (e executable) installWrapper(destFolder string, dotfileName string) (string, error) {
wrapperPath := filepath.Join(destFolder, e.wrapperName())
wrapper, err := os.Create(wrapperPath)
if err != nil {
return "", fmt.Errorf("error creating executable wrapper: %v", err)
}
defer wrapper.Close()
err = e.writeWrapperTo(wrapper, destFolder, dotfileName)
if err != nil {
return "", fmt.Errorf("error writing wrapper contents: %v", err)
}
err = ensureExecutable(wrapperPath)
if err != nil {
return "", fmt.Errorf("error making wrapper executable: %v", err)
}
return wrapperPath, nil
}
func (e executable) writeWrapperTo(wrapper io.Writer, destFolder string, dotfileName string) error {
r := newReplacements(destDirPattern, destFolder)
// Add the shebang
fmt.Fprintln(wrapper, "#! /bin/sh")
// Add the preceding lines if any
for _, line := range e.preLines {
fmt.Fprintf(wrapper, "%s\n", r.apply(line))
}
// Update the path to include the destination folder
var env map[string]string
if e.env == nil {
env = make(map[string]string)
} else {
env = e.env
}
path, specified := env["PATH"]
if !specified {
path = "$PATH"
}
env["PATH"] = strings.Join([]string{destFolder, path}, ":")
var sortedEnvvars []string
for e := range env {
sortedEnvvars = append(sortedEnvvars, e)
}
sort.Strings(sortedEnvvars)
for _, e := range sortedEnvvars {
v := env[e]
fmt.Fprintf(wrapper, "%s=%s \\\n", e, r.apply(v))
}
// Add the call to the target executable
fmt.Fprintf(wrapper, "%s \\\n", dotfileName)
// Insert additional lines in the `arg` list
for _, line := range e.argLines {
fmt.Fprintf(wrapper, "\t%s \\\n", r.apply(line))
}
// Add the script arguments "$@"
fmt.Fprintln(wrapper, "\t\"$@\"")
return nil
}
// ensureExecutable is equivalent to running chmod +x on the specified file
func ensureExecutable(path string) error {
info, err := os.Stat(path)
if err != nil {
return fmt.Errorf("error getting file info for '%v': %v", path, err)
}
executableMode := info.Mode() | 0111
err = os.Chmod(path, executableMode)
if err != nil {
return fmt.Errorf("error setting executable mode for '%v': %v", path, err)
}
return nil
}

View File

@ -1,162 +0,0 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package toolkit
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
testlog "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
)
func TestWrapper(t *testing.T) {
logger, _ := testlog.NewNullLogger()
const shebang = "#! /bin/sh"
const destFolder = "/dest/folder"
const dotfileName = "source.real"
testCases := []struct {
e executable
expectedLines []string
}{
{
e: executable{},
expectedLines: []string{
shebang,
"PATH=/dest/folder:$PATH \\",
"source.real \\",
"\t\"$@\"",
"",
},
},
{
e: executable{
env: map[string]string{
"PATH": "some-path",
},
},
expectedLines: []string{
shebang,
"PATH=/dest/folder:some-path \\",
"source.real \\",
"\t\"$@\"",
"",
},
},
{
e: executable{
preLines: []string{
"preline1",
"preline2",
},
},
expectedLines: []string{
shebang,
"preline1",
"preline2",
"PATH=/dest/folder:$PATH \\",
"source.real \\",
"\t\"$@\"",
"",
},
},
{
e: executable{
argLines: []string{
"argline1",
"argline2",
},
},
expectedLines: []string{
shebang,
"PATH=/dest/folder:$PATH \\",
"source.real \\",
"\targline1 \\",
"\targline2 \\",
"\t\"$@\"",
"",
},
},
}
for i, tc := range testCases {
buf := &bytes.Buffer{}
tc.e.logger = logger
err := tc.e.writeWrapperTo(buf, destFolder, dotfileName)
require.NoError(t, err)
exepectedContents := strings.Join(tc.expectedLines, "\n")
require.Equal(t, exepectedContents, buf.String(), "%v: %v", i, tc)
}
}
func TestInstallExecutable(t *testing.T) {
logger, _ := testlog.NewNullLogger()
inputFolder, err := os.MkdirTemp("", "")
require.NoError(t, err)
defer os.RemoveAll(inputFolder)
// Create the source file
source := filepath.Join(inputFolder, "input")
sourceFile, err := os.Create(source)
base := filepath.Base(source)
require.NoError(t, err)
require.NoError(t, sourceFile.Close())
e := executable{
fileInstaller: fileInstaller{
logger: logger,
},
source: source,
target: executableTarget{
dotfileName: "input.real",
wrapperName: "input",
},
}
destFolder, err := os.MkdirTemp("", "output-*")
require.NoError(t, err)
defer os.RemoveAll(destFolder)
installed, err := e.install(destFolder)
require.NoError(t, err)
require.Equal(t, filepath.Join(destFolder, base), installed)
// Now check the post conditions:
sourceInfo, err := os.Stat(source)
require.NoError(t, err)
destInfo, err := os.Stat(filepath.Join(destFolder, base+".real"))
require.NoError(t, err)
require.Equal(t, sourceInfo.Size(), destInfo.Size())
require.Equal(t, sourceInfo.Mode(), destInfo.Mode())
wrapperInfo, err := os.Stat(installed)
require.NoError(t, err)
require.NotEqual(t, 0, wrapperInfo.Mode()&0111)
}

View File

@ -1,95 +0,0 @@
/**
# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package toolkit
import (
"fmt"
"io"
"os"
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
)
type fileInstaller struct {
logger logger.Interface
// sourceRoot specifies the root that is searched for the components to install.
sourceRoot string
}
// installFileToFolder copies a source file to a destination folder.
// The path of the input file is ignored.
// e.g. installFileToFolder("/some/path/file.txt", "/output/path")
// will result in a file "/output/path/file.txt" being generated
func (t *fileInstaller) installFileToFolder(destFolder string, src string) (string, error) {
name := filepath.Base(src)
return t.installFileToFolderWithName(destFolder, name, src)
}
// cp src destFolder/name
func (t *fileInstaller) installFileToFolderWithName(destFolder string, name, src string) (string, error) {
dest := filepath.Join(destFolder, name)
err := t.installFile(dest, src)
if err != nil {
return "", fmt.Errorf("error copying '%v' to '%v': %v", src, dest, err)
}
return dest, nil
}
// installFile copies a file from src to dest and maintains
// file modes
func (t *fileInstaller) installFile(dest string, src string) error {
src = filepath.Join(t.sourceRoot, src)
t.logger.Infof("Installing '%v' to '%v'", src, dest)
source, err := os.Open(src)
if err != nil {
return fmt.Errorf("error opening source: %v", err)
}
defer source.Close()
destination, err := os.Create(dest)
if err != nil {
return fmt.Errorf("error creating destination: %v", err)
}
defer destination.Close()
_, err = io.Copy(destination, source)
if err != nil {
return fmt.Errorf("error copying file: %v", err)
}
err = applyModeFromSource(dest, src)
if err != nil {
return fmt.Errorf("error setting destination file mode: %v", err)
}
return nil
}
// applyModeFromSource sets the file mode for a destination file
// to match that of a specified source file
func applyModeFromSource(dest string, src string) error {
sourceInfo, err := os.Stat(src)
if err != nil {
return fmt.Errorf("error getting file info for '%v': %v", src, err)
}
err = os.Chmod(dest, sourceInfo.Mode())
if err != nil {
return fmt.Errorf("error setting mode for '%v': %v", dest, err)
}
return nil
}

View File

@ -0,0 +1,85 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"fmt"
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
)
// An artifactRoot is used as a source for installed artifacts.
// It is refined by a directory path, a library locator, and an executable locator.
type artifactRoot struct {
path string
libraries lookup.Locator
executables lookup.Locator
}
func newArtifactRoot(logger logger.Interface, rootDirectoryPath string) (*artifactRoot, error) {
relativeLibrarySearchPaths := []string{
"/usr/lib64",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib/aarch64-linux-gnu",
}
var librarySearchPaths []string
for _, l := range relativeLibrarySearchPaths {
librarySearchPaths = append(librarySearchPaths, filepath.Join(rootDirectoryPath, l))
}
a := artifactRoot{
path: rootDirectoryPath,
libraries: lookup.NewLibraryLocator(
lookup.WithLogger(logger),
lookup.WithCount(1),
lookup.WithSearchPaths(librarySearchPaths...),
),
executables: lookup.NewExecutableLocator(
logger,
rootDirectoryPath,
),
}
return &a, nil
}
func (r *artifactRoot) findLibrary(name string) (string, error) {
candidates, err := r.libraries.Locate(name)
if err != nil {
return "", fmt.Errorf("error locating library: %w", err)
}
if len(candidates) == 0 {
return "", fmt.Errorf("library %v not found", name)
}
return candidates[0], nil
}
func (r *artifactRoot) findExecutable(name string) (string, error) {
candidates, err := r.executables.Locate(name)
if err != nil {
return "", fmt.Errorf("error locating executable: %w", err)
}
if len(candidates) == 0 {
return "", fmt.Errorf("executable %v not found", name)
}
return candidates[0], nil
}

View File

@ -0,0 +1,47 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"fmt"
"os"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
)
type createDirectory struct {
logger logger.Interface
}
func (t *toolkitInstaller) createDirectory() Installer {
return &createDirectory{
logger: t.logger,
}
}
func (d *createDirectory) Install(dir string) error {
if dir == "" {
return nil
}
d.logger.Infof("Creating directory '%v'", dir)
err := os.MkdirAll(dir, 0755)
if err != nil {
return fmt.Errorf("error creating directory: %v", err)
}
return nil
}

View File

@ -0,0 +1,184 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"bytes"
"fmt"
"html/template"
"io"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
)
type executable struct {
requiresKernelModule bool
path string
symlink string
args []string
env map[string]string
}
func (t *toolkitInstaller) collectExecutables(destDir string) ([]Installer, error) {
configHome := filepath.Join(destDir, ".config")
configDir := filepath.Join(configHome, "nvidia-container-runtime")
configPath := filepath.Join(configDir, "config.toml")
executables := []executable{
{
path: "nvidia-ctk",
},
{
path: "nvidia-cdi-hook",
},
}
for _, runtime := range operator.GetRuntimes() {
e := executable{
path: runtime.Path,
requiresKernelModule: true,
env: map[string]string{
"XDG_CONFIG_HOME": configHome,
},
}
executables = append(executables, e)
}
executables = append(executables,
executable{
path: "nvidia-container-cli",
env: map[string]string{"LD_LIBRARY_PATH": destDir + ":$LD_LIBRARY_PATH"},
},
)
executables = append(executables,
executable{
path: "nvidia-container-runtime-hook",
symlink: "nvidia-container-toolkit",
args: []string{fmt.Sprintf("-config %s", configPath)},
},
)
var installers []Installer
for _, executable := range executables {
executablePath, err := t.artifactRoot.findExecutable(executable.path)
if err != nil {
if t.ignoreErrors {
log.Errorf("Ignoring error: %v", err)
continue
}
return nil, err
}
wrappedExecutableFilename := filepath.Base(executablePath)
dotRealFilename := wrappedExecutableFilename + ".real"
w := &wrapper{
Source: executablePath,
WrappedExecutable: dotRealFilename,
CheckModules: executable.requiresKernelModule,
Args: executable.args,
Envvars: map[string]string{
"PATH": strings.Join([]string{destDir, "$PATH"}, ":"),
},
}
for k, v := range executable.env {
w.Envvars[k] = v
}
installers = append(installers, w)
if executable.symlink == "" {
continue
}
link := symlink{
linkname: executable.symlink,
target: filepath.Base(executablePath),
}
installers = append(installers, link)
}
return installers, nil
}
type wrapper struct {
Source string
Envvars map[string]string
WrappedExecutable string
CheckModules bool
Args []string
}
type render struct {
*wrapper
DestDir string
}
func (w *wrapper) Install(destDir string) error {
// Copy the executable with a .real extension.
mode, err := installFile(w.Source, filepath.Join(destDir, w.WrappedExecutable))
if err != nil {
return err
}
// Create a wrapper file.
r := render{
wrapper: w,
DestDir: destDir,
}
content, err := r.render()
if err != nil {
return fmt.Errorf("failed to render wrapper: %w", err)
}
wrapperFile := filepath.Join(destDir, filepath.Base(w.Source))
return installContent(content, wrapperFile, mode|0111)
}
func (w *render) render() (io.Reader, error) {
wrapperTemplate := `#! /bin/sh
{{- if (.CheckModules) }}
cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
if [ "${?}" != "0" ]; then
echo "nvidia driver modules are not yet loaded, invoking runc directly"
exec runc "$@"
fi
{{- end }}
{{- range $key, $value := .Envvars }}
{{$key}}={{$value}} \
{{- end }}
{{ .DestDir }}/{{ .WrappedExecutable }} \
{{- range $arg := .Args }}
{{$arg}} \
{{- end }}
"$@"
`
var content bytes.Buffer
tmpl, err := template.New("wrapper").Parse(wrapperTemplate)
if err != nil {
return nil, err
}
if err := tmpl.Execute(&content, w); err != nil {
return nil, err
}
return &content, nil
}

View File

@ -0,0 +1,104 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
)
func TestWrapperRender(t *testing.T) {
testCases := []struct {
description string
w *wrapper
expected string
}{
{
description: "executable is added",
w: &wrapper{
WrappedExecutable: "some-runtime",
},
expected: `#! /bin/sh
/dest-dir/some-runtime \
"$@"
`,
},
{
description: "module check is added",
w: &wrapper{
WrappedExecutable: "some-runtime",
CheckModules: true,
},
expected: `#! /bin/sh
cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
if [ "${?}" != "0" ]; then
echo "nvidia driver modules are not yet loaded, invoking runc directly"
exec runc "$@"
fi
/dest-dir/some-runtime \
"$@"
`,
},
{
description: "environment is added",
w: &wrapper{
WrappedExecutable: "some-runtime",
Envvars: map[string]string{
"PATH": "/foo/bar/baz",
},
},
expected: `#! /bin/sh
PATH=/foo/bar/baz \
/dest-dir/some-runtime \
"$@"
`,
},
{
description: "args are added",
w: &wrapper{
WrappedExecutable: "some-runtime",
Args: []string{"--config foo", "bar"},
},
expected: `#! /bin/sh
/dest-dir/some-runtime \
--config foo \
bar \
"$@"
`,
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
r := render{
wrapper: tc.w,
DestDir: "/dest-dir",
}
reader, err := r.render()
require.NoError(t, err)
var content bytes.Buffer
_, err = content.ReadFrom(reader)
require.NoError(t, err)
require.Equal(t, tc.expected, content.String())
})
}
}

View File

@ -0,0 +1,188 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package installer
import (
"io"
"os"
"sync"
)
// Ensure, that fileInstallerMock does implement fileInstaller.
// If this is not the case, regenerate this file with moq.
var _ fileInstaller = &fileInstallerMock{}
// fileInstallerMock is a mock implementation of fileInstaller.
//
// func TestSomethingThatUsesfileInstaller(t *testing.T) {
//
// // make and configure a mocked fileInstaller
// mockedfileInstaller := &fileInstallerMock{
// installContentFunc: func(reader io.Reader, s string, v os.FileMode) error {
// panic("mock out the installContent method")
// },
// installFileFunc: func(s1 string, s2 string) (os.FileMode, error) {
// panic("mock out the installFile method")
// },
// installSymlinkFunc: func(s1 string, s2 string) error {
// panic("mock out the installSymlink method")
// },
// }
//
// // use mockedfileInstaller in code that requires fileInstaller
// // and then make assertions.
//
// }
type fileInstallerMock struct {
// installContentFunc mocks the installContent method.
installContentFunc func(reader io.Reader, s string, v os.FileMode) error
// installFileFunc mocks the installFile method.
installFileFunc func(s1 string, s2 string) (os.FileMode, error)
// installSymlinkFunc mocks the installSymlink method.
installSymlinkFunc func(s1 string, s2 string) error
// calls tracks calls to the methods.
calls struct {
// installContent holds details about calls to the installContent method.
installContent []struct {
// Reader is the reader argument value.
Reader io.Reader
// S is the s argument value.
S string
// V is the v argument value.
V os.FileMode
}
// installFile holds details about calls to the installFile method.
installFile []struct {
// S1 is the s1 argument value.
S1 string
// S2 is the s2 argument value.
S2 string
}
// installSymlink holds details about calls to the installSymlink method.
installSymlink []struct {
// S1 is the s1 argument value.
S1 string
// S2 is the s2 argument value.
S2 string
}
}
lockinstallContent sync.RWMutex
lockinstallFile sync.RWMutex
lockinstallSymlink sync.RWMutex
}
// installContent calls installContentFunc.
func (mock *fileInstallerMock) installContent(reader io.Reader, s string, v os.FileMode) error {
if mock.installContentFunc == nil {
panic("fileInstallerMock.installContentFunc: method is nil but fileInstaller.installContent was just called")
}
callInfo := struct {
Reader io.Reader
S string
V os.FileMode
}{
Reader: reader,
S: s,
V: v,
}
mock.lockinstallContent.Lock()
mock.calls.installContent = append(mock.calls.installContent, callInfo)
mock.lockinstallContent.Unlock()
return mock.installContentFunc(reader, s, v)
}
// installContentCalls gets all the calls that were made to installContent.
// Check the length with:
//
// len(mockedfileInstaller.installContentCalls())
func (mock *fileInstallerMock) installContentCalls() []struct {
Reader io.Reader
S string
V os.FileMode
} {
var calls []struct {
Reader io.Reader
S string
V os.FileMode
}
mock.lockinstallContent.RLock()
calls = mock.calls.installContent
mock.lockinstallContent.RUnlock()
return calls
}
// installFile calls installFileFunc.
func (mock *fileInstallerMock) installFile(s1 string, s2 string) (os.FileMode, error) {
if mock.installFileFunc == nil {
panic("fileInstallerMock.installFileFunc: method is nil but fileInstaller.installFile was just called")
}
callInfo := struct {
S1 string
S2 string
}{
S1: s1,
S2: s2,
}
mock.lockinstallFile.Lock()
mock.calls.installFile = append(mock.calls.installFile, callInfo)
mock.lockinstallFile.Unlock()
return mock.installFileFunc(s1, s2)
}
// installFileCalls gets all the calls that were made to installFile.
// Check the length with:
//
// len(mockedfileInstaller.installFileCalls())
func (mock *fileInstallerMock) installFileCalls() []struct {
S1 string
S2 string
} {
var calls []struct {
S1 string
S2 string
}
mock.lockinstallFile.RLock()
calls = mock.calls.installFile
mock.lockinstallFile.RUnlock()
return calls
}
// installSymlink calls installSymlinkFunc.
func (mock *fileInstallerMock) installSymlink(s1 string, s2 string) error {
if mock.installSymlinkFunc == nil {
panic("fileInstallerMock.installSymlinkFunc: method is nil but fileInstaller.installSymlink was just called")
}
callInfo := struct {
S1 string
S2 string
}{
S1: s1,
S2: s2,
}
mock.lockinstallSymlink.Lock()
mock.calls.installSymlink = append(mock.calls.installSymlink, callInfo)
mock.lockinstallSymlink.Unlock()
return mock.installSymlinkFunc(s1, s2)
}
// installSymlinkCalls gets all the calls that were made to installSymlink.
// Check the length with:
//
// len(mockedfileInstaller.installSymlinkCalls())
func (mock *fileInstallerMock) installSymlinkCalls() []struct {
S1 string
S2 string
} {
var calls []struct {
S1 string
S2 string
}
mock.lockinstallSymlink.RLock()
calls = mock.calls.installSymlink
mock.lockinstallSymlink.RUnlock()
return calls
}

View File

@ -0,0 +1,168 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
)
//go:generate moq -rm -fmt=goimports -out installer_mock.go . Installer
type Installer interface {
Install(string) error
}
type toolkitInstaller struct {
logger logger.Interface
ignoreErrors bool
sourceRoot string
artifactRoot *artifactRoot
ensureTargetDirectory Installer
}
var _ Installer = (*toolkitInstaller)(nil)
// New creates a toolkit installer with the specified options.
func New(opts ...Option) (Installer, error) {
t := &toolkitInstaller{}
for _, opt := range opts {
opt(t)
}
if t.logger == nil {
t.logger = logger.New()
}
if t.sourceRoot == "" {
t.sourceRoot = "/"
}
if t.artifactRoot == nil {
artifactRoot, err := newArtifactRoot(t.logger, t.sourceRoot)
if err != nil {
return nil, err
}
t.artifactRoot = artifactRoot
}
if t.ensureTargetDirectory == nil {
t.ensureTargetDirectory = t.createDirectory()
}
return t, nil
}
// Install ensures that the required toolkit files are installed in the specified directory.
func (t *toolkitInstaller) Install(destDir string) error {
var installers []Installer
installers = append(installers, t.ensureTargetDirectory)
libraries, err := t.collectLibraries()
if err != nil {
return fmt.Errorf("failed to collect libraries: %w", err)
}
installers = append(installers, libraries...)
executables, err := t.collectExecutables(destDir)
if err != nil {
return fmt.Errorf("failed to collect executables: %w", err)
}
installers = append(installers, executables...)
var errs error
for _, i := range installers {
errs = errors.Join(errs, i.Install(destDir))
}
return errs
}
type symlink struct {
linkname string
target string
}
func (s symlink) Install(destDir string) error {
symlinkPath := filepath.Join(destDir, s.linkname)
return installSymlink(s.target, symlinkPath)
}
//go:generate moq -rm -fmt=goimports -out file-installer_mock.go . fileInstaller
type fileInstaller interface {
installContent(io.Reader, string, os.FileMode) error
installFile(string, string) (os.FileMode, error)
installSymlink(string, string) error
}
var installSymlink = installSymlinkStub
func installSymlinkStub(target string, link string) error {
err := os.Symlink(target, link)
if err != nil {
return fmt.Errorf("error creating symlink '%v' => '%v': %v", link, target, err)
}
return nil
}
var installFile = installFileStub
func installFileStub(src string, dest string) (os.FileMode, error) {
sourceInfo, err := os.Stat(src)
if err != nil {
return 0, fmt.Errorf("error getting file info for '%v': %v", src, err)
}
source, err := os.Open(src)
if err != nil {
return 0, fmt.Errorf("error opening source: %w", err)
}
defer source.Close()
mode := sourceInfo.Mode()
if err := installContent(source, dest, mode); err != nil {
return 0, err
}
return mode, nil
}
var installContent = installContentStub
func installContentStub(content io.Reader, dest string, mode fs.FileMode) error {
destination, err := os.Create(dest)
if err != nil {
return fmt.Errorf("error creating destination: %w", err)
}
defer destination.Close()
_, err = io.Copy(destination, content)
if err != nil {
return fmt.Errorf("error copying file: %w", err)
}
err = os.Chmod(dest, mode)
if err != nil {
return fmt.Errorf("error setting mode for '%v': %v", dest, err)
}
return nil
}

View File

@ -0,0 +1,74 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package installer
import (
"sync"
)
// Ensure, that InstallerMock does implement Installer.
// If this is not the case, regenerate this file with moq.
var _ Installer = &InstallerMock{}
// InstallerMock is a mock implementation of Installer.
//
// func TestSomethingThatUsesInstaller(t *testing.T) {
//
// // make and configure a mocked Installer
// mockedInstaller := &InstallerMock{
// InstallFunc: func(s string) error {
// panic("mock out the Install method")
// },
// }
//
// // use mockedInstaller in code that requires Installer
// // and then make assertions.
//
// }
type InstallerMock struct {
// InstallFunc mocks the Install method.
InstallFunc func(s string) error
// calls tracks calls to the methods.
calls struct {
// Install holds details about calls to the Install method.
Install []struct {
// S is the s argument value.
S string
}
}
lockInstall sync.RWMutex
}
// Install calls InstallFunc.
func (mock *InstallerMock) Install(s string) error {
if mock.InstallFunc == nil {
panic("InstallerMock.InstallFunc: method is nil but Installer.Install was just called")
}
callInfo := struct {
S string
}{
S: s,
}
mock.lockInstall.Lock()
mock.calls.Install = append(mock.calls.Install, callInfo)
mock.lockInstall.Unlock()
return mock.InstallFunc(s)
}
// InstallCalls gets all the calls that were made to Install.
// Check the length with:
//
// len(mockedInstaller.InstallCalls())
func (mock *InstallerMock) InstallCalls() []struct {
S string
} {
var calls []struct {
S string
}
mock.lockInstall.RLock()
calls = mock.calls.Install
mock.lockInstall.RUnlock()
return calls
}

View File

@ -0,0 +1,251 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"bytes"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"testing"
testlog "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
)
func TestToolkitInstaller(t *testing.T) {
logger, _ := testlog.NewNullLogger()
type contentCall struct {
wrapper string
path string
mode fs.FileMode
}
var contentCalls []contentCall
installer := &fileInstallerMock{
installFileFunc: func(s1, s2 string) (os.FileMode, error) {
return 0666, nil
},
installContentFunc: func(reader io.Reader, s string, fileMode fs.FileMode) error {
var b bytes.Buffer
if _, err := b.ReadFrom(reader); err != nil {
return err
}
contents := contentCall{
wrapper: b.String(),
path: s,
mode: fileMode,
}
contentCalls = append(contentCalls, contents)
return nil
},
installSymlinkFunc: func(s1, s2 string) error {
return nil
},
}
installFile = installer.installFile
installContent = installer.installContent
installSymlink = installer.installSymlink
root := "/artifacts/test"
libraries := &lookup.LocatorMock{
LocateFunc: func(s string) ([]string, error) {
switch s {
case "libnvidia-container.so.1":
return []string{filepath.Join(root, "libnvidia-container.so.987.65.43")}, nil
case "libnvidia-container-go.so.1":
return []string{filepath.Join(root, "libnvidia-container-go.so.1.23.4")}, nil
}
return nil, fmt.Errorf("%v not found", s)
},
}
executables := &lookup.LocatorMock{
LocateFunc: func(s string) ([]string, error) {
switch s {
case "nvidia-container-runtime.cdi":
fallthrough
case "nvidia-container-runtime.legacy":
fallthrough
case "nvidia-container-runtime":
fallthrough
case "nvidia-ctk":
fallthrough
case "nvidia-container-cli":
fallthrough
case "nvidia-container-runtime-hook":
fallthrough
case "nvidia-cdi-hook":
return []string{filepath.Join(root, "usr/bin", s)}, nil
}
return nil, fmt.Errorf("%v not found", s)
},
}
r := &artifactRoot{
libraries: libraries,
executables: executables,
}
createDirectory := &InstallerMock{
InstallFunc: func(c string) error {
return nil
},
}
i := toolkitInstaller{
logger: logger,
artifactRoot: r,
ensureTargetDirectory: createDirectory,
}
err := i.Install("/foo/bar/baz")
require.NoError(t, err)
require.ElementsMatch(t,
[]struct {
S string
}{
{"/foo/bar/baz"},
},
createDirectory.InstallCalls(),
)
require.ElementsMatch(t,
installer.installFileCalls(),
[]struct {
S1 string
S2 string
}{
{"/artifacts/test/libnvidia-container-go.so.1.23.4", "/foo/bar/baz/libnvidia-container-go.so.1.23.4"},
{"/artifacts/test/libnvidia-container.so.987.65.43", "/foo/bar/baz/libnvidia-container.so.987.65.43"},
{"/artifacts/test/usr/bin/nvidia-container-runtime.cdi", "/foo/bar/baz/nvidia-container-runtime.cdi.real"},
{"/artifacts/test/usr/bin/nvidia-container-runtime.legacy", "/foo/bar/baz/nvidia-container-runtime.legacy.real"},
{"/artifacts/test/usr/bin/nvidia-container-runtime", "/foo/bar/baz/nvidia-container-runtime.real"},
{"/artifacts/test/usr/bin/nvidia-ctk", "/foo/bar/baz/nvidia-ctk.real"},
{"/artifacts/test/usr/bin/nvidia-cdi-hook", "/foo/bar/baz/nvidia-cdi-hook.real"},
{"/artifacts/test/usr/bin/nvidia-container-cli", "/foo/bar/baz/nvidia-container-cli.real"},
{"/artifacts/test/usr/bin/nvidia-container-runtime-hook", "/foo/bar/baz/nvidia-container-runtime-hook.real"},
},
)
require.ElementsMatch(t,
installer.installSymlinkCalls(),
[]struct {
S1 string
S2 string
}{
{"libnvidia-container-go.so.1.23.4", "/foo/bar/baz/libnvidia-container-go.so.1"},
{"libnvidia-container.so.987.65.43", "/foo/bar/baz/libnvidia-container.so.1"},
{"nvidia-container-runtime-hook", "/foo/bar/baz/nvidia-container-toolkit"},
},
)
require.ElementsMatch(t,
contentCalls,
[]contentCall{
{
path: "/foo/bar/baz/nvidia-container-runtime",
mode: 0777,
wrapper: `#! /bin/sh
cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
if [ "${?}" != "0" ]; then
echo "nvidia driver modules are not yet loaded, invoking runc directly"
exec runc "$@"
fi
PATH=/foo/bar/baz:$PATH \
XDG_CONFIG_HOME=/foo/bar/baz/.config \
/foo/bar/baz/nvidia-container-runtime.real \
"$@"
`,
},
{
path: "/foo/bar/baz/nvidia-container-runtime.cdi",
mode: 0777,
wrapper: `#! /bin/sh
cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
if [ "${?}" != "0" ]; then
echo "nvidia driver modules are not yet loaded, invoking runc directly"
exec runc "$@"
fi
PATH=/foo/bar/baz:$PATH \
XDG_CONFIG_HOME=/foo/bar/baz/.config \
/foo/bar/baz/nvidia-container-runtime.cdi.real \
"$@"
`,
},
{
path: "/foo/bar/baz/nvidia-container-runtime.legacy",
mode: 0777,
wrapper: `#! /bin/sh
cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
if [ "${?}" != "0" ]; then
echo "nvidia driver modules are not yet loaded, invoking runc directly"
exec runc "$@"
fi
PATH=/foo/bar/baz:$PATH \
XDG_CONFIG_HOME=/foo/bar/baz/.config \
/foo/bar/baz/nvidia-container-runtime.legacy.real \
"$@"
`,
},
{
path: "/foo/bar/baz/nvidia-ctk",
mode: 0777,
wrapper: `#! /bin/sh
PATH=/foo/bar/baz:$PATH \
/foo/bar/baz/nvidia-ctk.real \
"$@"
`,
},
{
path: "/foo/bar/baz/nvidia-cdi-hook",
mode: 0777,
wrapper: `#! /bin/sh
PATH=/foo/bar/baz:$PATH \
/foo/bar/baz/nvidia-cdi-hook.real \
"$@"
`,
},
{
path: "/foo/bar/baz/nvidia-container-cli",
mode: 0777,
wrapper: `#! /bin/sh
LD_LIBRARY_PATH=/foo/bar/baz:$LD_LIBRARY_PATH \
PATH=/foo/bar/baz:$PATH \
/foo/bar/baz/nvidia-container-cli.real \
"$@"
`,
},
{
path: "/foo/bar/baz/nvidia-container-runtime-hook",
mode: 0777,
wrapper: `#! /bin/sh
PATH=/foo/bar/baz:$PATH \
/foo/bar/baz/nvidia-container-runtime-hook.real \
-config /foo/bar/baz/.config/nvidia-container-runtime/config.toml \
"$@"
`,
},
},
)
}

View File

@ -0,0 +1,73 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import (
"path/filepath"
log "github.com/sirupsen/logrus"
)
// collectLibraries locates and installs the libraries that are part of
// the nvidia-container-toolkit.
// A predefined set of library candidates are considered, with the first one
// resulting in success being installed to the toolkit folder. The install process
// resolves the symlink for the library and copies the versioned library itself.
func (t *toolkitInstaller) collectLibraries() ([]Installer, error) {
requiredLibraries := []string{
"libnvidia-container.so.1",
"libnvidia-container-go.so.1",
}
var installers []Installer
for _, l := range requiredLibraries {
libraryPath, err := t.artifactRoot.findLibrary(l)
if err != nil {
if t.ignoreErrors {
log.Errorf("Ignoring error: %v", err)
continue
}
return nil, err
}
installers = append(installers, library(libraryPath))
if filepath.Base(libraryPath) == l {
continue
}
link := symlink{
linkname: l,
target: filepath.Base(libraryPath),
}
installers = append(installers, link)
}
return installers, nil
}
type library string
// Install copies the library l to the destination folder.
// The same basename is used in the destination folder.
func (l library) Install(destinationDir string) error {
dest := filepath.Join(destinationDir, filepath.Base(string(l)))
_, err := installFile(string(l), dest)
return err
}

View File

@ -0,0 +1,47 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/
package installer
import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
type Option func(*toolkitInstaller)
func WithLogger(logger logger.Interface) Option {
return func(ti *toolkitInstaller) {
ti.logger = logger
}
}
func WithArtifactRoot(artifactRoot *artifactRoot) Option {
return func(ti *toolkitInstaller) {
ti.artifactRoot = artifactRoot
}
}
func WithIgnoreErrors(ignoreErrors bool) Option {
return func(ti *toolkitInstaller) {
ti.ignoreErrors = ignoreErrors
}
}
// WithSourceRoot sets the root directory for locating artifacts to be installed.
func WithSourceRoot(sourceRoot string) Option {
return func(ti *toolkitInstaller) {
ti.sourceRoot = sourceRoot
}
}

View File

@ -1,45 +0,0 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package toolkit
import "strings"
const (
destDirPattern = "@destDir@"
)
type replacements map[string]string
func newReplacements(rules ...string) replacements {
r := make(replacements)
for i := 0; i < len(rules)-1; i += 2 {
old := rules[i]
new := rules[i+1]
r[old] = new
}
return r
}
func (r replacements) apply(input string) string {
output := input
for old, new := range r {
output = strings.ReplaceAll(output, old, new)
}
return output
}

View File

@ -1,85 +0,0 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package toolkit
import (
"fmt"
"path/filepath"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
)
const (
nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
)
// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
// and implementing the required wrapper
func (t *Installer) installContainerRuntimes(toolkitDir string) error {
runtimes := operator.GetRuntimes()
for _, runtime := range runtimes {
r := t.newNvidiaContainerRuntimeInstaller(runtime.Path)
_, err := r.install(toolkitDir)
if err != nil {
return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
}
}
return nil
}
// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime.
// This installer will copy the specified source executable to the toolkit directory.
// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is
// created to allow for the configuration of the runtime environment.
func (t *Installer) newNvidiaContainerRuntimeInstaller(source string) *executable {
wrapperName := filepath.Base(source)
dotfileName := wrapperName + ".real"
target := executableTarget{
dotfileName: dotfileName,
wrapperName: wrapperName,
}
return t.newRuntimeInstaller(source, target, nil)
}
func (t *Installer) newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {
preLines := []string{
"",
"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
"if [ \"${?}\" != \"0\" ]; then",
" echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
" exec runc \"$@\"",
"fi",
"",
}
runtimeEnv := make(map[string]string)
runtimeEnv["XDG_CONFIG_HOME"] = filepath.Join(destDirPattern, ".config")
for k, v := range env {
runtimeEnv[k] = v
}
r := executable{
fileInstaller: t.fileInstaller,
source: source,
target: target,
env: runtimeEnv,
preLines: preLines,
}
return &r
}

View File

@ -1,64 +0,0 @@
/**
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package toolkit
import (
"bytes"
"strings"
"testing"
testlog "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
)
func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
logger, _ := testlog.NewNullLogger()
i := Installer{
fileInstaller: fileInstaller{
logger: logger,
},
}
r := i.newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource)
const shebang = "#! /bin/sh"
const destFolder = "/dest/folder"
const dotfileName = "source.real"
buf := &bytes.Buffer{}
err := r.writeWrapperTo(buf, destFolder, dotfileName)
require.NoError(t, err)
expectedLines := []string{
shebang,
"",
"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
"if [ \"${?}\" != \"0\" ]; then",
" echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
" exec runc \"$@\"",
"fi",
"",
"PATH=/dest/folder:$PATH \\",
"XDG_CONFIG_HOME=/dest/folder/.config \\",
"source.real \\",
"\t\"$@\"",
"",
}
exepectedContents := strings.Join(expectedLines, "\n")
require.Equal(t, exepectedContents, buf.String())
}

View File

@ -26,6 +26,7 @@ import (
"tags.cncf.io/container-device-interface/pkg/cdi"
"tags.cncf.io/container-device-interface/pkg/parser"
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/toolkit/installer"
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
@ -37,9 +38,6 @@ const (
// DefaultNvidiaDriverRoot specifies the default NVIDIA driver run directory
DefaultNvidiaDriverRoot = "/run/nvidia/driver"
nvidiaContainerCliSource = "/usr/bin/nvidia-container-cli"
nvidiaContainerRuntimeHookSource = "/usr/bin/nvidia-container-runtime-hook"
configFilename = "config.toml"
)
@ -217,7 +215,8 @@ func Flags(opts *Options) []cli.Flag {
// An Installer is used to install the NVIDIA Container Toolkit from the toolkit container.
type Installer struct {
fileInstaller
logger logger.Interface
sourceRoot string
// toolkitRoot specifies the destination path at which the toolkit is installed.
toolkitRoot string
}
@ -232,6 +231,7 @@ func NewInstaller(opts ...Option) *Installer {
if i.logger == nil {
i.logger = logger.New()
}
return i
}
@ -296,59 +296,26 @@ func (t *Installer) Install(cli *cli.Context, opts *Options) error {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error removing toolkit directory: %v", err))
}
toolkitConfigDir := filepath.Join(t.toolkitRoot, ".config", "nvidia-container-runtime")
toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename)
err = t.createDirectories(t.toolkitRoot, toolkitConfigDir)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("could not create required directories: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not create required directories: %v", err))
// Create a toolkit installer to actually install the toolkit components.
toolkit, err := installer.New(
installer.WithLogger(t.logger),
installer.WithSourceRoot(t.sourceRoot),
installer.WithIgnoreErrors(opts.ignoreErrors),
)
if err != nil {
if !opts.ignoreErrors {
return fmt.Errorf("could not create toolkit installer: %w", err)
}
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not create toolkit installer: %w", err))
}
if err := toolkit.Install(t.toolkitRoot); err != nil {
if !opts.ignoreErrors {
return fmt.Errorf("could not install toolkit components: %w", err)
}
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not install toolkit components: %w", err))
}
err = t.installContainerLibraries(t.toolkitRoot)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA container library: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container library: %v", err))
}
err = t.installContainerRuntimes(t.toolkitRoot)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container runtime: %v", err))
}
nvidiaContainerCliExecutable, err := t.installContainerCLI(t.toolkitRoot)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA container CLI: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container CLI: %v", err))
}
nvidiaContainerRuntimeHookPath, err := t.installRuntimeHook(t.toolkitRoot, toolkitConfigPath)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err))
}
nvidiaCTKPath, err := t.installContainerToolkitCLI(t.toolkitRoot)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA Container Toolkit CLI: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA Container Toolkit CLI: %v", err))
}
nvidiaCDIHookPath, err := t.installContainerCDIHookCLI(t.toolkitRoot)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA Container CDI Hook CLI: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA Container CDI Hook CLI: %v", err))
}
err = t.installToolkitConfig(cli, toolkitConfigPath, nvidiaContainerCliExecutable, nvidiaCTKPath, nvidiaContainerRuntimeHookPath, opts)
err = t.installToolkitConfig(cli, opts)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err)
} else if err != nil {
@ -362,6 +329,7 @@ func (t *Installer) Install(cli *cli.Context, opts *Options) error {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error creating device nodes: %v", err))
}
nvidiaCDIHookPath := filepath.Join(t.toolkitRoot, "nvidia-cdi-hook")
err = t.generateCDISpec(opts, nvidiaCDIHookPath)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("error generating CDI specification: %v", err)
@ -372,59 +340,24 @@ func (t *Installer) Install(cli *cli.Context, opts *Options) error {
return nil
}
// installContainerLibraries locates and installs the libraries that are part of
// the nvidia-container-toolkit.
// A predefined set of library candidates are considered, with the first one
// resulting in success being installed to the toolkit folder. The install process
// resolves the symlink for the library and copies the versioned library itself.
func (t *Installer) installContainerLibraries(toolkitRoot string) error {
t.logger.Infof("Installing NVIDIA container library to '%v'", toolkitRoot)
libs := []string{
"libnvidia-container.so.1",
"libnvidia-container-go.so.1",
}
for _, l := range libs {
err := t.installLibrary(l, toolkitRoot)
if err != nil {
return fmt.Errorf("failed to install %s: %v", l, err)
}
}
return nil
}
// installLibrary installs the specified library to the toolkit directory.
func (t *Installer) installLibrary(libName string, toolkitRoot string) error {
libraryPath, err := t.findLibrary(libName)
if err != nil {
return fmt.Errorf("error locating NVIDIA container library: %v", err)
}
installedLibPath, err := t.installFileToFolder(toolkitRoot, libraryPath)
if err != nil {
return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitRoot, err)
}
t.logger.Infof("Installed '%v' to '%v'", libraryPath, installedLibPath)
if filepath.Base(installedLibPath) == libName {
return nil
}
err = t.installSymlink(toolkitRoot, libName, installedLibPath)
if err != nil {
return fmt.Errorf("error installing symlink for NVIDIA container library: %v", err)
}
return nil
}
// installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring
// that the settings are updated to match the desired install and nvidia driver directories.
func (t *Installer) installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContainerCliExecutablePath string, nvidiaCTKPath string, nvidaContainerRuntimeHookPath string, opts *Options) error {
func (t *Installer) installToolkitConfig(c *cli.Context, opts *Options) error {
toolkitConfigDir := filepath.Join(t.toolkitRoot, ".config", "nvidia-container-runtime")
toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename)
t.logger.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
err := t.createDirectories(toolkitConfigDir)
if err != nil && !opts.ignoreErrors {
return fmt.Errorf("could not create required directories: %v", err)
} else if err != nil {
t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not create required directories: %v", err))
}
nvidiaContainerCliExecutablePath := filepath.Join(t.toolkitRoot, "nvidia-container-cli")
nvidiaCTKPath := filepath.Join(t.toolkitRoot, "nvidia-ctk")
nvidiaContainerRuntimeHookPath := filepath.Join(t.toolkitRoot, "nvidia-container-runtime-hook")
cfg, err := config.New()
if err != nil {
return fmt.Errorf("could not open source config file: %v", err)
@ -453,7 +386,7 @@ func (t *Installer) installToolkitConfig(c *cli.Context, toolkitConfigPath strin
// Set nvidia-ctk options
"nvidia-ctk.path": nvidiaCTKPath,
// Set the nvidia-container-runtime-hook options
"nvidia-container-runtime-hook.path": nvidaContainerRuntimeHookPath,
"nvidia-container-runtime-hook.path": nvidiaContainerRuntimeHookPath,
"nvidia-container-runtime-hook.skip-mode-detection": opts.ContainerRuntimeHookSkipModeDetection,
}
@ -520,147 +453,6 @@ func (t *Installer) installToolkitConfig(c *cli.Context, toolkitConfigPath strin
return nil
}
// installContainerToolkitCLI installs the nvidia-ctk CLI executable and wrapper.
func (t *Installer) installContainerToolkitCLI(toolkitDir string) (string, error) {
e := executable{
fileInstaller: t.fileInstaller,
source: "/usr/bin/nvidia-ctk",
target: executableTarget{
dotfileName: "nvidia-ctk.real",
wrapperName: "nvidia-ctk",
},
}
return e.install(toolkitDir)
}
// installContainerCDIHookCLI installs the nvidia-cdi-hook CLI executable and wrapper.
func (t *Installer) installContainerCDIHookCLI(toolkitDir string) (string, error) {
e := executable{
fileInstaller: t.fileInstaller,
source: "/usr/bin/nvidia-cdi-hook",
target: executableTarget{
dotfileName: "nvidia-cdi-hook.real",
wrapperName: "nvidia-cdi-hook",
},
}
return e.install(toolkitDir)
}
// installContainerCLI sets up the NVIDIA container CLI executable, copying the executable
// and implementing the required wrapper
func (t *Installer) installContainerCLI(toolkitRoot string) (string, error) {
t.logger.Infof("Installing NVIDIA container CLI from '%v'", nvidiaContainerCliSource)
env := map[string]string{
"LD_LIBRARY_PATH": toolkitRoot,
}
e := executable{
fileInstaller: t.fileInstaller,
source: nvidiaContainerCliSource,
target: executableTarget{
dotfileName: "nvidia-container-cli.real",
wrapperName: "nvidia-container-cli",
},
env: env,
}
installedPath, err := e.install(toolkitRoot)
if err != nil {
return "", fmt.Errorf("error installing NVIDIA container CLI: %v", err)
}
return installedPath, nil
}
// installRuntimeHook sets up the NVIDIA runtime hook, copying the executable
// and implementing the required wrapper
func (t *Installer) installRuntimeHook(toolkitRoot string, configFilePath string) (string, error) {
t.logger.Infof("Installing NVIDIA container runtime hook from '%v'", nvidiaContainerRuntimeHookSource)
argLines := []string{
fmt.Sprintf("-config \"%s\"", configFilePath),
}
e := executable{
fileInstaller: t.fileInstaller,
source: nvidiaContainerRuntimeHookSource,
target: executableTarget{
dotfileName: "nvidia-container-runtime-hook.real",
wrapperName: "nvidia-container-runtime-hook",
},
argLines: argLines,
}
installedPath, err := e.install(toolkitRoot)
if err != nil {
return "", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
}
err = t.installSymlink(toolkitRoot, "nvidia-container-toolkit", installedPath)
if err != nil {
return "", fmt.Errorf("error installing symlink to NVIDIA container runtime hook: %v", err)
}
return installedPath, nil
}
// installSymlink creates a symlink in the toolkitDirectory that points to the specified target.
// Note: The target is assumed to be local to the toolkit directory
func (t *Installer) installSymlink(toolkitRoot string, link string, target string) error {
symlinkPath := filepath.Join(toolkitRoot, link)
targetPath := filepath.Base(target)
t.logger.Infof("Creating symlink '%v' -> '%v'", symlinkPath, targetPath)
err := os.Symlink(targetPath, symlinkPath)
if err != nil {
return fmt.Errorf("error creating symlink '%v' => '%v': %v", symlinkPath, targetPath, err)
}
return nil
}
// findLibrary searches a set of candidate libraries in the specified root for
// a given library name
func (t *Installer) findLibrary(libName string) (string, error) {
t.logger.Infof("Finding library %v (root=%v)", libName)
candidateDirs := []string{
"/usr/lib64",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib/aarch64-linux-gnu",
}
for _, d := range candidateDirs {
l := filepath.Join(t.sourceRoot, d, libName)
t.logger.Infof("Checking library candidate '%v'", l)
libraryCandidate, err := t.resolveLink(l)
if err != nil {
t.logger.Infof("Skipping library candidate '%v': %v", l, err)
continue
}
return strings.TrimPrefix(libraryCandidate, t.sourceRoot), nil
}
return "", fmt.Errorf("error locating library '%v'", libName)
}
// resolveLink finds the target of a symlink or the file itself in the
// case of a regular file.
// This is equivalent to running `readlink -f ${l}`
func (t *Installer) resolveLink(l string) (string, error) {
resolved, err := filepath.EvalSymlinks(l)
if err != nil {
return "", fmt.Errorf("error resolving link '%v': %v", l, err)
}
if l != resolved {
t.logger.Infof("Resolved link: '%v' => '%v'", l, resolved)
}
return resolved, nil
}
func (t *Installer) createDirectories(dir ...string) error {
for _, d := range dir {
t.logger.Infof("Creating directory '%v'", d)