From 2ce23c9af3a945f19fd826ed986cdbb33c4550f0 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 4 Sep 2024 12:02:00 +0200 Subject: [PATCH] [no-relnote] Refactor toolkit installation Signed-off-by: Evan Lezar --- tools/container/nvidia-toolkit/run.go | 2 +- tools/container/toolkit/executable.go | 153 --------- tools/container/toolkit/executable_test.go | 152 --------- .../toolkit/installer/artifact-root.go | 102 ++++++ .../toolkit/installer/executables.go | 169 +++++++++ .../toolkit/installer/executables_test.go | 101 ++++++ .../toolkit/installer/file-installer_mock.go | 188 ++++++++++ .../container/toolkit/installer/installer.go | 153 +++++++++ .../toolkit/installer/installer_test.go | 258 ++++++++++++++ .../container/toolkit/installer/libraries.go | 72 ++++ tools/container/toolkit/installer/options.go | 43 +++ tools/container/toolkit/replacements.go | 45 --- tools/container/toolkit/runtime.go | 84 ----- tools/container/toolkit/runtime_test.go | 57 ---- tools/container/toolkit/toolkit.go | 323 ++---------------- 15 files changed, 1118 insertions(+), 784 deletions(-) delete mode 100644 tools/container/toolkit/executable.go delete mode 100644 tools/container/toolkit/executable_test.go create mode 100644 tools/container/toolkit/installer/artifact-root.go create mode 100644 tools/container/toolkit/installer/executables.go create mode 100644 tools/container/toolkit/installer/executables_test.go create mode 100644 tools/container/toolkit/installer/file-installer_mock.go create mode 100644 tools/container/toolkit/installer/installer.go create mode 100644 tools/container/toolkit/installer/installer_test.go create mode 100644 tools/container/toolkit/installer/libraries.go create mode 100644 tools/container/toolkit/installer/options.go delete mode 100644 tools/container/toolkit/replacements.go delete mode 100644 tools/container/toolkit/runtime.go delete mode 100644 tools/container/toolkit/runtime_test.go diff --git a/tools/container/nvidia-toolkit/run.go b/tools/container/nvidia-toolkit/run.go index 265814a5..be8bf545 100644 --- a/tools/container/nvidia-toolkit/run.go +++ b/tools/container/nvidia-toolkit/run.go @@ -164,7 +164,7 @@ func Run(c *cli.Context, o *options) error { o.toolkitOptions.ContainerRuntimeRuntimes = *cli.NewStringSlice(lowlevelRuntimePaths...) } - err = toolkit.Install(c, &o.toolkitOptions, o.toolkitRoot()) + err = toolkit.Install(c, &o.toolkitOptions, o.toolkitRoot(), o.runtimeOptions.HostRootMount) if err != nil { return fmt.Errorf("unable to install toolkit: %v", err) } diff --git a/tools/container/toolkit/executable.go b/tools/container/toolkit/executable.go deleted file mode 100644 index 394ca007..00000000 --- a/tools/container/toolkit/executable.go +++ /dev/null @@ -1,153 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package toolkit - -import ( - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" - - log "github.com/sirupsen/logrus" -) - -type executableTarget struct { - dotfileName string - wrapperName string -} - -type executable struct { - source string - target executableTarget - env map[string]string - preLines []string - argLines []string -} - -// install installs an executable component of the NVIDIA container toolkit. The source executable -// is copied to a `.real` file and a wapper is created to set up the environment as required. -func (e executable) install(destFolder string) (string, error) { - log.Infof("Installing executable '%v' to %v", e.source, destFolder) - - dotfileName := e.dotfileName() - - installedDotfileName, err := installFileToFolderWithName(destFolder, dotfileName, e.source) - if err != nil { - return "", fmt.Errorf("error installing file '%v' as '%v': %v", e.source, dotfileName, err) - } - log.Infof("Installed '%v'", installedDotfileName) - - wrapperFilename, err := e.installWrapper(destFolder, installedDotfileName) - if err != nil { - return "", fmt.Errorf("error wrapping '%v': %v", installedDotfileName, err) - } - log.Infof("Installed wrapper '%v'", wrapperFilename) - - return wrapperFilename, nil -} - -func (e executable) dotfileName() string { - return e.target.dotfileName -} - -func (e executable) wrapperName() string { - return e.target.wrapperName -} - -func (e executable) installWrapper(destFolder string, dotfileName string) (string, error) { - wrapperPath := filepath.Join(destFolder, e.wrapperName()) - wrapper, err := os.Create(wrapperPath) - if err != nil { - return "", fmt.Errorf("error creating executable wrapper: %v", err) - } - defer wrapper.Close() - - err = e.writeWrapperTo(wrapper, destFolder, dotfileName) - if err != nil { - return "", fmt.Errorf("error writing wrapper contents: %v", err) - } - - err = ensureExecutable(wrapperPath) - if err != nil { - return "", fmt.Errorf("error making wrapper executable: %v", err) - } - return wrapperPath, nil -} - -func (e executable) writeWrapperTo(wrapper io.Writer, destFolder string, dotfileName string) error { - r := newReplacements(destDirPattern, destFolder) - - // Add the shebang - fmt.Fprintln(wrapper, "#! /bin/sh") - - // Add the preceding lines if any - for _, line := range e.preLines { - fmt.Fprintf(wrapper, "%s\n", r.apply(line)) - } - - // Update the path to include the destination folder - var env map[string]string - if e.env == nil { - env = make(map[string]string) - } else { - env = e.env - } - - path, specified := env["PATH"] - if !specified { - path = "$PATH" - } - env["PATH"] = strings.Join([]string{destFolder, path}, ":") - - var sortedEnvvars []string - for e := range env { - sortedEnvvars = append(sortedEnvvars, e) - } - sort.Strings(sortedEnvvars) - - for _, e := range sortedEnvvars { - v := env[e] - fmt.Fprintf(wrapper, "%s=%s \\\n", e, r.apply(v)) - } - // Add the call to the target executable - fmt.Fprintf(wrapper, "%s \\\n", dotfileName) - - // Insert additional lines in the `arg` list - for _, line := range e.argLines { - fmt.Fprintf(wrapper, "\t%s \\\n", r.apply(line)) - } - // Add the script arguments "$@" - fmt.Fprintln(wrapper, "\t\"$@\"") - - return nil -} - -// ensureExecutable is equivalent to running chmod +x on the specified file -func ensureExecutable(path string) error { - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("error getting file info for '%v': %v", path, err) - } - executableMode := info.Mode() | 0111 - err = os.Chmod(path, executableMode) - if err != nil { - return fmt.Errorf("error setting executable mode for '%v': %v", path, err) - } - return nil -} diff --git a/tools/container/toolkit/executable_test.go b/tools/container/toolkit/executable_test.go deleted file mode 100644 index 8cb47596..00000000 --- a/tools/container/toolkit/executable_test.go +++ /dev/null @@ -1,152 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package toolkit - -import ( - "bytes" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestWrapper(t *testing.T) { - const shebang = "#! /bin/sh" - const destFolder = "/dest/folder" - const dotfileName = "source.real" - - testCases := []struct { - e executable - expectedLines []string - }{ - { - e: executable{}, - expectedLines: []string{ - shebang, - "PATH=/dest/folder:$PATH \\", - "source.real \\", - "\t\"$@\"", - "", - }, - }, - { - e: executable{ - env: map[string]string{ - "PATH": "some-path", - }, - }, - expectedLines: []string{ - shebang, - "PATH=/dest/folder:some-path \\", - "source.real \\", - "\t\"$@\"", - "", - }, - }, - { - e: executable{ - preLines: []string{ - "preline1", - "preline2", - }, - }, - expectedLines: []string{ - shebang, - "preline1", - "preline2", - "PATH=/dest/folder:$PATH \\", - "source.real \\", - "\t\"$@\"", - "", - }, - }, - { - e: executable{ - argLines: []string{ - "argline1", - "argline2", - }, - }, - expectedLines: []string{ - shebang, - "PATH=/dest/folder:$PATH \\", - "source.real \\", - "\targline1 \\", - "\targline2 \\", - "\t\"$@\"", - "", - }, - }, - } - - for i, tc := range testCases { - buf := &bytes.Buffer{} - - err := tc.e.writeWrapperTo(buf, destFolder, dotfileName) - require.NoError(t, err) - - exepectedContents := strings.Join(tc.expectedLines, "\n") - require.Equal(t, exepectedContents, buf.String(), "%v: %v", i, tc) - } -} - -func TestInstallExecutable(t *testing.T) { - inputFolder, err := os.MkdirTemp("", "") - require.NoError(t, err) - defer os.RemoveAll(inputFolder) - - // Create the source file - source := filepath.Join(inputFolder, "input") - sourceFile, err := os.Create(source) - - base := filepath.Base(source) - - require.NoError(t, err) - require.NoError(t, sourceFile.Close()) - - e := executable{ - source: source, - target: executableTarget{ - dotfileName: "input.real", - wrapperName: "input", - }, - } - - destFolder, err := os.MkdirTemp("", "output-*") - require.NoError(t, err) - defer os.RemoveAll(destFolder) - - installed, err := e.install(destFolder) - - require.NoError(t, err) - require.Equal(t, filepath.Join(destFolder, base), installed) - - // Now check the post conditions: - sourceInfo, err := os.Stat(source) - require.NoError(t, err) - - destInfo, err := os.Stat(filepath.Join(destFolder, base+".real")) - require.NoError(t, err) - require.Equal(t, sourceInfo.Size(), destInfo.Size()) - require.Equal(t, sourceInfo.Mode(), destInfo.Mode()) - - wrapperInfo, err := os.Stat(installed) - require.NoError(t, err) - require.NotEqual(t, 0, wrapperInfo.Mode()&0111) -} diff --git a/tools/container/toolkit/installer/artifact-root.go b/tools/container/toolkit/installer/artifact-root.go new file mode 100644 index 00000000..23ca4b40 --- /dev/null +++ b/tools/container/toolkit/installer/artifact-root.go @@ -0,0 +1,102 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" +) + +type artifactRoot struct { + path string + libraries lookup.Locator + executables lookup.Locator +} + +func newArtifactRoot(packageType string) (*artifactRoot, error) { + path := "/" + switch packageType { + case "deb": + path = "/artifacts/deb" + case "rpm": + path = "/artifacts/rpm" + default: + return nil, fmt.Errorf("invalid package type: %v", packageType) + } + + a := artifactRoot{ + path: path, + libraries: lookup.NewLibraryLocator( + lookup.WithRoot(path), + lookup.WithCount(1), + lookup.WithSearchPaths( + "/usr/lib64", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + ), + ), + executables: lookup.NewExecutableLocator( + logger.New(), + path, + ), + } + + return &a, nil +} + +func resolvePackageType(hostRoot string, packageType string) (rPackageTypes string, rerr error) { + if packageType != "" && packageType != "auto" { + return packageType, nil + } + + if info, err := os.Stat(filepath.Join(hostRoot, "/usr/bin/rpm")); err != nil && !info.IsDir() { + return "rpm", nil + } + if info, err := os.Stat(filepath.Join(hostRoot, "/usr/bin/dpkg")); err != nil && !info.IsDir() { + return "deb", nil + } + + return "deb", nil +} + +func (r *artifactRoot) findLibrary(name string) (string, error) { + candidates, err := r.libraries.Locate(name) + if err != nil { + return "", fmt.Errorf("error locating library: %w", err) + } + if len(candidates) == 0 { + return "", fmt.Errorf("library %v not found", name) + } + + return candidates[0], nil +} + +func (r *artifactRoot) findExecutable(name string) (string, error) { + candidates, err := r.executables.Locate(name) + if err != nil { + return "", fmt.Errorf("error locating executable: %w", err) + } + if len(candidates) == 0 { + return "", fmt.Errorf("executable %v not found", name) + } + + return candidates[0], nil +} diff --git a/tools/container/toolkit/installer/executables.go b/tools/container/toolkit/installer/executables.go new file mode 100644 index 00000000..9a6fdb0d --- /dev/null +++ b/tools/container/toolkit/installer/executables.go @@ -0,0 +1,169 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +import ( + "bytes" + "fmt" + "html/template" + "io" + "path/filepath" + "strings" + + log "github.com/sirupsen/logrus" + + "github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator" +) + +type executable struct { + path string + symlink string + args []string + env map[string]string +} + +func (t *toolkitInstaller) collectExecutables(destDir string) ([]Installer, error) { + configHome := filepath.Join(destDir, ".config") + configDir := filepath.Join(configHome, "nvidia-container-runtime") + configPath := filepath.Join(configDir, "config.toml") + + executables := []executable{ + { + path: "nvidia-ctk", + }, + { + path: "nvidia-cdi-hook", + }, + } + for _, runtime := range operator.GetRuntimes() { + e := executable{ + path: runtime.Path, + env: map[string]string{ + "XDG_CONFIG_HOME": configHome, + }, + } + executables = append(executables, e) + } + executables = append(executables, + executable{ + path: "nvidia-container-cli", + env: map[string]string{"LD_LIBRARY_PATH": destDir + ":$LD_LIBRARY_PATH"}, + }, + ) + + executables = append(executables, + executable{ + path: "nvidia-container-runtime-hook", + symlink: "nvidia-container-toolkit", + args: []string{fmt.Sprintf("-config %s", configPath)}, + }, + ) + + var installers []Installer + for _, executable := range executables { + executablePath, err := t.artifactRoot.findExecutable(executable.path) + if err != nil { + if t.ignoreErrors { + log.Errorf("Ignoring error: %v", err) + continue + } + return nil, err + } + + wrappedExecutableFilename := filepath.Base(executablePath) + dotRealFilename := wrappedExecutableFilename + ".real" + + w := &wrapper{ + Source: executablePath, + WrappedExecutable: dotRealFilename, + Args: executable.args, + Envvars: map[string]string{ + "PATH": strings.Join([]string{destDir, "$PATH"}, ":"), + }, + } + for k, v := range executable.env { + w.Envvars[k] = v + } + + installers = append(installers, w) + + if executable.symlink == "" { + continue + } + link := symlink{ + linkname: executable.symlink, + target: filepath.Base(executablePath), + } + installers = append(installers, link) + } + + return installers, nil + +} + +type wrapper struct { + Source string + Envvars map[string]string + WrappedExecutable string + Args []string +} + +func (w *wrapper) Install(destDir string) error { + // Copy the executable with a .real extension. + mode, err := installFile(w.Source, filepath.Join(destDir, w.WrappedExecutable)) + if err != nil { + return err + } + + // Create a wrapper file. + content, err := w.render() + if err != nil { + return nil + } + wrapperFile := filepath.Join(destDir, filepath.Base(w.Source)) + return installContent(content, wrapperFile, mode|0111) +} + +func (w *wrapper) render() (io.Reader, error) { + wrapperTemplate := `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +{{- range $key, $value := .Envvars }} +{{$key}}={{$value}} \ +{{- end }} + {{ .WrappedExecutable }} \ +{{- range $arg := .Args }} + {{$arg}} \ +{{- end }} + "$@" +` + + var content bytes.Buffer + tmpl, err := template.New("wrapper").Parse(wrapperTemplate) + if err != nil { + return nil, err + } + if err := tmpl.Execute(&content, w); err != nil { + return nil, err + } + + return &content, nil +} diff --git a/tools/container/toolkit/installer/executables_test.go b/tools/container/toolkit/installer/executables_test.go new file mode 100644 index 00000000..18e09cad --- /dev/null +++ b/tools/container/toolkit/installer/executables_test.go @@ -0,0 +1,101 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWrapperRender(t *testing.T) { + testCases := []struct { + description string + w *wrapper + expected string + }{ + { + description: "executable is added", + w: &wrapper{ + WrappedExecutable: "some-runtime", + }, + expected: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi + some-runtime \ + "$@" +`, + }, + { + description: "environment is added", + w: &wrapper{ + WrappedExecutable: "some-runtime", + Envvars: map[string]string{ + "PATH": "/foo/bar/baz", + }, + }, + expected: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz \ + some-runtime \ + "$@" +`, + }, + { + description: "args are added", + w: &wrapper{ + WrappedExecutable: "some-runtime", + Args: []string{"--config foo", "bar"}, + }, + expected: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi + some-runtime \ + --config foo \ + bar \ + "$@" +`, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + reader, err := tc.w.render() + require.NoError(t, err) + + var content bytes.Buffer + _, err = content.ReadFrom(reader) + require.NoError(t, err) + + require.Equal(t, tc.expected, content.String()) + }) + } +} diff --git a/tools/container/toolkit/installer/file-installer_mock.go b/tools/container/toolkit/installer/file-installer_mock.go new file mode 100644 index 00000000..7ec594b8 --- /dev/null +++ b/tools/container/toolkit/installer/file-installer_mock.go @@ -0,0 +1,188 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package installer + +import ( + "io" + "io/fs" + "sync" +) + +// Ensure, that fileInstallerMock does implement fileInstaller. +// If this is not the case, regenerate this file with moq. +var _ fileInstaller = &fileInstallerMock{} + +// fileInstallerMock is a mock implementation of fileInstaller. +// +// func TestSomethingThatUsesfileInstaller(t *testing.T) { +// +// // make and configure a mocked fileInstaller +// mockedfileInstaller := &fileInstallerMock{ +// installContentFunc: func(reader io.Reader, s string, fileMode fs.FileMode) error { +// panic("mock out the installContent method") +// }, +// installFileFunc: func(s1 string, s2 string) (fs.FileMode, error) { +// panic("mock out the installFile method") +// }, +// installSymlinkFunc: func(s1 string, s2 string) error { +// panic("mock out the installSymlink method") +// }, +// } +// +// // use mockedfileInstaller in code that requires fileInstaller +// // and then make assertions. +// +// } +type fileInstallerMock struct { + // installContentFunc mocks the installContent method. + installContentFunc func(reader io.Reader, s string, fileMode fs.FileMode) error + + // installFileFunc mocks the installFile method. + installFileFunc func(s1 string, s2 string) (fs.FileMode, error) + + // installSymlinkFunc mocks the installSymlink method. + installSymlinkFunc func(s1 string, s2 string) error + + // calls tracks calls to the methods. + calls struct { + // installContent holds details about calls to the installContent method. + installContent []struct { + // Reader is the reader argument value. + Reader io.Reader + // S is the s argument value. + S string + // FileMode is the fileMode argument value. + FileMode fs.FileMode + } + // installFile holds details about calls to the installFile method. + installFile []struct { + // S1 is the s1 argument value. + S1 string + // S2 is the s2 argument value. + S2 string + } + // installSymlink holds details about calls to the installSymlink method. + installSymlink []struct { + // S1 is the s1 argument value. + S1 string + // S2 is the s2 argument value. + S2 string + } + } + lockinstallContent sync.RWMutex + lockinstallFile sync.RWMutex + lockinstallSymlink sync.RWMutex +} + +// installContent calls installContentFunc. +func (mock *fileInstallerMock) installContent(reader io.Reader, s string, fileMode fs.FileMode) error { + if mock.installContentFunc == nil { + panic("fileInstallerMock.installContentFunc: method is nil but fileInstaller.installContent was just called") + } + callInfo := struct { + Reader io.Reader + S string + FileMode fs.FileMode + }{ + Reader: reader, + S: s, + FileMode: fileMode, + } + mock.lockinstallContent.Lock() + mock.calls.installContent = append(mock.calls.installContent, callInfo) + mock.lockinstallContent.Unlock() + return mock.installContentFunc(reader, s, fileMode) +} + +// installContentCalls gets all the calls that were made to installContent. +// Check the length with: +// +// len(mockedfileInstaller.installContentCalls()) +func (mock *fileInstallerMock) installContentCalls() []struct { + Reader io.Reader + S string + FileMode fs.FileMode +} { + var calls []struct { + Reader io.Reader + S string + FileMode fs.FileMode + } + mock.lockinstallContent.RLock() + calls = mock.calls.installContent + mock.lockinstallContent.RUnlock() + return calls +} + +// installFile calls installFileFunc. +func (mock *fileInstallerMock) installFile(s1 string, s2 string) (fs.FileMode, error) { + if mock.installFileFunc == nil { + panic("fileInstallerMock.installFileFunc: method is nil but fileInstaller.installFile was just called") + } + callInfo := struct { + S1 string + S2 string + }{ + S1: s1, + S2: s2, + } + mock.lockinstallFile.Lock() + mock.calls.installFile = append(mock.calls.installFile, callInfo) + mock.lockinstallFile.Unlock() + return mock.installFileFunc(s1, s2) +} + +// installFileCalls gets all the calls that were made to installFile. +// Check the length with: +// +// len(mockedfileInstaller.installFileCalls()) +func (mock *fileInstallerMock) installFileCalls() []struct { + S1 string + S2 string +} { + var calls []struct { + S1 string + S2 string + } + mock.lockinstallFile.RLock() + calls = mock.calls.installFile + mock.lockinstallFile.RUnlock() + return calls +} + +// installSymlink calls installSymlinkFunc. +func (mock *fileInstallerMock) installSymlink(s1 string, s2 string) error { + if mock.installSymlinkFunc == nil { + panic("fileInstallerMock.installSymlinkFunc: method is nil but fileInstaller.installSymlink was just called") + } + callInfo := struct { + S1 string + S2 string + }{ + S1: s1, + S2: s2, + } + mock.lockinstallSymlink.Lock() + mock.calls.installSymlink = append(mock.calls.installSymlink, callInfo) + mock.lockinstallSymlink.Unlock() + return mock.installSymlinkFunc(s1, s2) +} + +// installSymlinkCalls gets all the calls that were made to installSymlink. +// Check the length with: +// +// len(mockedfileInstaller.installSymlinkCalls()) +func (mock *fileInstallerMock) installSymlinkCalls() []struct { + S1 string + S2 string +} { + var calls []struct { + S1 string + S2 string + } + mock.lockinstallSymlink.RLock() + calls = mock.calls.installSymlink + mock.lockinstallSymlink.RUnlock() + return calls +} diff --git a/tools/container/toolkit/installer/installer.go b/tools/container/toolkit/installer/installer.go new file mode 100644 index 00000000..cfd164f2 --- /dev/null +++ b/tools/container/toolkit/installer/installer.go @@ -0,0 +1,153 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +type Installer interface { + Install(string) error +} + +func New(opts ...Option) (Installer, error) { + t := &toolkitInstaller{} + for _, opt := range opts { + opt(t) + } + + if t.artifactRoot == nil { + resolvedPackageType, err := resolvePackageType(t.hostRoot, t.packageType) + if err != nil { + return nil, err + } + artifactRoot, err := newArtifactRoot(resolvedPackageType) + if err != nil { + return nil, err + } + t.artifactRoot = artifactRoot + } + + return t, nil +} + +type toolkitInstaller struct { + artifactRoot *artifactRoot + + hostRoot string + packageType string + + ignoreErrors bool +} + +// Install ensures that the required toolkit files are installed in the specified directory. +// The process is as follows: +func (t *toolkitInstaller) Install(destDir string) error { + var installers []Installer + + libraries, err := t.collectLibraries() + if err != nil { + return fmt.Errorf("failed to collect libraries: %w", err) + } + installers = append(installers, libraries...) + + executables, err := t.collectExecutables(destDir) + if err != nil { + return fmt.Errorf("failed to collect executables: %w", err) + } + installers = append(installers, executables...) + + var errs error + for _, i := range installers { + errs = errors.Join(errs, i.Install(destDir)) + } + + return errs +} + +type symlink struct { + linkname string + target string +} + +func (s symlink) Install(destDir string) error { + symlinkPath := filepath.Join(destDir, s.linkname) + return installSymlink(s.target, symlinkPath) +} + +//go:generate moq -rm -out file-installer_mock.go . fileInstaller +type fileInstaller interface { + installContent(io.Reader, string, os.FileMode) error + installFile(string, string) (os.FileMode, error) + installSymlink(string, string) error +} + +var installSymlink = installSymlinkStub + +func installSymlinkStub(target string, link string) error { + err := os.Symlink(target, link) + if err != nil { + return fmt.Errorf("error creating symlink '%v' => '%v': %v", link, target, err) + } + return nil +} + +var installFile = installFileStub + +func installFileStub(src string, dest string) (os.FileMode, error) { + sourceInfo, err := os.Stat(src) + if err != nil { + return 0, fmt.Errorf("error getting file info for '%v': %v", src, err) + } + + source, err := os.Open(src) + if err != nil { + return 0, fmt.Errorf("error opening source: %w", err) + } + defer source.Close() + + mode := sourceInfo.Mode() + if err := installContent(source, dest, mode); err != nil { + return 0, err + } + return mode, nil +} + +var installContent = installContentStub + +func installContentStub(content io.Reader, dest string, mode fs.FileMode) error { + destination, err := os.Create(dest) + if err != nil { + return fmt.Errorf("error creating destination: %w", err) + } + defer destination.Close() + + _, err = io.Copy(destination, content) + if err != nil { + return fmt.Errorf("error copying file: %w", err) + } + err = os.Chmod(dest, mode) + if err != nil { + return fmt.Errorf("error setting mode for '%v': %v", dest, err) + } + return nil +} diff --git a/tools/container/toolkit/installer/installer_test.go b/tools/container/toolkit/installer/installer_test.go new file mode 100644 index 00000000..2d538449 --- /dev/null +++ b/tools/container/toolkit/installer/installer_test.go @@ -0,0 +1,258 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" +) + +func TestToolkitInstaller(t *testing.T) { + type contentCall struct { + wrapper string + path string + mode fs.FileMode + } + var contentCalls []contentCall + + installer := &fileInstallerMock{ + installFileFunc: func(s1, s2 string) (os.FileMode, error) { + return 0666, nil + }, + installContentFunc: func(reader io.Reader, s string, fileMode fs.FileMode) error { + var b bytes.Buffer + if _, err := b.ReadFrom(reader); err != nil { + return err + } + contents := contentCall{ + wrapper: b.String(), + path: s, + mode: fileMode, + } + + contentCalls = append(contentCalls, contents) + return nil + }, + installSymlinkFunc: func(s1, s2 string) error { + return nil + }, + } + installFile = installer.installFile + installContent = installer.installContent + installSymlink = installer.installSymlink + + root := "/artifacts/test" + libraries := &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + switch s { + case "libnvidia-container.so.1": + return []string{filepath.Join(root, "libnvidia-container.so.987.65.43")}, nil + case "libnvidia-container-go.so.1": + return []string{filepath.Join(root, "libnvidia-container-go.so.1.23.4")}, nil + } + return nil, fmt.Errorf("%v not found", s) + }, + } + executables := &lookup.LocatorMock{ + LocateFunc: func(s string) ([]string, error) { + switch s { + case "/usr/bin/nvidia-container-runtime.cdi": + fallthrough + case "/usr/bin/nvidia-container-runtime.legacy": + fallthrough + case "/usr/bin/nvidia-container-runtime": + return []string{filepath.Join(root, s)}, nil + case "nvidia-ctk": + fallthrough + case "nvidia-container-cli": + fallthrough + case "nvidia-container-runtime-hook": + fallthrough + case "nvidia-cdi-hook": + return []string{filepath.Join(root, "usr/bin", s)}, nil + } + return nil, fmt.Errorf("%v not found", s) + }, + } + + r := &artifactRoot{ + libraries: libraries, + executables: executables, + } + + i := toolkitInstaller{ + artifactRoot: r, + } + + err := i.Install("/foo/bar/baz") + require.NoError(t, err) + + require.ElementsMatch(t, + installer.installFileCalls(), + []struct { + S1 string + S2 string + }{ + {"/artifacts/test/libnvidia-container-go.so.1.23.4", "/foo/bar/baz/libnvidia-container-go.so.1.23.4"}, + {"/artifacts/test/libnvidia-container.so.987.65.43", "/foo/bar/baz/libnvidia-container.so.987.65.43"}, + {"/artifacts/test/usr/bin/nvidia-container-runtime.cdi", "/foo/bar/baz/nvidia-container-runtime.cdi.real"}, + {"/artifacts/test/usr/bin/nvidia-container-runtime.legacy", "/foo/bar/baz/nvidia-container-runtime.legacy.real"}, + {"/artifacts/test/usr/bin/nvidia-container-runtime", "/foo/bar/baz/nvidia-container-runtime.real"}, + {"/artifacts/test/usr/bin/nvidia-ctk", "/foo/bar/baz/nvidia-ctk.real"}, + {"/artifacts/test/usr/bin/nvidia-cdi-hook", "/foo/bar/baz/nvidia-cdi-hook.real"}, + {"/artifacts/test/usr/bin/nvidia-container-cli", "/foo/bar/baz/nvidia-container-cli.real"}, + {"/artifacts/test/usr/bin/nvidia-container-runtime-hook", "/foo/bar/baz/nvidia-container-runtime-hook.real"}, + }, + ) + + require.ElementsMatch(t, + installer.installSymlinkCalls(), + []struct { + S1 string + S2 string + }{ + {"libnvidia-container-go.so.1.23.4", "/foo/bar/baz/libnvidia-container-go.so.1"}, + {"libnvidia-container.so.987.65.43", "/foo/bar/baz/libnvidia-container.so.1"}, + {"nvidia-container-runtime-hook", "/foo/bar/baz/nvidia-container-toolkit"}, + }, + ) + + require.ElementsMatch(t, + contentCalls, + []contentCall{ + { + path: "/foo/bar/baz/nvidia-container-runtime", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz:$PATH \ +XDG_CONFIG_HOME=/foo/bar/baz/.config \ + nvidia-container-runtime.real \ + "$@" +`, + }, + { + path: "/foo/bar/baz/nvidia-container-runtime.cdi", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz:$PATH \ +XDG_CONFIG_HOME=/foo/bar/baz/.config \ + nvidia-container-runtime.cdi.real \ + "$@" +`, + }, + { + path: "/foo/bar/baz/nvidia-container-runtime.legacy", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz:$PATH \ +XDG_CONFIG_HOME=/foo/bar/baz/.config \ + nvidia-container-runtime.legacy.real \ + "$@" +`, + }, + { + path: "/foo/bar/baz/nvidia-ctk", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz:$PATH \ + nvidia-ctk.real \ + "$@" +`, + }, + { + path: "/foo/bar/baz/nvidia-cdi-hook", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz:$PATH \ + nvidia-cdi-hook.real \ + "$@" +`, + }, + { + path: "/foo/bar/baz/nvidia-container-cli", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +LD_LIBRARY_PATH=/foo/bar/baz:$LD_LIBRARY_PATH \ +PATH=/foo/bar/baz:$PATH \ + nvidia-container-cli.real \ + "$@" +`, + }, + { + path: "/foo/bar/baz/nvidia-container-runtime-hook", + mode: 0777, + wrapper: `#! /bin/sh + +cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1 +if [ "${?}" != "0" ]; then + echo "nvidia driver modules are not yet loaded, invoking runc directly" + exec runc "$@" +fi +PATH=/foo/bar/baz:$PATH \ + nvidia-container-runtime-hook.real \ + -config /foo/bar/baz/.config/nvidia-container-runtime/config.toml \ + "$@" +`, + }, + }, + ) +} diff --git a/tools/container/toolkit/installer/libraries.go b/tools/container/toolkit/installer/libraries.go new file mode 100644 index 00000000..85031615 --- /dev/null +++ b/tools/container/toolkit/installer/libraries.go @@ -0,0 +1,72 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +import ( + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +// collectLibraries locates and installs the libraries that are part of +// the nvidia-container-toolkit. +// A predefined set of library candidates are considered, with the first one +// resulting in success being installed to the toolkit folder. The install process +// resolves the symlink for the library and copies the versioned library itself. +func (t *toolkitInstaller) collectLibraries() ([]Installer, error) { + requiredLibraries := []string{ + "libnvidia-container.so.1", + "libnvidia-container-go.so.1", + } + + var installers []Installer + for _, l := range requiredLibraries { + libraryPath, err := t.artifactRoot.findLibrary(l) + if err != nil { + if t.ignoreErrors { + log.Errorf("Ignoring error: %v", err) + continue + } + return nil, err + } + + installers = append(installers, library(libraryPath)) + + if filepath.Base(libraryPath) == l { + continue + } + + link := symlink{ + linkname: l, + target: filepath.Base(libraryPath), + } + installers = append(installers, link) + } + + return installers, nil +} + +type library string + +// Install copies the library l to the destination folder. +// The same basename is used in the destination folder. +func (l library) Install(destinationDir string) error { + dest := filepath.Join(destinationDir, filepath.Base(string(l))) + + _, err := installFile(string(l), dest) + return err +} diff --git a/tools/container/toolkit/installer/options.go b/tools/container/toolkit/installer/options.go new file mode 100644 index 00000000..3fb2ba05 --- /dev/null +++ b/tools/container/toolkit/installer/options.go @@ -0,0 +1,43 @@ +/** +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package installer + +type Option func(*toolkitInstaller) + +func WithArtifactRoot(artifactRoot *artifactRoot) Option { + return func(ti *toolkitInstaller) { + ti.artifactRoot = artifactRoot + } +} + +func WithIgnoreErrors(ignoreErrors bool) Option { + return func(ti *toolkitInstaller) { + ti.ignoreErrors = ignoreErrors + } +} + +func WithPackageType(packageType string) Option { + return func(ti *toolkitInstaller) { + ti.packageType = packageType + } +} + +func WithHostRoot(hostRoot string) Option { + return func(ti *toolkitInstaller) { + ti.hostRoot = hostRoot + } +} diff --git a/tools/container/toolkit/replacements.go b/tools/container/toolkit/replacements.go deleted file mode 100644 index 7605093c..00000000 --- a/tools/container/toolkit/replacements.go +++ /dev/null @@ -1,45 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package toolkit - -import "strings" - -const ( - destDirPattern = "@destDir@" -) - -type replacements map[string]string - -func newReplacements(rules ...string) replacements { - r := make(replacements) - for i := 0; i < len(rules)-1; i += 2 { - old := rules[i] - new := rules[i+1] - - r[old] = new - } - - return r -} - -func (r replacements) apply(input string) string { - output := input - for old, new := range r { - output = strings.ReplaceAll(output, old, new) - } - return output -} diff --git a/tools/container/toolkit/runtime.go b/tools/container/toolkit/runtime.go deleted file mode 100644 index bdfca983..00000000 --- a/tools/container/toolkit/runtime.go +++ /dev/null @@ -1,84 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package toolkit - -import ( - "fmt" - "path/filepath" - - "github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator" -) - -const ( - nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime" -) - -// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables -// and implementing the required wrapper -func installContainerRuntimes(toolkitDir string, driverRoot string) error { - runtimes := operator.GetRuntimes() - for _, runtime := range runtimes { - r := newNvidiaContainerRuntimeInstaller(runtime.Path) - - _, err := r.install(toolkitDir) - if err != nil { - return fmt.Errorf("error installing NVIDIA container runtime: %v", err) - } - } - return nil -} - -// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime. -// This installer will copy the specified source executable to the toolkit directory. -// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is -// created to allow for the configuration of the runtime environment. -func newNvidiaContainerRuntimeInstaller(source string) *executable { - wrapperName := filepath.Base(source) - dotfileName := wrapperName + ".real" - target := executableTarget{ - dotfileName: dotfileName, - wrapperName: wrapperName, - } - return newRuntimeInstaller(source, target, nil) -} - -func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable { - preLines := []string{ - "", - "cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1", - "if [ \"${?}\" != \"0\" ]; then", - " echo \"nvidia driver modules are not yet loaded, invoking runc directly\"", - " exec runc \"$@\"", - "fi", - "", - } - - runtimeEnv := make(map[string]string) - runtimeEnv["XDG_CONFIG_HOME"] = filepath.Join(destDirPattern, ".config") - for k, v := range env { - runtimeEnv[k] = v - } - - r := executable{ - source: source, - target: target, - env: runtimeEnv, - preLines: preLines, - } - - return &r -} diff --git a/tools/container/toolkit/runtime_test.go b/tools/container/toolkit/runtime_test.go deleted file mode 100644 index d2841506..00000000 --- a/tools/container/toolkit/runtime_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package toolkit - -import ( - "bytes" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) { - r := newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource) - - const shebang = "#! /bin/sh" - const destFolder = "/dest/folder" - const dotfileName = "source.real" - - buf := &bytes.Buffer{} - - err := r.writeWrapperTo(buf, destFolder, dotfileName) - require.NoError(t, err) - - expectedLines := []string{ - shebang, - "", - "cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1", - "if [ \"${?}\" != \"0\" ]; then", - " echo \"nvidia driver modules are not yet loaded, invoking runc directly\"", - " exec runc \"$@\"", - "fi", - "", - "PATH=/dest/folder:$PATH \\", - "XDG_CONFIG_HOME=/dest/folder/.config \\", - "source.real \\", - "\t\"$@\"", - "", - } - - exepectedContents := strings.Join(expectedLines, "\n") - require.Equal(t, exepectedContents, buf.String()) -} diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index 43e68ca5..6ccc5239 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -19,7 +19,6 @@ package toolkit import ( "errors" "fmt" - "io" "os" "path/filepath" "strings" @@ -33,15 +32,13 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices" "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi" transformroot "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform/root" + "github.com/NVIDIA/nvidia-container-toolkit/tools/container/toolkit/installer" ) const ( // DefaultNvidiaDriverRoot specifies the default NVIDIA driver run directory DefaultNvidiaDriverRoot = "/run/nvidia/driver" - nvidiaContainerCliSource = "/usr/bin/nvidia-container-cli" - nvidiaContainerRuntimeHookSource = "/usr/bin/nvidia-container-runtime-hook" - nvidiaContainerToolkitConfigSource = "/etc/nvidia-container-runtime/config.toml" configFilename = "config.toml" @@ -81,6 +78,8 @@ type Options struct { ignoreErrors bool optInFeatures cli.StringSlice + + packageType string } func Flags(opts *Options) []cli.Flag { @@ -210,6 +209,13 @@ func Flags(opts *Options) []cli.Flag { Destination: &opts.optInFeatures, EnvVars: []string{"NVIDIA_CONTAINER_TOOLKIT_OPT_IN_FEATURES"}, }, + &cli.StringFlag{ + Name: "package-type", + Usage: "specify the package type to use. One of ['deb', 'rpm', 'auto', '']. If 'auto' or '' are used, the type is inferred automatically.", + Value: "auto", + Destination: &opts.packageType, + EnvVars: []string{"PACKAGE_TYPE"}, + }, } return flags @@ -286,7 +292,7 @@ func TryDelete(cli *cli.Context, toolkitRoot string) error { // Install installs the components of the NVIDIA container toolkit. // Any existing installation is removed. -func Install(cli *cli.Context, opts *Options, toolkitRoot string) error { +func Install(cli *cli.Context, opts *Options, toolkitRoot string, hostRoot string) error { log.Infof("Installing NVIDIA container toolkit to '%v'", toolkitRoot) log.Infof("Removing existing NVIDIA container toolkit installation") @@ -307,48 +313,28 @@ func Install(cli *cli.Context, opts *Options, toolkitRoot string) error { log.Errorf("Ignoring error: %v", fmt.Errorf("could not create required directories: %v", err)) } - err = installContainerLibraries(toolkitRoot) - if err != nil && !opts.ignoreErrors { - return fmt.Errorf("error installing NVIDIA container library: %v", err) - } else if err != nil { - log.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container library: %v", err)) - } - - err = installContainerRuntimes(toolkitRoot, opts.DriverRoot) - if err != nil && !opts.ignoreErrors { - return fmt.Errorf("error installing NVIDIA container runtime: %v", err) - } else if err != nil { - log.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container runtime: %v", err)) - } - - nvidiaContainerCliExecutable, err := installContainerCLI(toolkitRoot) - if err != nil && !opts.ignoreErrors { - return fmt.Errorf("error installing NVIDIA container CLI: %v", err) - } else if err != nil { - log.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container CLI: %v", err)) - } - - nvidiaContainerRuntimeHookPath, err := installRuntimeHook(toolkitRoot, toolkitConfigPath) - if err != nil && !opts.ignoreErrors { - return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err) - } else if err != nil { - log.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)) - } - - nvidiaCTKPath, err := installContainerToolkitCLI(toolkitRoot) - if err != nil && !opts.ignoreErrors { - return fmt.Errorf("error installing NVIDIA Container Toolkit CLI: %v", err) - } else if err != nil { - log.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA Container Toolkit CLI: %v", err)) - } - - nvidiaCDIHookPath, err := installContainerCDIHookCLI(toolkitRoot) - if err != nil && !opts.ignoreErrors { - return fmt.Errorf("error installing NVIDIA Container CDI Hook CLI: %v", err) - } else if err != nil { - log.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA Container CDI Hook CLI: %v", err)) + toolkit, err := installer.New( + installer.WithHostRoot(hostRoot), + installer.WithPackageType(opts.packageType), + ) + if err != nil { + if !opts.ignoreErrors { + return fmt.Errorf("could not create toolkit installer: %w", err) + } + log.Errorf("Ignoring error: %v", fmt.Errorf("could not create toolkit installer: %w", err)) + } + if err := toolkit.Install(toolkitRoot); err != nil { + if !opts.ignoreErrors { + return fmt.Errorf("could not install toolkit components: %w", err) + } + log.Errorf("Ignoring error: %v", fmt.Errorf("could not install toolkit components: %w", err)) } + // TODO: The toolkit config installation should also use the installer. + nvidiaContainerCliExecutable := filepath.Join(toolkitRoot, "nvidia-container-cli") + nvidiaCTKPath := filepath.Join(toolkitRoot, "nvidia-ctk") + nvidiaCDIHookPath := filepath.Join(toolkitRoot, "nvidia-cdi-hook") + nvidiaContainerRuntimeHookPath := filepath.Join(toolkitRoot, "nvidia-container-runtime-hook") err = installToolkitConfig(cli, toolkitConfigPath, nvidiaContainerCliExecutable, nvidiaCTKPath, nvidiaContainerRuntimeHookPath, opts) if err != nil && !opts.ignoreErrors { return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err) @@ -373,54 +359,6 @@ func Install(cli *cli.Context, opts *Options, toolkitRoot string) error { return nil } -// installContainerLibraries locates and installs the libraries that are part of -// the nvidia-container-toolkit. -// A predefined set of library candidates are considered, with the first one -// resulting in success being installed to the toolkit folder. The install process -// resolves the symlink for the library and copies the versioned library itself. -func installContainerLibraries(toolkitRoot string) error { - log.Infof("Installing NVIDIA container library to '%v'", toolkitRoot) - - libs := []string{ - "libnvidia-container.so.1", - "libnvidia-container-go.so.1", - } - - for _, l := range libs { - err := installLibrary(l, toolkitRoot) - if err != nil { - return fmt.Errorf("failed to install %s: %v", l, err) - } - } - - return nil -} - -// installLibrary installs the specified library to the toolkit directory. -func installLibrary(libName string, toolkitRoot string) error { - libraryPath, err := findLibrary("", libName) - if err != nil { - return fmt.Errorf("error locating NVIDIA container library: %v", err) - } - - installedLibPath, err := installFileToFolder(toolkitRoot, libraryPath) - if err != nil { - return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitRoot, err) - } - log.Infof("Installed '%v' to '%v'", libraryPath, installedLibPath) - - if filepath.Base(installedLibPath) == libName { - return nil - } - - err = installSymlink(toolkitRoot, libName, installedLibPath) - if err != nil { - return fmt.Errorf("error installing symlink for NVIDIA container library: %v", err) - } - - return nil -} - // installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring // that the settings are updated to match the desired install and nvidia driver directories. func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContainerCliExecutablePath string, nvidiaCTKPath string, nvidaContainerRuntimeHookPath string, opts *Options) error { @@ -523,205 +461,6 @@ func installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContai return nil } -// installContainerToolkitCLI installs the nvidia-ctk CLI executable and wrapper. -func installContainerToolkitCLI(toolkitDir string) (string, error) { - e := executable{ - source: "/usr/bin/nvidia-ctk", - target: executableTarget{ - dotfileName: "nvidia-ctk.real", - wrapperName: "nvidia-ctk", - }, - } - - return e.install(toolkitDir) -} - -// installContainerCDIHookCLI installs the nvidia-cdi-hook CLI executable and wrapper. -func installContainerCDIHookCLI(toolkitDir string) (string, error) { - e := executable{ - source: "/usr/bin/nvidia-cdi-hook", - target: executableTarget{ - dotfileName: "nvidia-cdi-hook.real", - wrapperName: "nvidia-cdi-hook", - }, - } - - return e.install(toolkitDir) -} - -// installContainerCLI sets up the NVIDIA container CLI executable, copying the executable -// and implementing the required wrapper -func installContainerCLI(toolkitRoot string) (string, error) { - log.Infof("Installing NVIDIA container CLI from '%v'", nvidiaContainerCliSource) - - env := map[string]string{ - "LD_LIBRARY_PATH": toolkitRoot, - } - - e := executable{ - source: nvidiaContainerCliSource, - target: executableTarget{ - dotfileName: "nvidia-container-cli.real", - wrapperName: "nvidia-container-cli", - }, - env: env, - } - - installedPath, err := e.install(toolkitRoot) - if err != nil { - return "", fmt.Errorf("error installing NVIDIA container CLI: %v", err) - } - return installedPath, nil -} - -// installRuntimeHook sets up the NVIDIA runtime hook, copying the executable -// and implementing the required wrapper -func installRuntimeHook(toolkitRoot string, configFilePath string) (string, error) { - log.Infof("Installing NVIDIA container runtime hook from '%v'", nvidiaContainerRuntimeHookSource) - - argLines := []string{ - fmt.Sprintf("-config \"%s\"", configFilePath), - } - - e := executable{ - source: nvidiaContainerRuntimeHookSource, - target: executableTarget{ - dotfileName: "nvidia-container-runtime-hook.real", - wrapperName: "nvidia-container-runtime-hook", - }, - argLines: argLines, - } - - installedPath, err := e.install(toolkitRoot) - if err != nil { - return "", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err) - } - - err = installSymlink(toolkitRoot, "nvidia-container-toolkit", installedPath) - if err != nil { - return "", fmt.Errorf("error installing symlink to NVIDIA container runtime hook: %v", err) - } - - return installedPath, nil -} - -// installSymlink creates a symlink in the toolkitDirectory that points to the specified target. -// Note: The target is assumed to be local to the toolkit directory -func installSymlink(toolkitRoot string, link string, target string) error { - symlinkPath := filepath.Join(toolkitRoot, link) - targetPath := filepath.Base(target) - log.Infof("Creating symlink '%v' -> '%v'", symlinkPath, targetPath) - - err := os.Symlink(targetPath, symlinkPath) - if err != nil { - return fmt.Errorf("error creating symlink '%v' => '%v': %v", symlinkPath, targetPath, err) - } - return nil -} - -// installFileToFolder copies a source file to a destination folder. -// The path of the input file is ignored. -// e.g. installFileToFolder("/some/path/file.txt", "/output/path") -// will result in a file "/output/path/file.txt" being generated -func installFileToFolder(destFolder string, src string) (string, error) { - name := filepath.Base(src) - return installFileToFolderWithName(destFolder, name, src) -} - -// cp src destFolder/name -func installFileToFolderWithName(destFolder string, name, src string) (string, error) { - dest := filepath.Join(destFolder, name) - err := installFile(dest, src) - if err != nil { - return "", fmt.Errorf("error copying '%v' to '%v': %v", src, dest, err) - } - return dest, nil -} - -// installFile copies a file from src to dest and maintains -// file modes -func installFile(dest string, src string) error { - log.Infof("Installing '%v' to '%v'", src, dest) - - source, err := os.Open(src) - if err != nil { - return fmt.Errorf("error opening source: %v", err) - } - defer source.Close() - - destination, err := os.Create(dest) - if err != nil { - return fmt.Errorf("error creating destination: %v", err) - } - defer destination.Close() - - _, err = io.Copy(destination, source) - if err != nil { - return fmt.Errorf("error copying file: %v", err) - } - - err = applyModeFromSource(dest, src) - if err != nil { - return fmt.Errorf("error setting destination file mode: %v", err) - } - return nil -} - -// applyModeFromSource sets the file mode for a destination file -// to match that of a specified source file -func applyModeFromSource(dest string, src string) error { - sourceInfo, err := os.Stat(src) - if err != nil { - return fmt.Errorf("error getting file info for '%v': %v", src, err) - } - err = os.Chmod(dest, sourceInfo.Mode()) - if err != nil { - return fmt.Errorf("error setting mode for '%v': %v", dest, err) - } - return nil -} - -// findLibrary searches a set of candidate libraries in the specified root for -// a given library name -func findLibrary(root string, libName string) (string, error) { - log.Infof("Finding library %v (root=%v)", libName, root) - - candidateDirs := []string{ - "/usr/lib64", - "/usr/lib/x86_64-linux-gnu", - "/usr/lib/aarch64-linux-gnu", - } - - for _, d := range candidateDirs { - l := filepath.Join(root, d, libName) - log.Infof("Checking library candidate '%v'", l) - - libraryCandidate, err := resolveLink(l) - if err != nil { - log.Infof("Skipping library candidate '%v': %v", l, err) - continue - } - - return libraryCandidate, nil - } - - return "", fmt.Errorf("error locating library '%v'", libName) -} - -// resolveLink finds the target of a symlink or the file itself in the -// case of a regular file. -// This is equivalent to running `readlink -f ${l}` -func resolveLink(l string) (string, error) { - resolved, err := filepath.EvalSymlinks(l) - if err != nil { - return "", fmt.Errorf("error resolving link '%v': %v", l, err) - } - if l != resolved { - log.Infof("Resolved link: '%v' => '%v'", l, resolved) - } - return resolved, nil -} - func createDirectories(dir ...string) error { for _, d := range dir { log.Infof("Creating directory '%v'", d)