From d970d0a627b92befc7ab0219b5f0a153fe875c24 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 14 Mar 2022 14:16:46 +0200 Subject: [PATCH] Add discovery for ldconfig hook that updates the LDCache This change adds a discovered hook for updating the ldcache as a container-create hook. The mounts from a discoverer are inspected to determine the folders that must be added to the cache using the nvidia-ctk hook update-ldcache command. This is added to the "csv" discovery mode for the experimental runtime. Signed-off-by: Evan Lezar --- .../modifier/experimental.go | 17 ++- internal/discover/discover.go | 6 + internal/discover/ldconfig.go | 126 ++++++++++++++++++ internal/discover/legacy.go | 4 +- internal/discover/list.go | 9 ++ internal/discover/none.go | 2 +- 6 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 internal/discover/ldconfig.go diff --git a/cmd/nvidia-container-runtime/modifier/experimental.go b/cmd/nvidia-container-runtime/modifier/experimental.go index ead4ce19..b669de40 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental.go +++ b/cmd/nvidia-container-runtime/modifier/experimental.go @@ -59,13 +59,16 @@ func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec } logger.Infof("Constructing modifier from config: %+v", cfg) - root := cfg.NVIDIAContainerCLIConfig.Root + config := &discover.Config{ + Root: cfg.NVIDIAContainerCLIConfig.Root, + NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path, + } var d discover.Discover switch resolveAutoDiscoverMode(logger, cfg.NVIDIAContainerRuntimeConfig.DiscoverMode) { case "legacy": - legacyDiscoverer, err := discover.NewLegacyDiscoverer(logger, root) + legacyDiscoverer, err := discover.NewLegacyDiscoverer(logger, config) if err != nil { return nil, fmt.Errorf("failed to create legacy discoverer: %v", err) } @@ -81,11 +84,17 @@ func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec csvFiles = csv.BaseFilesOnly(csvFiles) } - csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, root) + csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root) if err != nil { return nil, fmt.Errorf("failed to create CSV discoverer: %v", err) } - d = csvDiscoverer + + hooks, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config) + if err != nil { + return nil, fmt.Errorf("failed to create hook discoverer: %v", err) + } + + d = discover.NewList(csvDiscoverer, hooks) default: return nil, fmt.Errorf("invalid discover mode: %v", cfg.NVIDIAContainerRuntimeConfig.DiscoverMode) } diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 77a8fdd5..64dbef85 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -16,6 +16,12 @@ package discover +// Config represents the configuration options for discovery +type Config struct { + Root string + NVIDIAContainerToolkitCLIExecutablePath string +} + // Device represents a discovered character device. type Device struct { Path string diff --git a/internal/discover/ldconfig.go b/internal/discover/ldconfig.go new file mode 100644 index 00000000..f40befb7 --- /dev/null +++ b/internal/discover/ldconfig.go @@ -0,0 +1,126 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/sirupsen/logrus" +) + +// NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified +func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) { + d := ldconfig{ + logger: logger, + mountsFrom: mounts, + lookup: lookup.NewExecutableLocator(logger, cfg.Root), + nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath, + } + + return &d, nil +} + +const ( + nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk" +) + +type ldconfig struct { + None + logger *logrus.Logger + mountsFrom Discover + lookup lookup.Locator + nvidiaCTKExecutablePath string +} + +// Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths. +func (d ldconfig) Hooks() ([]Hook, error) { + mounts, err := d.mountsFrom.Mounts() + if err != nil { + return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err) + } + + libDirs := getLibDirs(mounts) + + hookPath := nvidiaCTKDefaultFilePath + targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath) + if err != nil { + d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err) + } else if len(targets) == 0 { + d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath) + } else { + d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets) + hookPath = targets[0] + } + d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath) + + args := []string{hookPath, "hook", "update-ldcache"} + for _, f := range libDirs { + args = append(args, "--folders", f) + } + h := Hook{ + Lifecycle: cdi.CreateContainerHook, + Path: hookPath, + Args: args, + } + + return []Hook{h}, nil +} + +// getLibDirs extracts the library dirs from the specified mounts +func getLibDirs(mounts []Mount) []string { + var paths []string + checked := make(map[string]bool) + + for _, m := range mounts { + dir := filepath.Dir(m.Path) + if dir == "" { + continue + } + + _, exists := checked[dir] + if exists { + continue + } + checked[dir] = isLibName(filepath.Base(m.Path)) + + if checked[dir] { + paths = append(paths, dir) + } + } + + sort.Strings(paths) + + return paths +} + +// isLibName checks if the specified filename is a library (i.e. ends in `.so*`) +func isLibName(filename string) bool { + parts := strings.Split(filename, ".") + + for _, p := range parts { + if p == "so" { + return true + } + } + + return false +} diff --git a/internal/discover/legacy.go b/internal/discover/legacy.go index 0f7a425b..adc4c0c5 100644 --- a/internal/discover/legacy.go +++ b/internal/discover/legacy.go @@ -23,10 +23,10 @@ import ( ) // NewLegacyDiscoverer creates a discoverer for the experimental runtime -func NewLegacyDiscoverer(logger *logrus.Logger, root string) (Discover, error) { +func NewLegacyDiscoverer(logger *logrus.Logger, cfg *Config) (Discover, error) { d := legacy{ logger: logger, - lookup: lookup.NewExecutableLocator(logger, root), + lookup: lookup.NewExecutableLocator(logger, cfg.Root), } return &d, nil diff --git a/internal/discover/list.go b/internal/discover/list.go index f19c8c15..a30cf0fc 100644 --- a/internal/discover/list.go +++ b/internal/discover/list.go @@ -27,6 +27,15 @@ type list struct { var _ Discover = (*list)(nil) +// NewList creates a discoverer that is the composite of a list of discoveres. +func NewList(d ...Discover) Discover { + l := list{ + discoverers: d, + } + + return &l +} + // Devices returns all devices from the included discoverers func (d list) Devices() ([]Device, error) { var allDevices []Device diff --git a/internal/discover/none.go b/internal/discover/none.go index 989a2e16..2a1d2c57 100644 --- a/internal/discover/none.go +++ b/internal/discover/none.go @@ -32,7 +32,7 @@ func (e None) Mounts() ([]Mount, error) { return []Mount{}, nil } -// Hooks returns and empty list of hooks +// Hooks returns an empty list of hooks func (e None) Hooks() ([]Hook, error) { return []Hook{}, nil }