From 95797a82525a9e35a654a5375b42415672c07ffd Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 25 Mar 2022 14:24:10 +0200 Subject: [PATCH 1/3] Move reading of container state for internal/oci package Signed-off-by: Evan Lezar --- .../hook/update-ldcache/update-ldcache.go | 27 +------ internal/oci/state.go | 73 +++++++++++++++++++ 2 files changed, 76 insertions(+), 24 deletions(-) create mode 100644 internal/oci/state.go diff --git a/cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go b/cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go index 56f9f229..176cd777 100644 --- a/cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go +++ b/cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go @@ -17,14 +17,12 @@ package ldcache import ( - "encoding/json" "fmt" "os" "path/filepath" "syscall" "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" - "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -76,31 +74,12 @@ func (m command) build() *cli.Command { } func (m command) run(c *cli.Context, cfg *config) error { - var s specs.State - - inputReader := os.Stdin - if cfg.containerSpec != "" && cfg.containerSpec != "-" { - inputFile, err := os.Open(cfg.containerSpec) - if err != nil { - return fmt.Errorf("failed to open intput: %v", err) - } - defer inputFile.Close() - inputReader = inputFile - } - - d := json.NewDecoder(inputReader) - if err := d.Decode(&s); err != nil { - return fmt.Errorf("failed to decode container state: %v", err) - } - - specFilePath := oci.GetSpecFilePath(s.Bundle) - specFile, err := os.Open(specFilePath) + s, err := oci.LoadContainerState(cfg.containerSpec) if err != nil { - return fmt.Errorf("failed to open OCI spec file: %v", err) + return fmt.Errorf("failed to load container state: %v", err) } - defer specFile.Close() - spec, err := oci.LoadFrom(specFile) + spec, err := s.LoadSpec() if err != nil { return fmt.Errorf("failed to load OCI spec: %v", err) } diff --git a/internal/oci/state.go b/internal/oci/state.go new file mode 100644 index 00000000..d3939b68 --- /dev/null +++ b/internal/oci/state.go @@ -0,0 +1,73 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package oci + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/opencontainers/runtime-spec/specs-go" +) + +// State stores an OCI container state. This includes the spec path and the environment +type State specs.State + +// LoadContainerState loads the container state from the specified filename. If the filename is empty or '-' the state is loaded from STDIN +func LoadContainerState(filename string) (*State, error) { + if filename == "" || filename == "-" { + return ReadContainerState(os.Stdin) + } + + inputFile, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("failed to open file: %v", err) + } + defer inputFile.Close() + + return ReadContainerState(inputFile) +} + +// ReadContainerState reads the container state from the specified reader +func ReadContainerState(reader io.Reader) (*State, error) { + var s State + + d := json.NewDecoder(reader) + if err := d.Decode(&s); err != nil { + return nil, fmt.Errorf("failed to decode container state: %v", err) + } + + return &s, nil +} + +// LoadSpec loads the OCI spec associated with the container state +func (s State) LoadSpec() (*specs.Spec, error) { + specFilePath := GetSpecFilePath(s.Bundle) + specFile, err := os.Open(specFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open OCI spec file: %v", err) + } + defer specFile.Close() + + spec, err := LoadFrom(specFile) + if err != nil { + return nil, fmt.Errorf("failed to load OCI spec: %v", err) + } + + return spec, nil +} From ecb4ef495a26d2fb182962f03f66b5a1e5a86856 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 17 Mar 2022 15:16:24 +0200 Subject: [PATCH 2/3] Add create-symlinks subcommand to create symlinks in container for specified CSV files Signed-off-by: Evan Lezar --- .../hook/create-symlinks/create-symlinks.go | 209 ++++++++++++++++++ cmd/nvidia-ctk/hook/hook.go | 2 + cmd/nvidia-ctk/main.go | 1 - 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go diff --git a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go new file mode 100644 index 00000000..f8e6e835 --- /dev/null +++ b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go @@ -0,0 +1,209 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package symlinks + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" + "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +type command struct { + logger *logrus.Logger +} + +type config struct { + hostRoot string + filenames cli.StringSlice + containerSpec string +} + +// NewCommand constructs a hook command with the specified logger +func NewCommand(logger *logrus.Logger) *cli.Command { + c := command{ + logger: logger, + } + return c.build() +} + +// build +func (m command) build() *cli.Command { + cfg := config{} + + // Create the '' command + c := cli.Command{ + Name: "create-symlinks", + Usage: "A hook to create symlinks in the container. This can be used to proces CSV mount specs", + Action: func(c *cli.Context) error { + return m.run(c, &cfg) + }, + } + + c.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "host-root", + Usage: "The root on the host filesystem to use to resolve symlinks", + Destination: &cfg.hostRoot, + }, + &cli.StringSliceFlag{ + Name: "csv-filenames", + Aliases: []string{"f"}, + Usage: "Specify the (CSV) filenames to process", + Destination: &cfg.filenames, + }, + &cli.StringFlag{ + Name: "container-spec", + Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN", + Destination: &cfg.containerSpec, + }, + } + + return &c +} + +func (m command) run(c *cli.Context, cfg *config) error { + s, err := oci.LoadContainerState(cfg.containerSpec) + if err != nil { + return fmt.Errorf("failed to load container state: %v", err) + } + + spec, err := s.LoadSpec() + if err != nil { + return fmt.Errorf("failed to load OCI spec: %v", err) + } + + var containerRoot string + if spec.Root != nil { + containerRoot = spec.Root.Path + } + + csvFiles := cfg.filenames.Value() + + chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot) + + var candidates []string + for _, file := range csvFiles { + mountSpecs, err := csv.ParseFile(m.logger, file) + if err != nil { + m.logger.Debugf("Skipping CSV file %v: %v", file, err) + continue + } + + for _, ms := range mountSpecs { + if ms.Type != csv.MountSpecSym { + continue + } + targets, err := chainLocator.Locate(ms.Path) + if err != nil { + m.logger.Warnf("Failed to locate symlink %v", ms.Path) + } + candidates = append(candidates, targets...) + } + } + + created := make(map[string]bool) + // candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain. + for _, candidate := range candidates { + targets, err := m.Locate(candidate) + if err != nil { + m.logger.Debugf("Skipping invalid link: %v", err) + continue + } else if len(targets) != 1 { + m.logger.Debugf("Unexepected number of targets: %v", targets) + continue + } else if targets[0] == candidate { + m.logger.Debugf("%v is not a symlink", candidate) + continue + } + target, err := changeRoot(cfg.hostRoot, "/", targets[0]) + if err != nil { + m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, cfg.hostRoot, err) + continue + } + + linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate) + if err != nil { + m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", candidate, cfg.hostRoot, err) + continue + } + + if created[linkPath] { + m.logger.Debugf("Link %v already created", linkPath) + continue + } + m.logger.Infof("Symlinking %v to %v", linkPath, target) + err = os.MkdirAll(filepath.Dir(linkPath), 0755) + if err != nil { + m.logger.Warnf("Faild to create directory: %v", err) + continue + } + err = os.Symlink(target, linkPath) + if err != nil { + m.logger.Warnf("Failed to create symlink: %v", err) + continue + } + created[linkPath] = true + } + + return nil + +} + +func changeRoot(current string, new string, path string) (string, error) { + if !filepath.IsAbs(path) { + return path, nil + } + + relative := path + if current != "" { + r, err := filepath.Rel(current, path) + if err != nil { + return "", err + } + relative = r + } + + return filepath.Join(new, relative), nil +} + +// Locate returns the link target of the specified filename or an empty slice if the +// specified filename is not a symlink. +func (m command) Locate(filename string) ([]string, error) { + info, err := os.Lstat(filename) + if err != nil { + return nil, fmt.Errorf("failed to get file info: %v", info) + } + if info.Mode()&os.ModeSymlink == 0 { + m.logger.Debugf("%v is not a symlink", filename) + return nil, nil + } + + target, err := os.Readlink(filename) + if err != nil { + return nil, fmt.Errorf("error checking symlink: %v", err) + } + + m.logger.Debugf("Resolved link: '%v' => '%v'", filename, target) + + return []string{target}, nil +} diff --git a/cmd/nvidia-ctk/hook/hook.go b/cmd/nvidia-ctk/hook/hook.go index 4d85dcd5..feac8f5d 100644 --- a/cmd/nvidia-ctk/hook/hook.go +++ b/cmd/nvidia-ctk/hook/hook.go @@ -17,6 +17,7 @@ package hook import ( + symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks" ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -44,6 +45,7 @@ func (m hookCommand) build() *cli.Command { hook.Subcommands = []*cli.Command{ ldcache.NewCommand(m.logger), + symlinks.NewCommand(m.logger), } return &hook diff --git a/cmd/nvidia-ctk/main.go b/cmd/nvidia-ctk/main.go index 08374ed6..ca9de11b 100644 --- a/cmd/nvidia-ctk/main.go +++ b/cmd/nvidia-ctk/main.go @@ -63,7 +63,6 @@ func main() { if config.Debug { logLevel = log.DebugLevel } - logger.SetLevel(logLevel) return nil } From 907736b05322c888e5c2ce4e9736ecfe126dafc7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 17 Mar 2022 16:11:40 +0200 Subject: [PATCH 3/3] Inject symlinks hook for creating symlinks in a container Signed-off-by: Evan Lezar --- .../modifier/experimental.go | 11 ++- internal/discover/symlinks.go | 71 +++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 internal/discover/symlinks.go diff --git a/cmd/nvidia-container-runtime/modifier/experimental.go b/cmd/nvidia-container-runtime/modifier/experimental.go index b669de40..f8e927c5 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental.go +++ b/cmd/nvidia-container-runtime/modifier/experimental.go @@ -89,12 +89,17 @@ func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec return nil, fmt.Errorf("failed to create CSV discoverer: %v", err) } - hooks, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config) + ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config) if err != nil { - return nil, fmt.Errorf("failed to create hook discoverer: %v", err) + return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err) } - d = discover.NewList(csvDiscoverer, hooks) + createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, config) + if err != nil { + return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err) + } + + d = discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook) default: return nil, fmt.Errorf("invalid discover mode: %v", cfg.NVIDIAContainerRuntimeConfig.DiscoverMode) } diff --git a/internal/discover/symlinks.go b/internal/discover/symlinks.go new file mode 100644 index 00000000..9547f8ba --- /dev/null +++ b/internal/discover/symlinks.go @@ -0,0 +1,71 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/sirupsen/logrus" +) + +type symlinks struct { + None + logger *logrus.Logger + lookup lookup.Locator + nvidiaCTKExecutablePath string + csvFiles []string +} + +// NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container +func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, cfg *Config) (Discover, error) { + d := symlinks{ + logger: logger, + lookup: lookup.NewExecutableLocator(logger, cfg.Root), + nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath, + csvFiles: csvFiles, + } + + return &d, nil +} + +// Hooks returns a hook to create the symlinks from the required CSV files +func (d symlinks) Hooks() ([]Hook, error) { + hookPath := nvidiaCTKDefaultFilePath + targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath) + if err != nil { + d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err) + } else if len(targets) == 0 { + d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath) + } else { + d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets) + hookPath = targets[0] + } + d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath) + + args := []string{hookPath, "hook", "create-symlinks"} + for _, f := range d.csvFiles { + args = append(args, "--csv-filenames", f) + } + + h := Hook{ + Lifecycle: cdi.CreateContainerHook, + Path: hookPath, + Args: args, + } + + return []Hook{h}, nil +}