From 126c004ee05bc6453eece95e06c3c1e11fd69126 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 11 May 2022 15:17:15 +0200 Subject: [PATCH 1/4] Improve symlink creation loop Signed-off-by: Evan Lezar --- .../hook/create-symlinks/create-symlinks.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go index 9b0119df..55904e65 100644 --- a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go +++ b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go @@ -129,6 +129,17 @@ func (m command) run(c *cli.Context, cfg *config) error { m.logger.Debugf("%v is not a symlink", candidate) continue } + linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate) + if err != nil { + m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", candidate, cfg.hostRoot, err) + continue + } + + if created[linkPath] { + m.logger.Debugf("Link %v already created", linkPath) + continue + } + target, err := changeRoot(cfg.hostRoot, "/", targets[0]) if err != nil { m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, cfg.hostRoot, err) From b0f7a3809f04dcf5630068e19faf103db0ea79f4 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 11 May 2022 15:28:02 +0200 Subject: [PATCH 2/4] Factor linkCreation into method Signed-off-by: Evan Lezar --- .../hook/create-symlinks/create-symlinks.go | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go index 55904e65..23276cae 100644 --- a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go +++ b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go @@ -129,22 +129,12 @@ func (m command) run(c *cli.Context, cfg *config) error { m.logger.Debugf("%v is not a symlink", candidate) continue } - linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate) - if err != nil { - m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", candidate, cfg.hostRoot, err) - continue - } - if created[linkPath] { - m.logger.Debugf("Link %v already created", linkPath) - continue - } - - target, err := changeRoot(cfg.hostRoot, "/", targets[0]) + err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate) if err != nil { - m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, cfg.hostRoot, err) - continue + m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err) } + } linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate) if err != nil { @@ -174,6 +164,34 @@ func (m command) run(c *cli.Context, cfg *config) error { } +func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error { + linkPath, err := changeRoot(hostRoot, containerRoot, link) + if err != nil { + m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err) + } + if created[linkPath] { + m.logger.Debugf("Link %v already created", linkPath) + return nil + } + + targetPath, err := changeRoot(hostRoot, "/", target) + if err != nil { + m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err) + } + + m.logger.Infof("Symlinking %v to %v", linkPath, targetPath) + err = os.MkdirAll(filepath.Dir(linkPath), 0755) + if err != nil { + return fmt.Errorf("failed to create directory: %v", err) + } + err = os.Symlink(target, linkPath) + if err != nil { + return fmt.Errorf("failed to create symlink: %v", err) + } + + return nil +} + func changeRoot(current string, new string, path string) (string, error) { if !filepath.IsAbs(path) { return path, nil From eac326c5eaf18317e5e86a6fdf15ede723c189b1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 11 May 2022 15:28:53 +0200 Subject: [PATCH 3/4] Add --link option to nvidia-ctk hook create-symlinks command Signed-off-by: Evan Lezar --- .../hook/create-symlinks/create-symlinks.go | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go index 23276cae..9eed0976 100644 --- a/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go +++ b/cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" @@ -35,6 +36,7 @@ type command struct { type config struct { hostRoot string filenames cli.StringSlice + links cli.StringSlice containerSpec string } @@ -70,6 +72,11 @@ func (m command) build() *cli.Command { Usage: "Specify a (CSV) filename to process", Destination: &cfg.filenames, }, + &cli.StringSliceFlag{ + Name: "link", + Usage: "Specify a specific link to create. The link is specified as source:target", + Destination: &cfg.links, + }, &cli.StringFlag{ Name: "container-spec", Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN", @@ -136,28 +143,18 @@ func (m command) run(c *cli.Context, cfg *config) error { } } - linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate) - if err != nil { - m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", candidate, cfg.hostRoot, err) + links := cfg.links.Value() + for _, l := range links { + parts := strings.Split(l, ":") + if len(parts) != 2 { + m.logger.Warnf("Invalid link specification %v", l) continue } - if created[linkPath] { - m.logger.Debugf("Link %v already created", linkPath) - continue - } - m.logger.Infof("Symlinking %v to %v", linkPath, target) - err = os.MkdirAll(filepath.Dir(linkPath), 0755) + err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1]) if err != nil { - m.logger.Warnf("Faild to create directory: %v", err) - continue + m.logger.Warnf("Failed to create link %v: %v", parts, err) } - err = os.Symlink(target, linkPath) - if err != nil { - m.logger.Warnf("Failed to create symlink: %v", err) - continue - } - created[linkPath] = true } return nil From 675fbace0199f16692e9a96ed366994d2a88d5f6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 11 May 2022 16:03:13 +0200 Subject: [PATCH 4/4] Add hook to create specific links This change updates the create-symlinks hook to also create symlinks for libcuda.so, libGLX_indirect.so.0, and libnvidia-opticalflow.so Signed-off-by: Evan Lezar --- .../modifier/experimental.go | 2 +- internal/discover/symlinks.go | 56 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/cmd/nvidia-container-runtime/modifier/experimental.go b/cmd/nvidia-container-runtime/modifier/experimental.go index 0bf73bd8..baa57252 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental.go +++ b/cmd/nvidia-container-runtime/modifier/experimental.go @@ -109,7 +109,7 @@ func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err) } - createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, config) + createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config) if err != nil { return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err) } diff --git a/internal/discover/symlinks.go b/internal/discover/symlinks.go index 0753e7c6..6430fc15 100644 --- a/internal/discover/symlinks.go +++ b/internal/discover/symlinks.go @@ -17,6 +17,10 @@ package discover import ( + "fmt" + "path/filepath" + "strings" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/sirupsen/logrus" @@ -28,15 +32,17 @@ type symlinks struct { lookup lookup.Locator nvidiaCTKExecutablePath string csvFiles []string + mountsFrom Discover } // NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container -func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, cfg *Config) (Discover, error) { +func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) { d := symlinks{ logger: logger, lookup: lookup.NewExecutableLocator(logger, cfg.Root), nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath, csvFiles: csvFiles, + mountsFrom: mounts, } return &d, nil @@ -61,6 +67,12 @@ func (d symlinks) Hooks() ([]Hook, error) { args = append(args, "--csv-filename", f) } + links, err := d.getSpecificLinkArgs() + if err != nil { + return nil, fmt.Errorf("failed to determine specific links: %v", err) + } + args = append(args, links...) + h := Hook{ Lifecycle: cdi.CreateContainerHook, Path: hookPath, @@ -69,3 +81,45 @@ func (d symlinks) Hooks() ([]Hook, error) { return []Hook{h}, nil } + +// getSpecificLinkArgs returns the required specic links that need to be created +func (d symlinks) getSpecificLinkArgs() ([]string, error) { + mounts, err := d.mountsFrom.Mounts() + if err != nil { + return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err) + } + + linkProcessed := make(map[string]bool) + var links []string + for _, m := range mounts { + var target string + var link string + + lib := filepath.Base(m.Path) + + if strings.HasPrefix(lib, "libcuda.so") { + // XXX Many applications wrongly assume that libcuda.so exists (e.g. with dlopen). + target = "libcuda.so.1" + link = "libcuda.so" + } else if strings.HasPrefix(lib, "libGLX_nvidia.so") { + // XXX GLVND requires this symlink for indirect GLX support. + target = lib + link = "libGLX_indirect.so.0" + } else if strings.HasPrefix(lib, "libnvidia-opticalflow.so") { + // XXX Fix missing symlink for libnvidia-opticalflow.so. + target = "libnvidia-opticalflow.so.1" + link = "libnvidia-opticalflow.so" + } else { + continue + } + if linkProcessed[link] { + continue + } + + linkPath := filepath.Join(filepath.Dir(m.Path), link) + links = append(links, "--link", fmt.Sprintf("%v:%v", target, linkPath)) + linkProcessed[link] = true + } + + return links, nil +}