diff --git a/cmd/nvidia-container-runtime/runtime_factory.go b/cmd/nvidia-container-runtime/runtime_factory.go index ab7c037f..1754e525 100644 --- a/cmd/nvidia-container-runtime/runtime_factory.go +++ b/cmd/nvidia-container-runtime/runtime_factory.go @@ -67,6 +67,11 @@ func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec return nil, err } + graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, ociSpec) + if err != nil { + return nil, err + } + gdsModifier, err := modifier.NewGDSModifier(logger, cfg, ociSpec) if err != nil { return nil, err @@ -84,6 +89,7 @@ func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec modifiers := modifier.Merge( modeModifier, + graphicsModifier, gdsModifier, mofedModifier, tegraModifier, diff --git a/internal/discover/graphics.go b/internal/discover/graphics.go new file mode 100644 index 00000000..47d95a0d --- /dev/null +++ b/internal/discover/graphics.go @@ -0,0 +1,63 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "fmt" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" + "github.com/sirupsen/logrus" +) + +// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan. +func NewGraphicsDiscoverer(logger *logrus.Logger, root string) (Discover, error) { + locator, err := lookup.NewLibraryLocator(logger, root) + if err != nil { + return nil, fmt.Errorf("failed to construct library locator: %v", err) + } + libraries := NewMounts( + logger, + locator, + root, + []string{ + "libnvidia-egl-gbm.so", + }, + ) + + jsonMounts := NewMounts( + logger, + lookup.NewFileLocator(logger, root), + root, + []string{ + // TODO: We should handle this more cleanly + "/etc/glvnd/egl_vendor.d/10_nvidia.json", + "/etc/vulkan/icd.d/nvidia_icd.json", + "/etc/vulkan/implicit_layer.d/nvidia_layers.json", + "/usr/share/glvnd/egl_vendor.d/10_nvidia.json", + "/usr/share/vulkan/icd.d/nvidia_icd.json", + "/usr/share/vulkan/implicit_layer.d/nvidia_layers.json", + "/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json", + }, + ) + + discover := Merge( + libraries, + jsonMounts, + ) + + return discover, nil +} diff --git a/internal/ldcache/ldcache.go b/internal/ldcache/ldcache.go new file mode 100644 index 00000000..c26c47ca --- /dev/null +++ b/internal/ldcache/ldcache.go @@ -0,0 +1,292 @@ +/* +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/ + +// Adapted from https://github.com/rai-project/ldcache + +package ldcache + +import ( + "bytes" + "encoding/binary" + "errors" + "os" + "path/filepath" + "syscall" + "unsafe" + + log "github.com/sirupsen/logrus" +) + +const ldcachePath = "/etc/ld.so.cache" + +const ( + magicString1 = "ld.so-1.7.0" + magicString2 = "glibc-ld.so.cache" + magicVersion = "1.1" +) + +const ( + flagTypeMask = 0x00ff + flagTypeELF = 0x0001 + + flagArchMask = 0xff00 + flagArchI386 = 0x0000 + flagArchX8664 = 0x0300 + flagArchX32 = 0x0800 + flagArchPpc64le = 0x0500 +) + +var errInvalidCache = errors.New("invalid ld.so.cache file") + +type header1 struct { + Magic [len(magicString1) + 1]byte // include null delimiter + NLibs uint32 +} + +type entry1 struct { + Flags int32 + Key, Value uint32 +} + +type header2 struct { + Magic [len(magicString2)]byte + Version [len(magicVersion)]byte + NLibs uint32 + TableSize uint32 + _ [3]uint32 // unused + _ uint64 // force 8 byte alignment +} + +type entry2 struct { + Flags int32 + Key, Value uint32 + OSVersion uint32 + HWCap uint64 +} + +// LDCache represents the interface for performing lookups into the LDCache +type LDCache interface { + List() ([]string, []string) + Lookup(...string) ([]string, []string) +} + +type ldcache struct { + *bytes.Reader + + data, libs []byte + header header2 + entries []entry2 + + root string + logger *log.Logger +} + +// New creates a new LDCache with the specified logger and root. +func New(logger *log.Logger, root string) (LDCache, error) { + path := filepath.Join(root, ldcachePath) + + logger.Debugf("Opening ld.conf at %v", path) + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return nil, err + } + d, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), + syscall.PROT_READ, syscall.MAP_PRIVATE) + if err != nil { + return nil, err + } + + cache := &ldcache{ + data: d, + Reader: bytes.NewReader(d), + root: root, + logger: logger, + } + return cache, cache.parse() +} + +func (c *ldcache) Close() error { + return syscall.Munmap(c.data) +} + +func (c *ldcache) Magic() string { + return string(c.header.Magic[:]) +} + +func (c *ldcache) Version() string { + return string(c.header.Version[:]) +} + +func strn(b []byte, n int) string { + return string(b[:n]) +} + +func (c *ldcache) parse() error { + var header header1 + + // Check for the old format (< glibc-2.2) + if c.Len() <= int(unsafe.Sizeof(header)) { + return errInvalidCache + } + if strn(c.data, len(magicString1)) == magicString1 { + if err := binary.Read(c, binary.LittleEndian, &header); err != nil { + return err + } + n := int64(header.NLibs) * int64(unsafe.Sizeof(entry1{})) + offset, err := c.Seek(n, 1) // skip old entries + if err != nil { + return err + } + n = (-offset) & int64(unsafe.Alignof(c.header)-1) + _, err = c.Seek(n, 1) // skip padding + if err != nil { + return err + } + } + + c.libs = c.data[c.Size()-int64(c.Len()):] // kv offsets start here + if err := binary.Read(c, binary.LittleEndian, &c.header); err != nil { + return err + } + if c.Magic() != magicString2 || c.Version() != magicVersion { + return errInvalidCache + } + c.entries = make([]entry2, c.header.NLibs) + if err := binary.Read(c, binary.LittleEndian, &c.entries); err != nil { + return err + } + return nil +} + +// List creates a list of libraires in the ldcache. +// The 32-bit and 64-bit libraries are returned separately. +func (c *ldcache) List() ([]string, []string) { + paths := make(map[int][]string) + + processed := make(map[string]bool) + + for _, e := range c.entries { + bits := 0 + if ((e.Flags & flagTypeMask) & flagTypeELF) == 0 { + continue + } + switch e.Flags & flagArchMask { + case flagArchX8664: + fallthrough + case flagArchPpc64le: + bits = 64 + case flagArchX32: + fallthrough + case flagArchI386: + bits = 32 + default: + continue + } + if e.Key > uint32(len(c.libs)) || e.Value > uint32(len(c.libs)) { + continue + } + value := c.libs[e.Value:] + + n := bytes.IndexByte(value, 0) + if n < 0 { + break + } + + name := filepath.Join(c.root, strn(value, n)) + c.logger.Debugf("checking %v", string(name)) + + path, err := filepath.EvalSymlinks(name) + if err != nil { + c.logger.Debugf("could not resolve symlink for %v", name) + break + } + if processed[path] { + continue + } + paths[bits] = append(paths[bits], path) + processed[path] = true + } + + return paths[32], paths[64] +} + +// Lookup searches the ldcache for the specified prefixes. +// The 32-bit and 64-bit libraries matching the prefixes are returned. +func (c *ldcache) Lookup(libs ...string) (paths32, paths64 []string) { + c.logger.Debugf("Looking up %v in cache", libs) + type void struct{} + var paths *[]string + + set := make(map[string]void) + prefix := make([][]byte, len(libs)) + + for i := range libs { + prefix[i] = []byte(libs[i]) + } + for _, e := range c.entries { + if ((e.Flags & flagTypeMask) & flagTypeELF) == 0 { + continue + } + switch e.Flags & flagArchMask { + case flagArchX8664: + fallthrough + case flagArchPpc64le: + paths = &paths64 + case flagArchX32: + fallthrough + case flagArchI386: + paths = &paths32 + default: + continue + } + if e.Key > uint32(len(c.libs)) || e.Value > uint32(len(c.libs)) { + continue + } + lib := c.libs[e.Key:] + value := c.libs[e.Value:] + + for _, p := range prefix { + if bytes.HasPrefix(lib, p) { + n := bytes.IndexByte(value, 0) + if n < 0 { + break + } + + name := filepath.Join(c.root, strn(value, n)) + c.logger.Debugf("checking %v", string(name)) + + path, err := filepath.EvalSymlinks(name) + if err != nil { + c.logger.Debugf("could not resolve symlink for %v", name) + break + } + if _, ok := set[path]; ok { + break + } + set[path] = void{} + *paths = append(*paths, path) + break + } + } + } + return +} diff --git a/internal/lookup/library.go b/internal/lookup/library.go new file mode 100644 index 00000000..00fa3137 --- /dev/null +++ b/internal/lookup/library.go @@ -0,0 +1,68 @@ +/* +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/ + +package lookup + +import ( + "fmt" + "strings" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache" + log "github.com/sirupsen/logrus" +) + +type library struct { + logger *log.Logger + symlink Locator + cache ldcache.LDCache +} + +var _ Locator = (*library)(nil) + +// NewLibraryLocator creates a library locator using the specified logger. +func NewLibraryLocator(logger *log.Logger, root string) (Locator, error) { + cache, err := ldcache.New(logger, root) + if err != nil { + return nil, fmt.Errorf("error loading ldcache: %v", err) + } + + l := library{ + symlink: NewSymlinkLocator(logger, root), + cache: cache, + } + + return &l, nil +} + +// Locate finds the specified libraryname. +// If the input is a library name, the ldcache is searched otherwise the +// provided path is resolved as a symlink. +func (l library) Locate(libname string) ([]string, error) { + if strings.Contains(libname, "/") { + return l.symlink.Locate(libname) + } + + paths32, paths64 := l.cache.Lookup(libname) + if len(paths32) > 0 { + l.logger.Warnf("Ignoring 32-bit libraries for %v: %v", libname, paths32) + } + + if len(paths64) == 0 { + return nil, fmt.Errorf("64-bit library %v not found", libname) + } + + return paths64, nil +} diff --git a/internal/modifier/graphics.go b/internal/modifier/graphics.go new file mode 100644 index 00000000..722c2ae0 --- /dev/null +++ b/internal/modifier/graphics.go @@ -0,0 +1,67 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package modifier + +import ( + "fmt" + "strings" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/config" + "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" + "github.com/sirupsen/logrus" +) + +// NewGraphicsModifier constructs a modifier that injects graphics-related modifications into an OCI runtime specification. +// The value of the NVIDIA_DRIVER_CAPABILITIES environment variable is checked to determine if this modification should be made. +func NewGraphicsModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) { + rawSpec, err := ociSpec.Load() + if err != nil { + return nil, fmt.Errorf("failed to load OCI spec: %v", err) + } + + image, err := image.NewCUDAImageFromSpec(rawSpec) + if err != nil { + return nil, err + } + + if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 { + logger.Infof("No modification required; no devices requested") + return nil, nil + } + + var hasGraphics bool + for _, c := range strings.Split(image["NVIDIA_DRIVER_CAPABILITIES"], ",") { + if c == "graphics" || c == "all" { + hasGraphics = true + break + } + } + + if !hasGraphics { + logger.Debugf("Capability %q not selected", "graphics") + return nil, nil + } + + d, err := discover.NewGraphicsDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root) + if err != nil { + return nil, fmt.Errorf("failed to construct discoverer: %v", err) + } + + return NewModifierFromDiscoverer(logger, d) +}