Merge branch 'opengl-poc' into 'main'

Add support for injecting vulkan configs and libraries

See merge request nvidia/container-toolkit/container-toolkit!196

cmd/nvidia-container-runtime/runtime_factory.go

@@ -67,6 +67,11 @@ func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec
 		return nil, err
 	}
 
+	graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
 	gdsModifier, err := modifier.NewGDSModifier(logger, cfg, ociSpec)
 	if err != nil {
 		return nil, err
@@ -84,6 +89,7 @@ func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec
 
 	modifiers := modifier.Merge(
 		modeModifier,
+		graphicsModifier,
 		gdsModifier,
 		mofedModifier,
 		tegraModifier,

internal/discover/graphics.go

@@ -0,0 +1,63 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
+func NewGraphicsDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct library locator: %v", err)
+	}
+	libraries := NewMounts(
+		logger,
+		locator,
+		root,
+		[]string{
+			"libnvidia-egl-gbm.so",
+		},
+	)
+
+	jsonMounts := NewMounts(
+		logger,
+		lookup.NewFileLocator(logger, root),
+		root,
+		[]string{
+			// TODO: We should handle this more cleanly
+			"/etc/glvnd/egl_vendor.d/10_nvidia.json",
+			"/etc/vulkan/icd.d/nvidia_icd.json",
+			"/etc/vulkan/implicit_layer.d/nvidia_layers.json",
+			"/usr/share/glvnd/egl_vendor.d/10_nvidia.json",
+			"/usr/share/vulkan/icd.d/nvidia_icd.json",
+			"/usr/share/vulkan/implicit_layer.d/nvidia_layers.json",
+			"/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json",
+		},
+	)
+
+	discover := Merge(
+		libraries,
+		jsonMounts,
+	)
+
+	return discover, nil
+}

internal/ldcache/ldcache.go

@@ -0,0 +1,292 @@
+/*
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+// Adapted from https://github.com/rai-project/ldcache
+
+package ldcache
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const ldcachePath = "/etc/ld.so.cache"
+
+const (
+	magicString1 = "ld.so-1.7.0"
+	magicString2 = "glibc-ld.so.cache"
+	magicVersion = "1.1"
+)
+
+const (
+	flagTypeMask = 0x00ff
+	flagTypeELF  = 0x0001
+
+	flagArchMask    = 0xff00
+	flagArchI386    = 0x0000
+	flagArchX8664   = 0x0300
+	flagArchX32     = 0x0800
+	flagArchPpc64le = 0x0500
+)
+
+var errInvalidCache = errors.New("invalid ld.so.cache file")
+
+type header1 struct {
+	Magic [len(magicString1) + 1]byte // include null delimiter
+	NLibs uint32
+}
+
+type entry1 struct {
+	Flags      int32
+	Key, Value uint32
+}
+
+type header2 struct {
+	Magic     [len(magicString2)]byte
+	Version   [len(magicVersion)]byte
+	NLibs     uint32
+	TableSize uint32
+	_         [3]uint32 // unused
+	_         uint64    // force 8 byte alignment
+}
+
+type entry2 struct {
+	Flags      int32
+	Key, Value uint32
+	OSVersion  uint32
+	HWCap      uint64
+}
+
+// LDCache represents the interface for performing lookups into the LDCache
+type LDCache interface {
+	List() ([]string, []string)
+	Lookup(...string) ([]string, []string)
+}
+
+type ldcache struct {
+	*bytes.Reader
+
+	data, libs []byte
+	header     header2
+	entries    []entry2
+
+	root   string
+	logger *log.Logger
+}
+
+// New creates a new LDCache with the specified logger and root.
+func New(logger *log.Logger, root string) (LDCache, error) {
+	path := filepath.Join(root, ldcachePath)
+
+	logger.Debugf("Opening ld.conf at %v", path)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	d, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()),
+		syscall.PROT_READ, syscall.MAP_PRIVATE)
+	if err != nil {
+		return nil, err
+	}
+
+	cache := &ldcache{
+		data:   d,
+		Reader: bytes.NewReader(d),
+		root:   root,
+		logger: logger,
+	}
+	return cache, cache.parse()
+}
+
+func (c *ldcache) Close() error {
+	return syscall.Munmap(c.data)
+}
+
+func (c *ldcache) Magic() string {
+	return string(c.header.Magic[:])
+}
+
+func (c *ldcache) Version() string {
+	return string(c.header.Version[:])
+}
+
+func strn(b []byte, n int) string {
+	return string(b[:n])
+}
+
+func (c *ldcache) parse() error {
+	var header header1
+
+	// Check for the old format (< glibc-2.2)
+	if c.Len() <= int(unsafe.Sizeof(header)) {
+		return errInvalidCache
+	}
+	if strn(c.data, len(magicString1)) == magicString1 {
+		if err := binary.Read(c, binary.LittleEndian, &header); err != nil {
+			return err
+		}
+		n := int64(header.NLibs) * int64(unsafe.Sizeof(entry1{}))
+		offset, err := c.Seek(n, 1) // skip old entries
+		if err != nil {
+			return err
+		}
+		n = (-offset) & int64(unsafe.Alignof(c.header)-1)
+		_, err = c.Seek(n, 1) // skip padding
+		if err != nil {
+			return err
+		}
+	}
+
+	c.libs = c.data[c.Size()-int64(c.Len()):] // kv offsets start here
+	if err := binary.Read(c, binary.LittleEndian, &c.header); err != nil {
+		return err
+	}
+	if c.Magic() != magicString2 || c.Version() != magicVersion {
+		return errInvalidCache
+	}
+	c.entries = make([]entry2, c.header.NLibs)
+	if err := binary.Read(c, binary.LittleEndian, &c.entries); err != nil {
+		return err
+	}
+	return nil
+}
+
+// List creates a list of libraires in the ldcache.
+// The 32-bit and 64-bit libraries are returned separately.
+func (c *ldcache) List() ([]string, []string) {
+	paths := make(map[int][]string)
+
+	processed := make(map[string]bool)
+
+	for _, e := range c.entries {
+		bits := 0
+		if ((e.Flags & flagTypeMask) & flagTypeELF) == 0 {
+			continue
+		}
+		switch e.Flags & flagArchMask {
+		case flagArchX8664:
+			fallthrough
+		case flagArchPpc64le:
+			bits = 64
+		case flagArchX32:
+			fallthrough
+		case flagArchI386:
+			bits = 32
+		default:
+			continue
+		}
+		if e.Key > uint32(len(c.libs)) || e.Value > uint32(len(c.libs)) {
+			continue
+		}
+		value := c.libs[e.Value:]
+
+		n := bytes.IndexByte(value, 0)
+		if n < 0 {
+			break
+		}
+
+		name := filepath.Join(c.root, strn(value, n))
+		c.logger.Debugf("checking %v", string(name))
+
+		path, err := filepath.EvalSymlinks(name)
+		if err != nil {
+			c.logger.Debugf("could not resolve symlink for %v", name)
+			break
+		}
+		if processed[path] {
+			continue
+		}
+		paths[bits] = append(paths[bits], path)
+		processed[path] = true
+	}
+
+	return paths[32], paths[64]
+}
+
+// Lookup searches the ldcache for the specified prefixes.
+// The 32-bit and 64-bit libraries matching the prefixes are returned.
+func (c *ldcache) Lookup(libs ...string) (paths32, paths64 []string) {
+	c.logger.Debugf("Looking up %v in cache", libs)
+	type void struct{}
+	var paths *[]string
+
+	set := make(map[string]void)
+	prefix := make([][]byte, len(libs))
+
+	for i := range libs {
+		prefix[i] = []byte(libs[i])
+	}
+	for _, e := range c.entries {
+		if ((e.Flags & flagTypeMask) & flagTypeELF) == 0 {
+			continue
+		}
+		switch e.Flags & flagArchMask {
+		case flagArchX8664:
+			fallthrough
+		case flagArchPpc64le:
+			paths = &paths64
+		case flagArchX32:
+			fallthrough
+		case flagArchI386:
+			paths = &paths32
+		default:
+			continue
+		}
+		if e.Key > uint32(len(c.libs)) || e.Value > uint32(len(c.libs)) {
+			continue
+		}
+		lib := c.libs[e.Key:]
+		value := c.libs[e.Value:]
+
+		for _, p := range prefix {
+			if bytes.HasPrefix(lib, p) {
+				n := bytes.IndexByte(value, 0)
+				if n < 0 {
+					break
+				}
+
+				name := filepath.Join(c.root, strn(value, n))
+				c.logger.Debugf("checking %v", string(name))
+
+				path, err := filepath.EvalSymlinks(name)
+				if err != nil {
+					c.logger.Debugf("could not resolve symlink for %v", name)
+					break
+				}
+				if _, ok := set[path]; ok {
+					break
+				}
+				set[path] = void{}
+				*paths = append(*paths, path)
+				break
+			}
+		}
+	}
+	return
+}

internal/lookup/library.go

@@ -0,0 +1,68 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package lookup
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	log "github.com/sirupsen/logrus"
+)
+
+type library struct {
+	logger  *log.Logger
+	symlink Locator
+	cache   ldcache.LDCache
+}
+
+var _ Locator = (*library)(nil)
+
+// NewLibraryLocator creates a library locator using the specified logger.
+func NewLibraryLocator(logger *log.Logger, root string) (Locator, error) {
+	cache, err := ldcache.New(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("error loading ldcache: %v", err)
+	}
+
+	l := library{
+		symlink: NewSymlinkLocator(logger, root),
+		cache:   cache,
+	}
+
+	return &l, nil
+}
+
+// Locate finds the specified libraryname.
+// If the input is a library name, the ldcache is searched otherwise the
+// provided path is resolved as a symlink.
+func (l library) Locate(libname string) ([]string, error) {
+	if strings.Contains(libname, "/") {
+		return l.symlink.Locate(libname)
+	}
+
+	paths32, paths64 := l.cache.Lookup(libname)
+	if len(paths32) > 0 {
+		l.logger.Warnf("Ignoring 32-bit libraries for %v: %v", libname, paths32)
+	}
+
+	if len(paths64) == 0 {
+		return nil, fmt.Errorf("64-bit library %v not found", libname)
+	}
+
+	return paths64, nil
+}

internal/modifier/graphics.go

@@ -0,0 +1,67 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+)
+
+// NewGraphicsModifier constructs a modifier that injects graphics-related modifications into an OCI runtime specification.
+// The value of the NVIDIA_DRIVER_CAPABILITIES environment variable is checked to determine if this modification should be made.
+func NewGraphicsModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	rawSpec, err := ociSpec.Load()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	image, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+		logger.Infof("No modification required; no devices requested")
+		return nil, nil
+	}
+
+	var hasGraphics bool
+	for _, c := range strings.Split(image["NVIDIA_DRIVER_CAPABILITIES"], ",") {
+		if c == "graphics" || c == "all" {
+			hasGraphics = true
+			break
+		}
+	}
+
+	if !hasGraphics {
+		logger.Debugf("Capability %q not selected", "graphics")
+		return nil, nil
+	}
+
+	d, err := discover.NewGraphicsDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct discoverer: %v", err)
+	}
+
+	return NewModifierFromDiscoverer(logger, d)
+}