From 52bb9e186b0c9d2f49bd848445f58f411494371f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 20 Jul 2022 10:52:43 +0200 Subject: [PATCH] Add vulkan support through OCI spec modification This change allows the NVIDIA Container Runtime to inject vulkan loaders and libraries by modifying the OCI runtime specification. This allows vulkan applications to run in containers without additional modifications. Signed-off-by: Evan Lezar --- .../runtime_factory.go | 6 ++ internal/discover/graphics.go | 63 +++++++++++++++++ internal/modifier/graphics.go | 67 +++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 internal/discover/graphics.go create mode 100644 internal/modifier/graphics.go diff --git a/cmd/nvidia-container-runtime/runtime_factory.go b/cmd/nvidia-container-runtime/runtime_factory.go index ab7c037f..1754e525 100644 --- a/cmd/nvidia-container-runtime/runtime_factory.go +++ b/cmd/nvidia-container-runtime/runtime_factory.go @@ -67,6 +67,11 @@ func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec return nil, err } + graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, ociSpec) + if err != nil { + return nil, err + } + gdsModifier, err := modifier.NewGDSModifier(logger, cfg, ociSpec) if err != nil { return nil, err @@ -84,6 +89,7 @@ func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec modifiers := modifier.Merge( modeModifier, + graphicsModifier, gdsModifier, mofedModifier, tegraModifier, diff --git a/internal/discover/graphics.go b/internal/discover/graphics.go new file mode 100644 index 00000000..47d95a0d --- /dev/null +++ b/internal/discover/graphics.go @@ -0,0 +1,63 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "fmt" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" + "github.com/sirupsen/logrus" +) + +// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan. +func NewGraphicsDiscoverer(logger *logrus.Logger, root string) (Discover, error) { + locator, err := lookup.NewLibraryLocator(logger, root) + if err != nil { + return nil, fmt.Errorf("failed to construct library locator: %v", err) + } + libraries := NewMounts( + logger, + locator, + root, + []string{ + "libnvidia-egl-gbm.so", + }, + ) + + jsonMounts := NewMounts( + logger, + lookup.NewFileLocator(logger, root), + root, + []string{ + // TODO: We should handle this more cleanly + "/etc/glvnd/egl_vendor.d/10_nvidia.json", + "/etc/vulkan/icd.d/nvidia_icd.json", + "/etc/vulkan/implicit_layer.d/nvidia_layers.json", + "/usr/share/glvnd/egl_vendor.d/10_nvidia.json", + "/usr/share/vulkan/icd.d/nvidia_icd.json", + "/usr/share/vulkan/implicit_layer.d/nvidia_layers.json", + "/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json", + }, + ) + + discover := Merge( + libraries, + jsonMounts, + ) + + return discover, nil +} diff --git a/internal/modifier/graphics.go b/internal/modifier/graphics.go new file mode 100644 index 00000000..722c2ae0 --- /dev/null +++ b/internal/modifier/graphics.go @@ -0,0 +1,67 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package modifier + +import ( + "fmt" + "strings" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/config" + "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" + "github.com/sirupsen/logrus" +) + +// NewGraphicsModifier constructs a modifier that injects graphics-related modifications into an OCI runtime specification. +// The value of the NVIDIA_DRIVER_CAPABILITIES environment variable is checked to determine if this modification should be made. +func NewGraphicsModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) { + rawSpec, err := ociSpec.Load() + if err != nil { + return nil, fmt.Errorf("failed to load OCI spec: %v", err) + } + + image, err := image.NewCUDAImageFromSpec(rawSpec) + if err != nil { + return nil, err + } + + if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 { + logger.Infof("No modification required; no devices requested") + return nil, nil + } + + var hasGraphics bool + for _, c := range strings.Split(image["NVIDIA_DRIVER_CAPABILITIES"], ",") { + if c == "graphics" || c == "all" { + hasGraphics = true + break + } + } + + if !hasGraphics { + logger.Debugf("Capability %q not selected", "graphics") + return nil, nil + } + + d, err := discover.NewGraphicsDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root) + if err != nil { + return nil, fmt.Errorf("failed to construct discoverer: %v", err) + } + + return NewModifierFromDiscoverer(logger, d) +}