Implement experimental modifier for NVIDIA Container Runtime

This change enables the experimental mode of the NVIDIA Container Runtime. If
enabled, the nvidia-container-runtime.discover-mode config option is
queried to determine how required OCI spec modifications should be defined.
If "legacy" is selected, the existing NVIDIA Container Runtime hooks is
discovered and injected into the OCI spec.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

internal/config/runtime.go

@@ -38,6 +38,7 @@ var (
 type RuntimeConfig struct {
 	DebugFilePath string
 	Experimental  bool
+	DiscoverMode  string
 }
 
 // GetRuntimeConfig sets up the config struct. Values are read from a toml file
@@ -74,6 +75,7 @@ func getRuntimeConfigFrom(reader io.Reader) (*RuntimeConfig, error) {
 
 	cfg.DebugFilePath = toml.GetDefault("nvidia-container-runtime.debug", cfg.DebugFilePath).(string)
 	cfg.Experimental = toml.GetDefault("nvidia-container-runtime.experimental", cfg.Experimental).(bool)
+	cfg.DiscoverMode = toml.GetDefault("nvidia-container-runtime.discover-mode", cfg.DiscoverMode).(string)
 
 	return cfg, nil
 }
@@ -83,6 +85,7 @@ func getDefaultRuntimeConfig() *RuntimeConfig {
 	c := RuntimeConfig{
 		DebugFilePath: "/dev/null",
 		Experimental:  false,
+		DiscoverMode:  "legacy",
 	}
 
 	return &c

internal/config/runtime_test.go

@@ -58,15 +58,21 @@ func TestGerRuntimeConfig(t *testing.T) {
 			description: "empty config is default",
 			expectedConfig: &RuntimeConfig{
 				DebugFilePath: "/dev/null",
+				Experimental:  false,
+				DiscoverMode:  "legacy",
 			},
 		},
 		{
 			description: "config options set inline",
 			contents: []string{
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
+				"nvidia-container-runtime.experimental = true",
+				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 			},
 			expectedConfig: &RuntimeConfig{
 				DebugFilePath: "/foo/bar",
+				Experimental:  true,
+				DiscoverMode:  "not-legacy",
 			},
 		},
 		{
@@ -74,9 +80,13 @@ func TestGerRuntimeConfig(t *testing.T) {
 			contents: []string{
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
+				"experimental = true",
+				"discover-mode = \"not-legacy\"",
 			},
 			expectedConfig: &RuntimeConfig{
 				DebugFilePath: "/foo/bar",
+				Experimental:  true,
+				DiscoverMode:  "not-legacy",
 			},
 		},
 	}

internal/discover/legacy.go

@@ -22,21 +22,21 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type stable struct {
+type legacy struct {
 	logger *logrus.Logger
 	lookup lookup.Locator
 }
 
 const (
-	nvidiaContainerRuntimeHookExecuable = "nvidia-container-runtime-hook"
-	hookDefaultFilePath                 = "/usr/bin/nvidia-container-runtime-hook"
+	nvidiaContainerRuntimeHookExecutable = "nvidia-container-runtime-hook"
+	hookDefaultFilePath                  = "/usr/bin/nvidia-container-runtime-hook"
 )
 
-var _ Discover = (*stable)(nil)
+var _ Discover = (*legacy)(nil)
 
-// NewStableDiscoverer creates a discoverer for the stable runtime
-func NewStableDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
-	d := stable{
+// NewLegacyDiscoverer creates a discoverer for the legacy runtime
+func NewLegacyDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	d := legacy{
 		logger: logger,
 		lookup: lookup.NewPathLocator(logger, root),
 	}
@@ -44,18 +44,20 @@ func NewStableDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
 	return &d, nil
 }
 
-// Hooks returns the "stable" NVIDIA Container Runtime hook
-func (d stable) Hooks() ([]Hook, error) {
+// Hooks returns the "legacy" NVIDIA Container Runtime hook. This hook calls out
+// to the nvidia-container-cli to make modifications to the container as defined
+// in libnvidia-container.
+func (d legacy) Hooks() ([]Hook, error) {
 	var hooks []Hook
 
 	hookPath := hookDefaultFilePath
-	targets, err := d.lookup.Locate(nvidiaContainerRuntimeHookExecuable)
+	targets, err := d.lookup.Locate(nvidiaContainerRuntimeHookExecutable)
 	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", nvidiaContainerRuntimeHookExecuable, err)
+		d.logger.Warnf("Failed to locate %v: %v", nvidiaContainerRuntimeHookExecutable, err)
 	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", nvidiaContainerRuntimeHookExecuable)
+		d.logger.Warnf("%v not found", nvidiaContainerRuntimeHookExecutable)
 	} else {
-		d.logger.Debugf("Found %v candidates: %v", nvidiaContainerRuntimeHookExecuable, targets)
+		d.logger.Debugf("Found %v candidates: %v", nvidiaContainerRuntimeHookExecutable, targets)
 		hookPath = targets[0]
 	}
 	d.logger.Debugf("Using NVIDIA Container Runtime Hook path %v", hookPath)

internal/edits/edits.go

@@ -0,0 +1,66 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	ociSpecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+type edits struct {
+	cdi.ContainerEdits
+	logger *logrus.Logger
+}
+
+// NewSpecEdits creates a SpecModifier that defines the required OCI spec edits (as CDI ContainerEdits) from the specified
+// discoverer.
+func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	hooks, err := d.Hooks()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover hooks: %v", err)
+	}
+
+	c := cdi.ContainerEdits{}
+	for _, h := range hooks {
+		c.Append(hook(h).toEdits())
+	}
+
+	e := edits{
+		ContainerEdits: c,
+		logger:         logger,
+	}
+
+	return &e, nil
+}
+
+// Modify applies the defined edits to the incoming OCI spec
+func (e *edits) Modify(spec *ociSpecs.Spec) error {
+	if e == nil || e.ContainerEdits.ContainerEdits == nil {
+		return nil
+	}
+
+	e.logger.Infof("Hooks:")
+	for _, hook := range e.Hooks {
+		e.logger.Infof("Injecting %v", hook.Args)
+	}
+	return e.Apply(spec)
+}

internal/edits/hook.go

@@ -0,0 +1,46 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type hook discover.Hook
+
+// toEdits converts a discovered hook to CDI Container Edits.
+func (d hook) toEdits() *cdi.ContainerEdits {
+	e := cdi.ContainerEdits{
+		ContainerEdits: &specs.ContainerEdits{
+			Hooks: []*specs.Hook{d.toSpec()},
+		},
+	}
+	return &e
+}
+
+// toSpec converts a discovered Hook to a CDI Spec Hook. Note
+// that missing info is filled in when edits are applied by querying the Hook node.
+func (d hook) toSpec() *specs.Hook {
+	s := specs.Hook{
+		HookName: d.Lifecycle,
+		Path:     d.Path,
+		Args:     d.Args,
+	}
+	return &s
+}