Add support for creating oci hook to nvidia-ctk

This change extends the nvidia-ctk runtime configure command
with a --config-mode=oci-hook that creates an OCI hook json file.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

CHANGELOG.md

@@ -1,6 +1,7 @@
 # NVIDIA Container Toolkit Changelog
 
 ## v1.14.0-rc.3
+* Added support for generating OCI hook JSON file to `nvidia-ctk runtime configure` command.
 
 ## v1.14.0-rc.2
 

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -25,6 +25,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
 	"github.com/urfave/cli/v2"
 )
 
@@ -34,8 +35,9 @@ const (
 	// defaultNVIDIARuntimeName is the default name to use in configs for the NVIDIA Container Runtime
 	defaultNVIDIARuntimeName = "nvidia"
 	// defaultNVIDIARuntimeExecutable is the default NVIDIA Container Runtime executable file name
-	defaultNVIDIARuntimeExecutable      = "nvidia-container-runtime"
+	defaultNVIDIARuntimeExecutable          = "nvidia-container-runtime"
-	defailtNVIDIARuntimeExpecutablePath = "/usr/bin/nvidia-container-runtime"
+	defaultNVIDIARuntimeExpecutablePath     = "/usr/bin/nvidia-container-runtime"
+	defaultNVIDIARuntimeHookExpecutablePath = "/usr/bin/nvidia-container-runtime-hook"
 
 	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
 	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
@@ -60,10 +62,13 @@ type config struct {
 	dryRun         bool
 	runtime        string
 	configFilePath string
+	mode           string
+	hookFilePath   string
 
 	nvidiaRuntime struct {
 		name         string
 		path         string
+		hookPath     string
 		setAsDefault bool
 	}
 }
@@ -77,7 +82,7 @@ func (m command) build() *cli.Command {
 		Name:  "configure",
 		Usage: "Add a runtime to the specified container engine",
 		Before: func(c *cli.Context) error {
-			return validateFlags(c, &config)
+			return m.validateFlags(c, &config)
 		},
 		Action: func(c *cli.Context) error {
 			return m.configureWrapper(c, &config)
@@ -101,6 +106,16 @@ func (m command) build() *cli.Command {
 			Usage:       "path to the config file for the target runtime",
 			Destination: &config.configFilePath,
 		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the config mode for runtimes that support multiple configuration mechanisms",
+			Destination: &config.mode,
+		},
+		&cli.StringFlag{
+			Name:        "oci-hook-path",
+			Usage:       "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
+			Destination: &config.hookFilePath,
+		},
 		&cli.StringFlag{
 			Name:        "nvidia-runtime-name",
 			Usage:       "specify the name of the NVIDIA runtime that will be added",
@@ -114,6 +129,12 @@ func (m command) build() *cli.Command {
 			Value:       defaultNVIDIARuntimeExecutable,
 			Destination: &config.nvidiaRuntime.path,
 		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-hook-path",
+			Usage:       "specify the path to the NVIDIA Container Runtime hook executable",
+			Value:       defaultNVIDIARuntimeHookExpecutablePath,
+			Destination: &config.nvidiaRuntime.hookPath,
+		},
 		&cli.BoolFlag{
 			Name:        "nvidia-set-as-default",
 			Aliases:     []string{"set-as-default"},
@@ -125,7 +146,18 @@ func (m command) build() *cli.Command {
 	return &configure
 }
 
-func validateFlags(c *cli.Context, config *config) error {
+func (m command) validateFlags(c *cli.Context, config *config) error {
+	if config.mode == "oci-hook" {
+		if !filepath.IsAbs(config.nvidiaRuntime.hookPath) {
+			return fmt.Errorf("the NVIDIA runtime hook path %q is not an absolute path", config.nvidiaRuntime.hookPath)
+		}
+		return nil
+	}
+	if config.mode != "" && config.mode != "config-file" {
+		m.logger.Warningf("Ignoring unsupported config mode for %v: %q", config.runtime, config.mode)
+	}
+	config.mode = "config-file"
+
 	switch config.runtime {
 	case "containerd", "crio", "docker":
 		break
@@ -136,7 +168,7 @@ func validateFlags(c *cli.Context, config *config) error {
 	switch config.runtime {
 	case "containerd", "crio":
 		if config.nvidiaRuntime.path == defaultNVIDIARuntimeExecutable {
-			config.nvidiaRuntime.path = defailtNVIDIARuntimeExpecutablePath
+			config.nvidiaRuntime.path = defaultNVIDIARuntimeExpecutablePath
 		}
 		if !filepath.IsAbs(config.nvidiaRuntime.path) {
 			return fmt.Errorf("the NVIDIA runtime path %q is not an absolute path", config.nvidiaRuntime.path)
@@ -148,6 +180,17 @@ func validateFlags(c *cli.Context, config *config) error {
 
 // configureWrapper updates the specified container engine config to enable the NVIDIA runtime
 func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.mode {
+	case "oci-hook":
+		return m.configureOCIHook(c, config)
+	case "config-file":
+		return m.configureConfigFile(c, config)
+	}
+	return fmt.Errorf("unsupported config-mode: %v", config.mode)
+}
+
+// configureConfigFile updates the specified container engine config file to enable the NVIDIA runtime.
+func (m command) configureConfigFile(c *cli.Context, config *config) error {
 	configFilePath := config.resolveConfigFilePath()
 
 	var cfg engine.Interface
@@ -225,3 +268,12 @@ func (c *config) getOuputConfigPath() string {
 	}
 	return c.resolveConfigFilePath()
 }
+
+// configureOCIHook creates and configures the OCI hook for the NVIDIA runtime
+func (m *command) configureOCIHook(c *cli.Context, config *config) error {
+	err := ocihook.CreateHook(config.hookFilePath, config.nvidiaRuntime.hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating OCI hook: %v", err)
+	}
+	return nil
+}

pkg/config/ocihook/hooks.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 **/
 
-package main
+package ocihook
 
 // podmanHook is the hook configuration structure.
 // This is taken from `Hook` at https://github.com/containers/podman/blob/3c53200e9d61fdf95fe1da825bb2a89372551350/pkg/hooks/1.0.0/hook.go#L18

pkg/config/ocihook/oci-hook.go

@@ -0,0 +1,89 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package ocihook
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// CreateHook creates an OCI hook file for the specified NVIDIA Container Runtime hook path
+func CreateHook(hookFilePath string, nvidiaContainerRuntimeHookExecutablePath string) error {
+	var output io.Writer
+	if hookFilePath == "" {
+		output = os.Stdout
+	} else {
+		if hooksDir := filepath.Dir(hookFilePath); hooksDir != "" {
+			err := os.MkdirAll(hooksDir, 0755)
+			if err != nil {
+				return fmt.Errorf("error creating hooks directory %v: %v", hooksDir, err)
+			}
+		}
+
+		hookFile, err := os.Create(hookFilePath)
+		if err != nil {
+			return fmt.Errorf("error creating hook file '%v': %v", hookFilePath, err)
+		}
+		defer hookFile.Close()
+		output = hookFile
+	}
+
+	encoder := json.NewEncoder(output)
+	encoder.SetIndent("", "  ")
+	if err := encoder.Encode(generateOciHook(nvidiaContainerRuntimeHookExecutablePath)); err != nil {
+		return fmt.Errorf("error writing hook file: %v", err)
+	}
+	return nil
+}
+
+func generateOciHook(executablePath string) podmanHook {
+	pathParts := []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
+
+	dir := filepath.Dir(executablePath)
+	var found bool
+	for _, pathPart := range pathParts {
+		if pathPart == dir {
+			found = true
+			break
+		}
+	}
+	if !found {
+		pathParts = append(pathParts, dir)
+	}
+
+	envPath := "PATH=" + strings.Join(pathParts, ":")
+	always := true
+
+	hook := podmanHook{
+		Version: "1.0.0",
+		Stages:  []string{"prestart"},
+		Hook: specHook{
+			Path: executablePath,
+			Args: []string{filepath.Base(executablePath), "prestart"},
+			Env:  []string{envPath},
+		},
+		When: When{
+			Always:   &always,
+			Commands: []string{".*"},
+		},
+	}
+	return hook
+}

tools/container/crio/crio.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -25,6 +24,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
 	"github.com/NVIDIA/nvidia-container-toolkit/tools/container"
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
@@ -206,13 +206,8 @@ func Setup(c *cli.Context, o *options) error {
 func setupHook(o *options) error {
 	log.Infof("Installing prestart hook")
 
-	err := os.MkdirAll(o.hooksDir, 0755)
+	hookPath := filepath.Join(o.hooksDir, o.hookFilename)
-	if err != nil {
+	err := ocihook.CreateHook(hookPath, filepath.Join(o.RuntimeDir, config.NVIDIAContainerRuntimeHookExecutable))
-		return fmt.Errorf("error creating hooks directory %v: %v", o.hooksDir, err)
-	}
-
-	hookPath := getHookPath(o.hooksDir, o.hookFilename)
-	err = createHook(o.RuntimeDir, hookPath)
 	if err != nil {
 		return fmt.Errorf("error creating hook: %v", err)
 	}
@@ -262,7 +257,7 @@ func Cleanup(c *cli.Context, o *options) error {
 func cleanupHook(o *options) error {
 	log.Infof("Removing prestart hook")
 
-	hookPath := getHookPath(o.hooksDir, o.hookFilename)
+	hookPath := filepath.Join(o.hooksDir, o.hookFilename)
 	err := os.Remove(hookPath)
 	if err != nil {
 		return fmt.Errorf("error removing hook '%v': %v", hookPath, err)
@@ -295,46 +290,6 @@ func cleanupConfig(o *options) error {
 	return nil
 }
 
-func createHook(toolkitDir string, hookPath string) error {
-	hook, err := os.Create(hookPath)
-	if err != nil {
-		return fmt.Errorf("error creating hook file '%v': %v", hookPath, err)
-	}
-	defer hook.Close()
-
-	encoder := json.NewEncoder(hook)
-	err = encoder.Encode(generateOciHook(toolkitDir))
-	if err != nil {
-		return fmt.Errorf("error writing hook file '%v': %v", hookPath, err)
-	}
-	return nil
-}
-
-func getHookPath(hooksDir string, hookFilename string) string {
-	return filepath.Join(hooksDir, hookFilename)
-}
-
-func generateOciHook(toolkitDir string) podmanHook {
-	hookPath := filepath.Join(toolkitDir, config.NVIDIAContainerRuntimeHookExecutable)
-	envPath := "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:" + toolkitDir
-	always := true
-
-	hook := podmanHook{
-		Version: "1.0.0",
-		Stages:  []string{"prestart"},
-		Hook: specHook{
-			Path: hookPath,
-			Args: []string{filepath.Base(config.NVIDIAContainerRuntimeHookExecutable), "prestart"},
-			Env:  []string{envPath},
-		},
-		When: When{
-			Always:   &always,
-			Commands: []string{".*"},
-		},
-	}
-	return hook
-}
-
 // RestartCrio restarts crio depending on the value of restartModeFlag
 func RestartCrio(o *options) error {
 	return o.Restart("crio", func(string) error { return fmt.Errorf("supporting crio via signal is unsupported") })