Merge branch 'update-libnvidia-container' into 'main'

Update libnvidia-container to v1.12.0-rc.3

See merge request nvidia/container-toolkit/container-toolkit!259

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,6 +47,7 @@ stages:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: amazonlinux2
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-centos7:
   rules:
@@ -54,35 +55,41 @@ stages:
   variables:
     DIST: centos7
     CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-centos8:
   variables:
     DIST: centos8
     CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-debian10:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: debian10
+    PACKAGE_REPO_TYPE: debian
 
 .dist-debian9:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: debian9
+    PACKAGE_REPO_TYPE: debian
 
 .dist-fedora35:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: fedora35
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-opensuse-leap15.1:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: opensuse-leap15.1
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-ubi8:
   rules:
@@ -90,17 +97,20 @@ stages:
   variables:
     DIST: ubi8
     CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-ubuntu16.04:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: ubuntu16.04
+    PACKAGE_REPO_TYPE: debian
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
     CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+    PACKAGE_REPO_TYPE: debian
 
 .dist-ubuntu20.04:
   rules:
@@ -108,6 +118,7 @@ stages:
   variables:
     DIST: ubuntu20.04
     CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+    PACKAGE_REPO_TYPE: debian
 
 .dist-packaging:
   variables:
@@ -169,7 +180,7 @@ test-packaging:
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.3.10
+    - export REGCTL_VERSION=v0.4.5
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,6 +35,9 @@ variables:
   # Define the public staging registry
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  # TODO: We should set the kitmaker release folder here once we have the end-to-end workflow set up
+  KITMAKER_RELEASE_FOLDER: "testing"
 
 .image-pull:
   stage: image-build
@@ -92,6 +95,109 @@ image-packaging:
     - .dist-packaging
     - .image-pull
 
+# Define the package release targets
+release:packages:amazonlinux2-aarch64:
+  extends:
+    - .release:packages
+    - .dist-amazonlinux2
+    - .arch-aarch64
+
+release:packages:amazonlinux2-x86_64:
+  extends:
+    - .release:packages
+    - .dist-amazonlinux2
+    - .arch-x86_64
+
+release:packages:centos7-ppc64le:
+  extends:
+    - .release:packages
+    - .dist-centos7
+    - .arch-ppc64le
+
+release:packages:centos7-x86_64:
+  extends:
+    - .release:packages
+    - .dist-centos7
+    - .arch-x86_64
+
+release:packages:centos8-aarch64:
+  extends:
+    - .release:packages
+    - .dist-centos8
+    - .arch-aarch64
+
+release:packages:centos8-ppc64le:
+  extends:
+    - .release:packages
+    - .dist-centos8
+    - .arch-ppc64le
+
+release:packages:centos8-x86_64:
+  extends:
+    - .release:packages
+    - .dist-centos8
+    - .arch-x86_64
+
+release:packages:debian10-amd64:
+  extends:
+    - .release:packages
+    - .dist-debian10
+    - .arch-amd64
+
+release:packages:debian9-amd64:
+  extends:
+    - .release:packages
+    - .dist-debian9
+    - .arch-amd64
+
+release:packages:fedora35-aarch64:
+  extends:
+    - .release:packages
+    - .dist-fedora35
+    - .arch-aarch64
+
+release:packages:fedora35-x86_64:
+  extends:
+    - .release:packages
+    - .dist-fedora35
+    - .arch-x86_64
+
+release:packages:opensuse-leap15.1-x86_64:
+  extends:
+    - .release:packages
+    - .dist-opensuse-leap15.1
+    - .arch-x86_64
+
+release:packages:ubuntu16.04-amd64:
+  extends:
+    - .release:packages
+    - .dist-ubuntu16.04
+    - .arch-amd64
+
+release:packages:ubuntu16.04-ppc64le:
+  extends:
+    - .release:packages
+    - .dist-ubuntu16.04
+    - .arch-ppc64le
+
+release:packages:ubuntu18.04-amd64:
+  extends:
+    - .release:packages
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+release:packages:ubuntu18.04-arm64:
+  extends:
+    - .release:packages
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+release:packages:ubuntu18.04-ppc64le:
+  extends:
+    - .release:packages
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
 # We skip the integration tests for the internal CI:
 .integration:
   stage: test
@@ -202,6 +308,27 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
+.release:packages:
+  stage: release
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}"
+    # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
+    - ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}"
+
 release:staging-ubuntu18.04:
   extends:
     - .release:staging

CHANGELOG.md

@@ -1,11 +1,22 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.12.0-rc.3
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
 ## v1.12.0-rc.1
 
 * Add support for multiple Docker Swarm resources
 * Improve injection of Vulkan configurations and libraries
 * Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
 * [libnvidia-container] Include NVVM compiler library in compute libs
+
 ## v1.11.0
 
 * Promote v1.11.0-rc.3 to v1.11.0

build/container/Dockerfile.packaging

@@ -25,5 +25,7 @@ ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' > /artifacts/manifest.txt
 
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

cmd/nvidia-container-runtime-hook/container_config.go

@@ -164,10 +164,23 @@ func isPrivileged(s *Spec) bool {
 }
 
 func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
-	// Build a list of envvars to consider. Note that the Swarm Resource envvars have a higher precedence.
-	envVars := append(swarmResourceEnvvars, envNVVisibleDevices)
+	// We check if the image has at least one of the Swarm resource envvars defined and use this
+	// if specified.
+	var hasSwarmEnvvar bool
+	for _, envvar := range swarmResourceEnvvars {
+		if _, exists := image[envvar]; exists {
+			hasSwarmEnvvar = true
+			break
+		}
+	}
+
+	var devices []string
+	if hasSwarmEnvvar {
+		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
+	} else {
+		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
+	}
 
-	devices := image.DevicesFromEnvvars(envVars...)
 	if len(devices) == 0 {
 		return nil
 	}

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"path/filepath"
 	"testing"
 
@@ -906,17 +907,20 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			expectedDevices: &anotherGPUID,
 		},
 		{
-			description:          "First available swarm resource envvar is selected and overrides NVIDIA_VISIBLE_DEVICES if present",
+			description:          "All available swarm resource envvars are selected and override NVIDIA_VISIBLE_DEVICES if present",
 			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
 			env: map[string]string{
 				envNVVisibleDevices:               gpuID,
 				"DOCKER_RESOURCE_GPUS":            thirdGPUID,
 				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
 			},
-			expectedDevices: &thirdGPUID,
+			expectedDevices: func() *string {
+				result := fmt.Sprintf("%s,%s", thirdGPUID, anotherGPUID)
+				return &result
+			}(),
 		},
 		{
-			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS override NVIDIA_VISIBLE_DEVICES if present",
 			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
 			env: map[string]string{
 				envNVVisibleDevices:               gpuID,

cmd/nvidia-container-runtime/main.go

@@ -48,7 +48,12 @@ func run(argv []string) (rerr error) {
 	if err != nil {
 		return fmt.Errorf("failed to set up logger: %v", err)
 	}
-	defer logger.Reset()
+	defer func() {
+		if rerr != nil {
+			logger.Errorf("%v", rerr)
+		}
+		logger.Reset()
+	}()
 
 	logger.Debugf("Command line arguments: %v", argv)
 	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)

cmd/nvidia-ctk/README.md

@@ -15,3 +15,34 @@ nvidia-ctk runtime configure --set-as-default
 ```
 will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
 engine.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,50 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/common.go

@@ -0,0 +1,60 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
+// This includes driver libraries and meta devices, for example.
+func NewCommonDiscoverer(logger *logrus.Logger, root string, nvmllib nvml.Interface) (discover.Discover, error) {
+	metaDevices := discover.NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(logger, root),
+		root,
+		[]string{
+			"/dev/nvidia-modeset",
+			"/dev/nvidia-uvm-tools",
+			"/dev/nvidia-uvm",
+			"/dev/nvidiactl",
+		},
+	)
+
+	graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing discoverer for graphics mounts: %v", err)
+	}
+
+	driverFiles, err := NewDriverDiscoverer(logger, root, nvmllib)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err)
+	}
+
+	d := discover.Merge(
+		metaDevices,
+		graphicsMounts,
+		driverFiles,
+	)
+
+	return d, nil
+}

cmd/nvidia-ctk/cdi/generate/device-folder-permissions.go

@@ -0,0 +1,106 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+type deviceFolderPermissions struct {
+	logger  *logrus.Logger
+	root    string
+	folders []string
+}
+
+var _ discover.Discover = (*deviceFolderPermissions)(nil)
+
+// NewDeviceFolderPermissionHookDiscoverer creates a discoverer that can be used to update the permissions for the parent folders of nested device nodes from the specified set of device specs.
+// This works around an issue with rootless podman when using crun as a low-level runtime.
+// See https://github.com/containers/crun/issues/1047
+// The nested devices that are applicable to the NVIDIA GPU devices are:
+//   - DRM devices at /dev/dri/*
+//   - NVIDIA Caps devices at /dev/nvidia-caps/*
+func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string, deviceSpecs []specs.Device) (discover.Discover, error) {
+	var folders []string
+	seen := make(map[string]bool)
+	for _, device := range deviceSpecs {
+		for _, dn := range device.ContainerEdits.DeviceNodes {
+			df := filepath.Dir(dn.Path)
+			if seen[df] {
+				continue
+			}
+			// We only consider the special case paths
+			if df != "/dev/dri" && df != "/dev/nvidia-caps" {
+				continue
+			}
+			folders = append(folders, df)
+			seen[df] = true
+		}
+		if len(folders) == 2 {
+			break
+		}
+	}
+
+	if len(folders) == 0 {
+		return discover.None{}, nil
+	}
+
+	d := &deviceFolderPermissions{
+		logger:  logger,
+		root:    root,
+		folders: folders,
+	}
+
+	return d, nil
+}
+
+// Devices are empty for this discoverer
+func (d *deviceFolderPermissions) Devices() ([]discover.Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a set of hooks that sets the file mode to 755 of parent folders for nested device nodes.
+func (d *deviceFolderPermissions) Hooks() ([]discover.Hook, error) {
+	if len(d.folders) == 0 {
+		return nil, nil
+	}
+
+	args := []string{"--mode", "755"}
+	for _, folder := range d.folders {
+		args = append(args, "--path", folder)
+	}
+	hook := discover.CreateNvidiaCTKHook(
+		d.logger,
+		lookup.NewExecutableLocator(d.logger, d.root),
+		nvidiaCTKExecutable,
+		nvidiaCTKDefaultFilePath,
+		"chmod",
+		args...,
+	)
+
+	return []discover.Hook{hook}, nil
+}
+
+// Mounts are empty for this discoverer
+func (d *deviceFolderPermissions) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}

cmd/nvidia-ctk/cdi/generate/device.go

@@ -0,0 +1,57 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/sirupsen/logrus"
+)
+
+// deviceDiscoverer defines a discoverer for device nodes
+type deviceDiscoverer struct {
+	logger          *logrus.Logger
+	root            string
+	deviceNodePaths []string
+}
+
+var _ discover.Discover = (*deviceDiscoverer)(nil)
+
+// Devices returns the device nodes for the full GPU.
+func (d *deviceDiscoverer) Devices() ([]discover.Device, error) {
+	var deviceNodes []discover.Device
+	for _, dn := range d.deviceNodePaths {
+		deviceNode := discover.Device{
+			HostPath: filepath.Join(d.root, dn),
+			Path:     dn,
+		}
+		deviceNodes = append(deviceNodes, deviceNode)
+	}
+
+	return deviceNodes, nil
+}
+
+// Hooks returns no hooks for a device discoverer
+func (d *deviceDiscoverer) Hooks() ([]discover.Hook, error) {
+	return nil, nil
+}
+
+// Mounts returns no mounts for a device discoverer
+func (d *deviceDiscoverer) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}

cmd/nvidia-ctk/cdi/generate/driver.go

@@ -0,0 +1,171 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+type driverLibraries struct {
+	logger    *logrus.Logger
+	root      string
+	libraries []string
+}
+
+var _ discover.Discover = (*driverLibraries)(nil)
+
+// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
+// The supplied NVML Library is used to query the expected driver version.
+func NewDriverDiscoverer(logger *logrus.Logger, root string, nvmllib nvml.Interface) (discover.Discover, error) {
+	version, r := nvmllib.SystemGetDriverVersion()
+	if r != nvml.SUCCESS {
+		return nil, fmt.Errorf("failed to determine driver version: %v", r)
+	}
+
+	libraries, err := NewDriverLibraryDiscoverer(logger, root, version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
+	}
+
+	firmwares := NewDriverFirmwareDiscoverer(logger, root, version)
+
+	binaries := NewDriverBinariesDiscoverer(logger, root)
+
+	d := discover.Merge(
+		libraries,
+		firmwares,
+		binaries,
+	)
+
+	return d, nil
+}
+
+// NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version.
+func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, version string) (discover.Discover, error) {
+	libraries, err := findVersionLibs(logger, root, version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get libraries for driver version: %v", err)
+	}
+
+	d := driverLibraries{
+		logger:    logger,
+		root:      root,
+		libraries: libraries,
+	}
+
+	return &d, nil
+}
+
+// NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version.
+func NewDriverFirmwareDiscoverer(logger *logrus.Logger, root string, version string) discover.Discover {
+	gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp.bin")
+	return discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{gspFirmwarePath},
+	)
+}
+
+// NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver.
+func NewDriverBinariesDiscoverer(logger *logrus.Logger, root string) discover.Discover {
+	return discover.NewMounts(
+		logger,
+		lookup.NewExecutableLocator(logger, root),
+		root,
+		[]string{
+			"nvidia-smi",              /* System management interface */
+			"nvidia-debugdump",        /* GPU coredump utility */
+			"nvidia-persistenced",     /* Persistence mode utility */
+			"nvidia-cuda-mps-control", /* Multi process service CLI */
+			"nvidia-cuda-mps-server",  /* Multi process service server */
+		},
+	)
+}
+
+// Devices are empty for this discoverer
+func (d *driverLibraries) Devices() ([]discover.Device, error) {
+	return nil, nil
+}
+
+// Mounts returns the mounts for the driver libraries
+func (d *driverLibraries) Mounts() ([]discover.Mount, error) {
+	var mounts []discover.Mount
+	for _, d := range d.libraries {
+		mount := discover.Mount{
+			HostPath: d,
+			Path:     d,
+		}
+		mounts = append(mounts, mount)
+	}
+
+	return mounts, nil
+}
+
+// Hooks returns a hook that updates the LDCache for the specified driver library paths.
+func (d *driverLibraries) Hooks() ([]discover.Hook, error) {
+	locator := lookup.NewExecutableLocator(d.logger, d.root)
+
+	hook := discover.CreateLDCacheUpdateHook(
+		d.logger,
+		locator,
+		nvidiaCTKExecutable,
+		nvidiaCTKDefaultFilePath,
+		d.libraries,
+	)
+
+	return []discover.Hook{hook}, nil
+}
+
+func findVersionLibs(logger *logrus.Logger, root string, version string) ([]string, error) {
+	logger.Infof("Using driver version %v", version)
+
+	cache, err := ldcache.New(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load ldcache: %v", err)
+	}
+
+	libs32, libs64 := cache.List()
+
+	var libs []string
+	for _, l := range libs64 {
+		if strings.HasSuffix(l, version) {
+			logger.Infof("found 64-bit driver lib: %v", l)
+			libs = append(libs, l)
+		}
+	}
+
+	for _, l := range libs32 {
+		if strings.HasSuffix(l, version) {
+			logger.Infof("found 32-bit driver lib: %v", l)
+			libs = append(libs, l)
+		}
+	}
+
+	return libs, nil
+}

cmd/nvidia-ctk/cdi/generate/full-gpu.go

@@ -0,0 +1,148 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// fullGPUDiscoverer wraps a deviceDiscoverer and adds specifics required for discovering full GPUs
+type fullGPUDiscoverer struct {
+	deviceDiscoverer
+
+	pciBusID string
+}
+
+var _ discover.Discover = (*fullGPUDiscoverer)(nil)
+
+// NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device.
+func NewFullGPUDiscoverer(logger *logrus.Logger, root string, d device.Device) (discover.Discover, error) {
+	// TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface.
+	// This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin.
+	minor, ret := d.GetMinorNumber()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
+	}
+	path := fmt.Sprintf("/dev/nvidia%d", minor)
+
+	pciInfo, ret := d.GetPciInfo()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting PCI info for device: %v", ret)
+	}
+	pciBusID := getBusID(pciInfo)
+
+	drmDeviceNodes, err := drm.GetDeviceNodesByBusID(pciBusID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", pciBusID, err)
+	}
+
+	deviceNodePaths := append([]string{path}, drmDeviceNodes...)
+
+	device := fullGPUDiscoverer{
+		deviceDiscoverer: deviceDiscoverer{
+			logger:          logger,
+			root:            root,
+			deviceNodePaths: deviceNodePaths,
+		},
+		pciBusID: pciBusID,
+	}
+
+	return &device, nil
+}
+
+// Hooks returns the hooks for the GPU device.
+// The following hooks are detected:
+//  1. A hook to create /dev/dri/by-path symlinks
+func (d *fullGPUDiscoverer) Hooks() ([]discover.Hook, error) {
+	links, err := d.deviceNodeLinks()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover DRA device links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	hookPath := "nvidia-ctk"
+	args := []string{hookPath, "hook", "create-symlinks"}
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	var hooks []discover.Hook
+	hook := discover.Hook{
+		Lifecycle: "createContainer",
+		Path:      hookPath,
+		Args:      args,
+	}
+	hooks = append(hooks, hook)
+
+	return hooks, nil
+}
+
+// Mounts returns an empty slice for a full GPU
+func (d *fullGPUDiscoverer) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}
+
+func (d *fullGPUDiscoverer) deviceNodeLinks() ([]string, error) {
+	candidates := []string{
+		fmt.Sprintf("/dev/dri/by-path/pci-%s-card", d.pciBusID),
+		fmt.Sprintf("/dev/dri/by-path/pci-%s-render", d.pciBusID),
+	}
+
+	var links []string
+	for _, c := range candidates {
+		linkPath := filepath.Join(d.root, c)
+		device, err := os.Readlink(linkPath)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", linkPath)
+			continue
+		}
+
+		d.logger.Debugf("adding device symlink %v -> %v", linkPath, device)
+		links = append(links, fmt.Sprintf("%v::%v", device, linkPath))
+	}
+
+	return links, nil
+}
+
+// getBusID provides a utility function that returns the string representation of the bus ID.
+func getBusID(p nvml.PciInfo) string {
+	var bytes []byte
+	for _, b := range p.BusId {
+		if byte(b) == '\x00' {
+			break
+		}
+		bytes = append(bytes, byte(b))
+	}
+	id := strings.ToLower(string(bytes))
+
+	if id != "0000" {
+		id = strings.TrimPrefix(id, "0000")
+	}
+
+	return id
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,344 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/" + nvidiaCTKExecutable
+
+	formatJSON = "json"
+	formatYAML = "yaml"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	output string
+	format string
+	root   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &cfg.output,
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       formatYAML,
+			Destination: &cfg.format,
+		},
+		&cli.StringFlag{
+			Name:        "root",
+			Usage:       "Specify the root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &cfg.root,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	cfg.format = strings.ToLower(cfg.format)
+	switch cfg.format {
+	case formatJSON:
+	case formatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", cfg.format)
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	spec, err := m.generateSpec(cfg.root)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+
+	var outputTo io.Writer
+	if cfg.output == "" {
+		outputTo = os.Stdout
+	} else {
+		err := createParentDirsIfRequired(cfg.output)
+		if err != nil {
+			return fmt.Errorf("failed to create parent folders for output file: %v", err)
+		}
+		outputFile, err := os.Create(cfg.output)
+		if err != nil {
+			return fmt.Errorf("failed to create output file: %v", err)
+		}
+		defer outputFile.Close()
+		outputTo = outputFile
+	}
+
+	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			cfg.format = outputFileFormat
+		} else if outputFileFormat != cfg.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", cfg.format, outputFileFormat)
+		}
+	}
+
+	data, err := yaml.Marshal(spec)
+	if err != nil {
+		return fmt.Errorf("failed to marshal CDI spec: %v", err)
+	}
+
+	if strings.ToLower(cfg.format) == formatJSON {
+		data, err = yaml.YAMLToJSONStrict(data)
+		if err != nil {
+			return fmt.Errorf("failed to convert CDI spec from YAML to JSON: %v", err)
+		}
+	}
+
+	err = writeToOutput(cfg.format, data, outputTo)
+	if err != nil {
+		return fmt.Errorf("failed to write output: %v", err)
+	}
+
+	return nil
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return formatJSON
+	case ".yaml":
+		return formatYAML
+	case ".yml":
+		return formatYAML
+	}
+
+	return ""
+}
+
+func writeToOutput(format string, data []byte, output io.Writer) error {
+	if format == formatYAML {
+		_, err := output.Write([]byte("---\n"))
+		if err != nil {
+			return fmt.Errorf("failed to write YAML separator: %v", err)
+		}
+	}
+	_, err := output.Write(data)
+	if err != nil {
+		return fmt.Errorf("failed to write data: %v", err)
+	}
+
+	return nil
+}
+
+func (m command) generateSpec(root string) (*specs.Spec, error) {
+	nvmllib := nvml.New()
+	if r := nvmllib.Init(); r != nvml.SUCCESS {
+		return nil, r
+	}
+	defer nvmllib.Shutdown()
+
+	devicelib := device.New(device.WithNvml(nvmllib))
+
+	deviceSpecs, err := m.generateDeviceSpecs(devicelib, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+
+	allDevice := createAllDevice(deviceSpecs)
+
+	deviceSpecs = append(deviceSpecs, allDevice)
+
+	allEdits := edits.NewContainerEdits()
+
+	ipcs, err := NewIPCDiscoverer(m.logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
+	}
+
+	ipcEdits, err := edits.FromDiscoverer(ipcs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for IPC sockets: %v", err)
+	}
+	// TODO: We should not have to update this after the fact
+	for _, s := range ipcEdits.Mounts {
+		s.Options = append(s.Options, "noexec")
+	}
+
+	allEdits.Append(ipcEdits)
+
+	common, err := NewCommonDiscoverer(m.logger, root, nvmllib)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
+	}
+
+	deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, root, deviceSpecs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
+	}
+
+	commonEdits, err := edits.FromDiscoverer(discover.Merge(common, deviceFolderPermissionHooks))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for common entities: %v", err)
+	}
+
+	allEdits.Append(commonEdits)
+
+	// Construct the spec
+	// TODO: Use the code to determine the minimal version
+	spec := specs.Spec{
+		Version:        "0.4.0",
+		Kind:           "nvidia.com/gpu",
+		Devices:        deviceSpecs,
+		ContainerEdits: *allEdits.ContainerEdits,
+	}
+
+	return &spec, nil
+}
+
+func (m command) generateDeviceSpecs(devicelib device.Interface, root string) ([]specs.Device, error) {
+	var deviceSpecs []specs.Device
+
+	err := devicelib.VisitDevices(func(i int, d device.Device) error {
+		isMigEnabled, err := d.IsMigEnabled()
+		if err != nil {
+			return fmt.Errorf("failed to check whether device is MIG device: %v", err)
+		}
+		if isMigEnabled {
+			return nil
+		}
+		device, err := NewFullGPUDiscoverer(m.logger, root, d)
+		if err != nil {
+			return fmt.Errorf("failed to create device: %v", err)
+		}
+
+		deviceEdits, err := edits.FromDiscoverer(device)
+		if err != nil {
+			return fmt.Errorf("failed to create container edits for device: %v", err)
+		}
+
+		deviceSpec := specs.Device{
+			Name:           fmt.Sprintf("gpu%d", i),
+			ContainerEdits: *deviceEdits.ContainerEdits,
+		}
+
+		deviceSpecs = append(deviceSpecs, deviceSpec)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate CDI spec for GPU devices: %v", err)
+	}
+
+	err = devicelib.VisitMigDevices(func(i int, d device.Device, j int, mig device.MigDevice) error {
+		device, err := NewMigDeviceDiscoverer(m.logger, "", d, mig)
+		if err != nil {
+			return fmt.Errorf("failed to create MIG device: %v", err)
+		}
+
+		deviceEdits, err := edits.FromDiscoverer(device)
+		if err != nil {
+			return fmt.Errorf("failed to create container edits for MIG device: %v", err)
+		}
+
+		deviceSpec := specs.Device{
+			Name:           fmt.Sprintf("mig%v:%v", i, j),
+			ContainerEdits: *deviceEdits.ContainerEdits,
+		}
+
+		deviceSpecs = append(deviceSpecs, deviceSpec)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("falied to generate CDI spec for MIG devices: %v", err)
+	}
+
+	return deviceSpecs, nil
+}
+
+// createAllDevice creates an 'all' device which combines the edits from the previous devices
+func createAllDevice(deviceSpecs []specs.Device) specs.Device {
+	edits := edits.NewContainerEdits()
+
+	for _, d := range deviceSpecs {
+		edit := cdi.ContainerEdits{
+			ContainerEdits: &d.ContainerEdits,
+		}
+		edits.Append(&edit)
+	}
+
+	all := specs.Device{
+		Name:           "all",
+		ContainerEdits: *edits.ContainerEdits,
+	}
+	return all
+}
+
+// createParentDirsIfRequired creates the parent folders of the specified path if requried.
+// Note that MkdirAll does not specifically check whether the specified path is non-empty and raises an error if it is.
+// The path will be empty if filename in the current folder is specified, for example
+func createParentDirsIfRequired(filename string) error {
+	dir := filepath.Dir(filename)
+	if dir == "" {
+		return nil
+	}
+	return os.MkdirAll(dir, 0755)
+}

cmd/nvidia-ctk/cdi/generate/ipc.go

@@ -0,0 +1,42 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger *logrus.Logger, root string) (discover.Discover, error) {
+	d := discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{
+			"/var/run/nvidia-persistenced/socket",
+			"/var/run/nvidia-fabricmanager/socket",
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	return d, nil
+}

cmd/nvidia-ctk/cdi/generate/mig-device.go

@@ -0,0 +1,84 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// migDeviceDiscoverer wraps a deviceDiscoverer and adds specifics required for discovering MIG devices.
+type migDeviceDiscoverer struct {
+	deviceDiscoverer
+}
+
+var _ discover.Discover = (*migDeviceDiscoverer)(nil)
+
+// NewMigDeviceDiscoverer creates a discoverer for the specified mig device and its parent.
+func NewMigDeviceDiscoverer(logger *logrus.Logger, root string, parent device.Device, d device.MigDevice) (discover.Discover, error) {
+	minor, ret := parent.GetMinorNumber()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
+	}
+	parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
+
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
+	}
+
+	gi, ret := d.GetGpuInstanceId()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
+	}
+
+	ci, ret := d.GetComputeInstanceId()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
+	}
+
+	giCap := nvcaps.NewGPUInstanceCap(minor, gi)
+	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
+	}
+
+	ciCap := nvcaps.NewComputeInstanceCap(minor, gi, ci)
+	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
+	}
+
+	m := migDeviceDiscoverer{
+		deviceDiscoverer: deviceDiscoverer{
+			logger: logger,
+			root:   root,
+			deviceNodePaths: []string{
+				parentPath,
+				giCapDevicePath,
+				ciCapDevicePath,
+			},
+		},
+	}
+
+	return &m, nil
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	paths         cli.StringSlice
+	mode          string
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specifiy a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.mode) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value())
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	locator := lookup.NewExecutableLocator(m.logger, "")
+	targets, err := locator.Locate("chmod")
+	if err != nil {
+		return fmt.Errorf("failed to locate chmod: %v", err)
+	}
+	chmodPath := targets[0]
+
+	args := append([]string{filepath.Base(chmodPath), cfg.mode}, paths...)
+
+	return syscall.Exec(chmodPath, args, nil)
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		pathsInRoot = append(pathsInRoot, filepath.Join(root, f))
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -74,7 +74,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringSliceFlag{
 			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as source:target",
+			Usage:       "Specify a specific link to create. The link is specified as target::link",
 			Destination: &cfg.links,
 		},
 		&cli.StringFlag{
@@ -145,7 +145,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 
 	links := cfg.links.Value()
 	for _, l := range links {
-		parts := strings.Split(l, ":")
+		parts := strings.Split(l, "::")
 		if len(parts) != 2 {
 			m.logger.Warnf("Invalid link specification %v", l)
 			continue

cmd/nvidia-ctk/hook/hook.go

@@ -17,6 +17,7 @@
 package hook
 
 import (
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
 	"github.com/sirupsen/logrus"
@@ -46,6 +47,7 @@ func (m hookCommand) build() *cli.Command {
 	hook.Subcommands = []*cli.Command{
 		ldcache.NewCommand(m.logger),
 		symlinks.NewCommand(m.logger),
+		chmod.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/info/generate-cdi/generate-cdi.go

@@ -1,386 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package cdi
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-	"sigs.k8s.io/yaml"
-)
-
-const (
-	nvidiaCTKExecutable      = "nvidia-ctk"
-	nvidiaCTKDefaultFilePath = "/usr/bin/" + nvidiaCTKExecutable
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	output   string
-	jsonMode bool
-}
-
-// NewCommand constructs a generate-cdi command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build creates the CLI command
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the 'generate-cdi' command
-	c := cli.Command{
-		Name:  "generate-cdi",
-		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "output",
-			Usage:       "Specify the file to output the generated CDI specification to. If this is '-' or '' the specification is output to STDOUT",
-			Destination: &cfg.output,
-		},
-		&cli.BoolFlag{
-			Name:        "json",
-			Usage:       "Output the generated CDI spec in JSON mode instead of YAML",
-			Destination: &cfg.jsonMode,
-		},
-	}
-
-	return &c
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	spec, err := m.generateSpec()
-	if err != nil {
-		return fmt.Errorf("failed to generate CDI spec: %v", err)
-	}
-
-	var outputTo io.Writer
-	if cfg.output == "" || cfg.output == "-" {
-		outputTo = os.Stdout
-	} else {
-		outputFile, err := os.Create(cfg.output)
-		if err != nil {
-			return fmt.Errorf("failed to create output file: %v", err)
-		}
-		defer outputFile.Close()
-		outputTo = outputFile
-	}
-
-	if filepath.Ext(cfg.output) == ".json" {
-		cfg.jsonMode = true
-	} else if filepath.Ext(cfg.output) == ".yaml" || filepath.Ext(cfg.output) == ".yml" {
-		cfg.jsonMode = false
-	}
-
-	data, err := yaml.Marshal(spec)
-	if err != nil {
-		return fmt.Errorf("failed to marshal CDI spec: %v", err)
-	}
-
-	if cfg.jsonMode {
-		data, err = yaml.YAMLToJSONStrict(data)
-		if err != nil {
-			return fmt.Errorf("failed to convert CDI spec from YAML to JSON: %v", err)
-		}
-	}
-
-	_, err = outputTo.Write(data)
-	if err != nil {
-		return fmt.Errorf("failed to write output: %v", err)
-	}
-
-	return nil
-}
-
-func (m command) generateSpec() (*specs.Spec, error) {
-	nvmllib := nvml.New()
-	if r := nvmllib.Init(); r != nvml.SUCCESS {
-		return nil, r
-	}
-	defer nvmllib.Shutdown()
-
-	devicelib := device.New(device.WithNvml(nvmllib))
-
-	spec := specs.Spec{
-		Version:        "0.4.0",
-		Kind:           "nvidia.com/gpu",
-		ContainerEdits: specs.ContainerEdits{},
-	}
-	err := devicelib.VisitDevices(func(i int, d device.Device) error {
-		isMig, err := d.IsMigEnabled()
-		if err != nil {
-			return fmt.Errorf("failed to check whether device is MIG device: %v", err)
-		}
-		if isMig {
-			return nil
-		}
-		device, err := generateEditsForDevice(newGPUDevice(i, d))
-		if err != nil {
-			return fmt.Errorf("failed to generate CDI spec for device %v: %v", i, err)
-		}
-
-		spec.Devices = append(spec.Devices, device)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate CDI spec for GPU devices: %v", err)
-	}
-
-	err = devicelib.VisitMigDevices(func(i int, d device.Device, j int, m device.MigDevice) error {
-		device, err := generateEditsForDevice(newMigDevice(i, j, m))
-		if err != nil {
-			return fmt.Errorf("failed to generate CDI spec for device %v: %v", i, err)
-		}
-
-		spec.Devices = append(spec.Devices, device)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("falied to generate CDI spec for MIG devices: %v", err)
-	}
-
-	// We create an "all" device with all the discovered device nodes
-	var allDeviceNodes []*specs.DeviceNode
-	for _, d := range spec.Devices {
-		for _, dn := range d.ContainerEdits.DeviceNodes {
-			allDeviceNodes = append(allDeviceNodes, dn)
-		}
-	}
-	all := specs.Device{
-		Name: "all",
-		ContainerEdits: specs.ContainerEdits{
-			DeviceNodes: allDeviceNodes,
-		},
-	}
-
-	spec.Devices = append(spec.Devices, all)
-	spec.ContainerEdits.DeviceNodes = m.getExistingMetaDeviceNodes()
-
-	libraries, err := m.findLibs(nvmllib)
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate driver libraries: %v", err)
-	}
-
-	binaries, err := m.findBinaries()
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate driver binaries: %v", err)
-	}
-
-	ipcs, err := m.findIPC()
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate driver IPC sockets: %v", err)
-	}
-
-	spec.ContainerEdits.Mounts = generateMountsForPaths(libraries, binaries, ipcs)
-
-	ldcacheUpdateHook := m.generateUpdateLdCacheHook(libraries)
-
-	spec.ContainerEdits.Hooks = []*specs.Hook{ldcacheUpdateHook}
-
-	return &spec, nil
-}
-
-func generateEditsForDevice(name string, d deviceInfo) (specs.Device, error) {
-	deviceNodePaths, err := d.GetDeviceNodes()
-	if err != nil {
-		return specs.Device{}, fmt.Errorf("failed to get paths for device: %v", err)
-	}
-
-	deviceNodes := getDeviceNodesFromPaths(deviceNodePaths)
-
-	device := specs.Device{
-		Name: name,
-		ContainerEdits: specs.ContainerEdits{
-			DeviceNodes: deviceNodes,
-		},
-	}
-
-	return device, nil
-}
-
-func (m command) getExistingMetaDeviceNodes() []*specs.DeviceNode {
-	metaDeviceNodePaths := []string{
-		"/dev/nvidia-modeset",
-		"/dev/nvidia-uvm-tools",
-		"/dev/nvidia-uvm",
-		"/dev/nvidiactl",
-	}
-
-	var existingDeviceNodePaths []string
-	for _, p := range metaDeviceNodePaths {
-		if _, err := os.Stat(p); err != nil {
-			m.logger.Infof("Ignoring missing meta device %v", p)
-			continue
-		}
-		existingDeviceNodePaths = append(existingDeviceNodePaths, p)
-	}
-
-	return getDeviceNodesFromPaths(existingDeviceNodePaths)
-}
-
-func getDeviceNodesFromPaths(deviceNodePaths []string) []*specs.DeviceNode {
-	var deviceNodes []*specs.DeviceNode
-	for _, p := range deviceNodePaths {
-		deviceNode := specs.DeviceNode{
-			Path: p,
-		}
-		deviceNodes = append(deviceNodes, &deviceNode)
-	}
-
-	return deviceNodes
-}
-
-func (m command) findLibs(nvmllib nvml.Interface) ([]string, error) {
-	version, r := nvmllib.SystemGetDriverVersion()
-	if r != nvml.SUCCESS {
-		return nil, fmt.Errorf("failed to determine driver version: %v", r)
-	}
-	m.logger.Infof("Using driver version %v", version)
-
-	cache, err := ldcache.New(m.logger, "")
-	if err != nil {
-		return nil, fmt.Errorf("failed to load ldcache: %v", err)
-	}
-
-	libs32, libs64 := cache.List()
-
-	var libs []string
-	for _, l := range libs64 {
-		if strings.HasSuffix(l, version) {
-			m.logger.Infof("found 64-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
-	}
-
-	for _, l := range libs32 {
-		if strings.HasSuffix(l, version) {
-			m.logger.Infof("found 32-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
-	}
-
-	return libs, nil
-}
-
-func (m command) findBinaries() ([]string, error) {
-	candidates := []string{
-		"nvidia-smi",              /* System management interface */
-		"nvidia-debugdump",        /* GPU coredump utility */
-		"nvidia-persistenced",     /* Persistence mode utility */
-		"nvidia-cuda-mps-control", /* Multi process service CLI */
-		"nvidia-cuda-mps-server",  /* Multi process service server */
-	}
-
-	locator := lookup.NewExecutableLocator(m.logger, "")
-
-	var binaries []string
-	for _, c := range candidates {
-		targets, err := locator.Locate(c)
-		if err != nil {
-			m.logger.Warningf("skipping %v: %v", c, err)
-			continue
-		}
-
-		binaries = append(binaries, targets[0])
-	}
-	return binaries, nil
-}
-
-func (m command) findIPC() ([]string, error) {
-	candidates := []string{
-		"/var/run/nvidia-persistenced/socket",
-		"/var/run/nvidia-fabricmanager/socket",
-		// TODO: This can be controlled by the NV_MPS_PIPE_DIR envvar
-		"/tmp/nvidia-mps",
-	}
-
-	locator := lookup.NewFileLocator(m.logger, "")
-
-	var ipcs []string
-	for _, c := range candidates {
-		targets, err := locator.Locate(c)
-		if err != nil {
-			m.logger.Warningf("skipping %v: %v", c, err)
-			continue
-		}
-
-		ipcs = append(ipcs, targets[0])
-	}
-	return ipcs, nil
-}
-
-func generateMountsForPaths(pathSets ...[]string) []*specs.Mount {
-	var mounts []*specs.Mount
-	for _, paths := range pathSets {
-		for _, p := range paths {
-			mount := specs.Mount{
-				HostPath: p,
-				// We may want to adjust the container path
-				ContainerPath: p,
-				Type:          "bind",
-				Options: []string{
-					"ro",
-					"nosuid",
-					"nodev",
-					"bind",
-				},
-			}
-			mounts = append(mounts, &mount)
-		}
-	}
-	return mounts
-}
-
-func (m command) generateUpdateLdCacheHook(libraries []string) *specs.Hook {
-	locator := lookup.NewExecutableLocator(m.logger, "")
-
-	hook := discover.CreateLDCacheUpdateHook(
-		m.logger,
-		locator,
-		nvidiaCTKExecutable,
-		nvidiaCTKDefaultFilePath,
-		libraries,
-	)
-	return &specs.Hook{
-		HookName: hook.Lifecycle,
-		Path:     hook.Path,
-		Args:     hook.Args,
-	}
-}

cmd/nvidia-ctk/info/generate-cdi/nvml_devices.go

@@ -1,123 +0,0 @@
-/*
- * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY Type, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cdi
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-)
-
-// nvmlDevice wraps an nvml.Device with more functions.
-type nvmlDevice struct {
-	nvml.Device
-}
-
-// nvmlMigDevice allows for specific functions of nvmlDevice to be overridden.
-type nvmlMigDevice nvmlDevice
-
-// deviceInfo defines the information the required to construct a Device
-type deviceInfo interface {
-	GetUUID() (string, error)
-	GetDeviceNodes() ([]string, error)
-}
-
-var _ deviceInfo = (*nvmlDevice)(nil)
-var _ deviceInfo = (*nvmlMigDevice)(nil)
-
-func newGPUDevice(i int, gpu device.Device) (string, nvmlDevice) {
-	return fmt.Sprintf("gpu%v", i), nvmlDevice{gpu}
-}
-
-func newMigDevice(i int, j int, mig device.MigDevice) (string, nvmlMigDevice) {
-	return fmt.Sprintf("mig%v:%v", i, j), nvmlMigDevice{mig}
-}
-
-// GetUUID returns the UUID of the device
-func (d nvmlDevice) GetUUID() (string, error) {
-	uuid, ret := d.Device.GetUUID()
-	if ret != nvml.SUCCESS {
-		return "", ret
-	}
-	return uuid, nil
-}
-
-// GetUUID returns the UUID of the device
-func (d nvmlMigDevice) GetUUID() (string, error) {
-	return nvmlDevice(d).GetUUID()
-}
-
-// GetDeviceNodes returns the device node paths for a GPU device
-func (d nvmlDevice) GetDeviceNodes() ([]string, error) {
-	minor, ret := d.GetMinorNumber()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
-	}
-	path := fmt.Sprintf("/dev/nvidia%d", minor)
-
-	return []string{path}, nil
-}
-
-// GetDeviceNodes returns the device node paths for a MIG device
-func (d nvmlMigDevice) GetDeviceNodes() ([]string, error) {
-	parent, ret := d.GetDeviceHandleFromMigDeviceHandle()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting parent device: %v", ret)
-	}
-	minor, ret := parent.GetMinorNumber()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
-	}
-	parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
-
-	migCaps, err := nvcaps.NewMigCaps()
-	if err != nil {
-		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
-	}
-
-	gi, ret := d.GetGpuInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
-	}
-
-	ci, ret := d.GetComputeInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
-	}
-
-	giCap := nvcaps.NewGPUInstanceCap(minor, gi)
-	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
-	}
-
-	ciCap := nvcaps.NewComputeInstanceCap(minor, gi, ci)
-	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
-	}
-
-	devicePaths := []string{
-		parentPath,
-		giCapDevicePath,
-		ciCapDevicePath,
-	}
-
-	return devicePaths, nil
-}

cmd/nvidia-ctk/info/info.go

@@ -17,7 +17,6 @@
 package info
 
 import (
-	cdi "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info/generate-cdi"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -42,9 +41,7 @@ func (m command) build() *cli.Command {
 		Usage: "Provide information about the system",
 	}
 
-	hook.Subcommands = []*cli.Command{
-		cdi.NewCommand(m.logger),
-	}
+	hook.Subcommands = []*cli.Command{}
 
 	return &hook
 }

cmd/nvidia-ctk/main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"os"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
 	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
@@ -75,6 +76,7 @@ func main() {
 		hook.NewCommand(logger),
 		runtime.NewCommand(logger),
 		infoCLI.NewCommand(logger),
+		cdi.NewCommand(logger),
 	}
 
 	// Run the CLI

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -22,7 +22,9 @@ import (
 	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
+	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -31,6 +33,7 @@ const (
 	defaultRuntime = "docker"
 
 	defaultDockerConfigFilePath = "/etc/docker/daemon.json"
+	defaultCrioConfigFilePath   = "/etc/crio/crio.conf"
 )
 
 type command struct {
@@ -75,7 +78,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringFlag{
 			Name:        "runtime",
-			Usage:       "the target runtime engine. One of [docker]",
+			Usage:       "the target runtime engine. One of [crio, docker]",
 			Value:       defaultRuntime,
 			Destination: &config.runtime,
 		},
@@ -108,6 +111,8 @@ func (m command) build() *cli.Command {
 
 func (m command) configureWrapper(c *cli.Context, config *config) error {
 	switch config.runtime {
+	case "crio":
+		return m.configureCrio(c, config)
 	case "docker":
 		return m.configureDocker(c, config)
 	}
@@ -127,9 +132,12 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
 		return fmt.Errorf("unable to load config: %v", err)
 	}
 
-	defaultRuntime := config.nvidiaOptions.DefaultRuntime()
-	runtimeConfig := config.nvidiaOptions.Runtime().DockerRuntimesConfig()
-	err = docker.UpdateConfig(cfg, defaultRuntime, runtimeConfig)
+	err = docker.UpdateConfig(
+		cfg,
+		config.nvidiaOptions.RuntimeName,
+		config.nvidiaOptions.RuntimePath,
+		config.nvidiaOptions.SetAsDefault,
+	)
 	if err != nil {
 		return fmt.Errorf("unable to update config: %v", err)
 	}
@@ -152,3 +160,44 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
 
 	return nil
 }
+
+// configureCrio updates the crio config to enable the NVIDIA Container Runtime
+func (m command) configureCrio(c *cli.Context, config *config) error {
+	configFilePath := config.configFilePath
+	if configFilePath == "" {
+		configFilePath = defaultCrioConfigFilePath
+	}
+
+	cfg, err := crio.LoadConfig(configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = crio.UpdateConfig(
+		cfg,
+		config.nvidiaOptions.RuntimeName,
+		config.nvidiaOptions.RuntimePath,
+		config.nvidiaOptions.SetAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.dryRun {
+		output, err := toml.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("unable to convert to TOML: %v", err)
+		}
+		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
+		return nil
+	}
+	err = crio.FlushConfig(configFilePath, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
+
+	return nil
+}

docker/Dockerfile.devel

@@ -14,8 +14,8 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go install golang.org/x/lint/golint@latest
+RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
 RUN go install github.com/matryer/moq@latest
-RUN go install github.com/gordonklaus/ineffassign@latest
+RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
 RUN go install github.com/client9/misspell/cmd/misspell@latest
 RUN go install github.com/google/go-licenses@latest

go.mod

@@ -1,6 +1,6 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.17
+go 1.18
 
 require (
 	github.com/BurntSushi/toml v1.0.0

go.sum

@@ -1,74 +1,18 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82 h1:x751Xx1tdxkiA/sdkv2J769n21UbYKzVOpe9S/h1M3k=
 github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/container-orchestrated-devices/container-device-interface v0.5.2 h1:Bf/Zq8UBhbSBtB+pFBVIQ2Rh7sNK/x2ZEr6uW5YjNv8=
 github.com/container-orchestrated-devices/container-device-interface v0.5.2/go.mod h1:ZToWfSyUH5l9Rk7/bjkUUkNLz4b1mE+CVUVafuikDPY=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -77,209 +21,63 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
-github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
 github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg=
 github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
 github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab h1:YQZXa3elcHgKXAa2GjVFC9M3JeP7ZPyFD1YByDx/dgQ=
 github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/runtime-tools v0.9.1-0.20220110225228-7e2d60f1e41f h1:MMcsVl0FAVEahmXTy+uXoDTw3yJq7nGrK8ITs/kkreo=
 github.com/opencontainers/runtime-tools v0.9.1-0.20220110225228-7e2d60f1e41f/go.mod h1:/tgP02fPXGHkU3/qKK1Y0Db4yqNyGm03vLq/mzHzcS4=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w=
 github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
-github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
@@ -292,176 +90,16 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220922133427-1049a7fa76a9 h1:3C1cK0/n1HGfH0MPCCPRntK8xPNlrVq9OWR4RA72jW0=
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220922133427-1049a7fa76a9/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
-go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -470,198 +108,20 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6 h1:cy1ko5847T/lJ45eyg/7uLprIE/amW5IXxGtEnQdYMI=
 golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

internal/config/crio/crio.go

@@ -0,0 +1,125 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+// LoadConfig loads the cri-o config from disk
+func LoadConfig(config string) (*toml.Tree, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return cfg, nil
+}
+
+// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
+func UpdateConfig(config *toml.Tree, runtimeClass string, runtimePath string, setAsDefault bool) error {
+	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass}, runc)
+	}
+
+	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_path"}, runtimePath)
+	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_type"}, "oci")
+
+	if setAsDefault {
+		config.SetPath([]string{"crio", "runtime", "default_runtime"}, runtimeClass)
+	}
+
+	return nil
+}
+
+// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
+func RevertConfig(config *toml.Tree, runtimeClass string) error {
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		if runtimeClass == runtime {
+			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
+		}
+	}
+
+	runtimeClassPath := []string{"crio", "runtime", "runtimes", runtimeClass}
+	config.DeletePath(runtimeClassPath)
+	for i := 0; i < len(runtimeClassPath); i++ {
+		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
+		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
+			if len(entry.Keys()) != 0 {
+				break
+			}
+			config.DeletePath(remainingPath)
+		}
+	}
+
+	return nil
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(config string, cfg *toml.Tree) error {
+	log.Infof("Flushing config")
+
+	output, err := cfg.ToTomlString()
+	if err != nil {
+		return fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(config)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(config)
+		if err != nil {
+			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(output)
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}

internal/config/docker/docker.go

@@ -57,11 +57,7 @@ func LoadConfig(configFilePath string) (map[string]interface{}, error) {
 }
 
 // UpdateConfig updates the docker config to include the nvidia runtimes
-func UpdateConfig(config map[string]interface{}, defaultRuntime string, newRuntimes map[string]interface{}) error {
-	if defaultRuntime != "" {
-		config["default-runtime"] = defaultRuntime
-	}
-
+func UpdateConfig(config map[string]interface{}, runtimeName string, runtimePath string, setAsDefault bool) error {
 	// Read the existing runtimes
 	runtimes := make(map[string]interface{})
 	if _, exists := config["runtimes"]; exists {
@@ -69,14 +65,20 @@ func UpdateConfig(config map[string]interface{}, defaultRuntime string, newRunti
 	}
 
 	// Add / update the runtime definitions
-	for name, rt := range newRuntimes {
-		runtimes[name] = rt
+	runtimes[runtimeName] = map[string]interface{}{
+		"path": runtimePath,
+		"args": []string{},
 	}
 
 	// Update the runtimes definition
 	if len(runtimes) > 0 {
 		config["runtimes"] = runtimes
 	}
+
+	if setAsDefault {
+		config["default-runtime"] = runtimeName
+	}
+
 	return nil
 }
 

internal/config/docker/docker_test.go

@@ -27,30 +27,33 @@ import (
 func TestUpdateConfigDefaultRuntime(t *testing.T) {
 	testCases := []struct {
 		config                     map[string]interface{}
-		defaultRuntime             string
 		runtimeName                string
+		setAsDefault               bool
 		expectedDefaultRuntimeName interface{}
 	}{
 		{
-			defaultRuntime:             "",
+			setAsDefault:               false,
 			expectedDefaultRuntimeName: nil,
 		},
 		{
-			defaultRuntime:             "NAME",
+			runtimeName:                "NAME",
+			setAsDefault:               true,
 			expectedDefaultRuntimeName: "NAME",
 		},
 		{
 			config: map[string]interface{}{
 				"default-runtime": "ALREADY_SET",
 			},
-			defaultRuntime:             "",
+			runtimeName:                "NAME",
+			setAsDefault:               false,
 			expectedDefaultRuntimeName: "ALREADY_SET",
 		},
 		{
 			config: map[string]interface{}{
 				"default-runtime": "ALREADY_SET",
 			},
-			defaultRuntime:             "NAME",
+			runtimeName:                "NAME",
+			setAsDefault:               true,
 			expectedDefaultRuntimeName: "NAME",
 		},
 	}
@@ -60,7 +63,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			if tc.config == nil {
 				tc.config = make(map[string]interface{})
 			}
-			err := UpdateConfig(tc.config, tc.defaultRuntime, nil)
+			err := UpdateConfig(tc.config, tc.runtimeName, "", tc.setAsDefault)
 			require.NoError(t, err)
 
 			defaultRuntimeName := tc.config["default-runtime"]
@@ -72,20 +75,14 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 func TestUpdateConfigRuntimes(t *testing.T) {
 	testCases := []struct {
 		config         map[string]interface{}
-		runtimes       map[string]interface{}
+		runtimes       map[string]string
 		expectedConfig map[string]interface{}
 	}{
 		{
 			config: map[string]interface{}{},
-			runtimes: map[string]interface{}{
-				"runtime1": map[string]interface{}{
-					"path": "/test/runtime/dir/runtime1",
-					"args": []string{},
-				},
-				"runtime2": map[string]interface{}{
-					"path": "/test/runtime/dir/runtime2",
-					"args": []string{},
-				},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+				"runtime2": "/test/runtime/dir/runtime2",
 			},
 			expectedConfig: map[string]interface{}{
 				"runtimes": map[string]interface{}{
@@ -109,15 +106,9 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 					},
 				},
 			},
-			runtimes: map[string]interface{}{
-				"runtime1": map[string]interface{}{
-					"path": "/test/runtime/dir/runtime1",
-					"args": []string{},
-				},
-				"runtime2": map[string]interface{}{
-					"path": "/test/runtime/dir/runtime2",
-					"args": []string{},
-				},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+				"runtime2": "/test/runtime/dir/runtime2",
 			},
 			expectedConfig: map[string]interface{}{
 				"runtimes": map[string]interface{}{
@@ -141,11 +132,8 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 					},
 				},
 			},
-			runtimes: map[string]interface{}{
-				"runtime1": map[string]interface{}{
-					"path": "/test/runtime/dir/runtime1",
-					"args": []string{},
-				},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
 			},
 			expectedConfig: map[string]interface{}{
 				"runtimes": map[string]interface{}{
@@ -169,11 +157,8 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 				},
 				"storage-driver": "overlay2",
 			},
-			runtimes: map[string]interface{}{
-				"runtime1": map[string]interface{}{
-					"path": "/test/runtime/dir/runtime1",
-					"args": []string{},
-				},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
 			},
 			expectedConfig: map[string]interface{}{
 				"exec-opts":  []string{"native.cgroupdriver=systemd"},
@@ -212,8 +197,10 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			err := UpdateConfig(tc.config, "", tc.runtimes)
-			require.NoError(t, err)
+			for runtimeName, runtimePath := range tc.runtimes {
+				err := UpdateConfig(tc.config, runtimeName, runtimePath, false)
+				require.NoError(t, err)
+			}
 
 			configContent, err := json.MarshalIndent(tc.config, "", "    ")
 			require.NoError(t, err)

internal/config/image/capabilities.go

@@ -0,0 +1,54 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+// DriverCapability represents the possible values of NVIDIA_DRIVER_CAPABILITIES
+type DriverCapability string
+
+// Constants for the supported driver capabilities
+const (
+	DriverCapabilityAll      DriverCapability = "all"
+	DriverCapabilityCompat32 DriverCapability = "compat32"
+	DriverCapabilityCompute  DriverCapability = "compute"
+	DriverCapabilityDisplay  DriverCapability = "display"
+	DriverCapabilityGraphics DriverCapability = "graphics"
+	DriverCapabilityNgx      DriverCapability = "ngx"
+	DriverCapabilityUtility  DriverCapability = "utility"
+	DriverCapabilityVideo    DriverCapability = "video"
+)
+
+// DriverCapabilities represents the NVIDIA_DRIVER_CAPABILITIES set for the specified image.
+type DriverCapabilities map[DriverCapability]bool
+
+// Has check whether the specified capability is selected.
+func (c DriverCapabilities) Has(capability DriverCapability) bool {
+	if c[DriverCapabilityAll] {
+		return true
+	}
+	return c[capability]
+}
+
+// Any checks whether any of the specified capabilites are set
+func (c DriverCapabilities) Any(capabilities ...DriverCapability) bool {
+	for _, cap := range capabilities {
+		if c.Has(cap) {
+			return true
+		}
+	}
+
+	return false
+}

internal/config/image/cuda_image.go

@@ -26,11 +26,12 @@ import (
 )
 
 const (
-	envCUDAVersion      = "CUDA_VERSION"
-	envNVRequirePrefix  = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA    = envNVRequirePrefix + "CUDA"
-	envNVRequireJetpack = envNVRequirePrefix + "JETPACK"
-	envNVDisableRequire = "NVIDIA_DISABLE_REQUIRE"
+	envCUDAVersion          = "CUDA_VERSION"
+	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
+	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
+	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
 // CUDA represents a CUDA image that can be used for GPU computing. This wraps
@@ -113,33 +114,48 @@ func (i CUDA) HasDisableRequire() bool {
 }
 
 // DevicesFromEnvvars returns the devices requested by the image through environment variables
-func (i CUDA) DevicesFromEnvvars(envVars ...string) []string {
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
+func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+	// We concantenate all the devices from the specified envvars.
+	var isSet bool
+	var devices []string
+	requested := make(map[string]bool)
 	for _, envVar := range envVars {
 		if devs, ok := i[envVar]; ok {
-			devices = &devs
-			break
+			isSet = true
+			for _, d := range strings.Split(devs, ",") {
+				trimmed := strings.TrimSpace(d)
+				if len(trimmed) == 0 {
+					continue
+				}
+				devices = append(devices, trimmed)
+				requested[trimmed] = true
+			}
 		}
 	}
 
 	// Environment variable unset with legacy image: default to "all".
-	if devices == nil && i.IsLegacy() {
-		return []string{"all"}
+	if !isSet && len(devices) == 0 && i.IsLegacy() {
+		return NewVisibleDevices("all")
 	}
 
 	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
-		return nil
+	if len(devices) == 0 || requested["void"] {
+		return NewVisibleDevices("void")
 	}
 
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		return []string{""}
+	return NewVisibleDevices(devices...)
+}
+
+// GetDriverCapabilities returns the requested driver capabilities.
+func (i CUDA) GetDriverCapabilities() DriverCapabilities {
+	env := i[envNVDriverCapabilities]
+
+	capabilites := make(DriverCapabilities)
+	for _, c := range strings.Split(env, ",") {
+		capabilites[DriverCapability(c)] = true
 	}
 
-	return strings.Split(*devices, ",")
+	return capabilites
 }
 
 func (i CUDA) legacyVersion() (string, error) {

internal/config/image/devices.go

@@ -0,0 +1,127 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"strings"
+)
+
+// VisibleDevices represents the devices selected in a container image
+// through the NVIDIA_VISIBLE_DEVICES or other environment variables
+type VisibleDevices interface {
+	List() []string
+	Has(string) bool
+}
+
+var _ VisibleDevices = (*all)(nil)
+var _ VisibleDevices = (*none)(nil)
+var _ VisibleDevices = (*void)(nil)
+var _ VisibleDevices = (*devices)(nil)
+
+// NewVisibleDevices creates a VisibleDevices based on the value of the specified envvar.
+func NewVisibleDevices(envvars ...string) VisibleDevices {
+	for _, envvar := range envvars {
+		if envvar == "all" {
+			return all{}
+		}
+		if envvar == "none" {
+			return none{}
+		}
+		if envvar == "" || envvar == "void" {
+			return void{}
+		}
+	}
+
+	return newDevices(envvars...)
+}
+
+type all struct{}
+
+// List returns ["all"] for all devices
+func (a all) List() []string {
+	return []string{"all"}
+}
+
+// Has for all devices is true for any id except the empty ID
+func (a all) Has(id string) bool {
+	return id != ""
+}
+
+type none struct{}
+
+// List returns [""] for the none devices
+func (n none) List() []string {
+	return []string{""}
+}
+
+// Has for none devices is false for any id
+func (n none) Has(id string) bool {
+	return false
+}
+
+type void struct {
+	none
+}
+
+// List returns nil for the void devices
+func (v void) List() []string {
+	return nil
+}
+
+type devices struct {
+	len    int
+	lookup map[string]int
+}
+
+func newDevices(idOrCommaSeparated ...string) devices {
+	lookup := make(map[string]int)
+
+	i := 0
+	for _, commaSeparated := range idOrCommaSeparated {
+		for _, id := range strings.Split(commaSeparated, ",") {
+			lookup[id] = i
+			i++
+		}
+	}
+
+	d := devices{
+		len:    i,
+		lookup: lookup,
+	}
+	return d
+}
+
+// List returns the list of requested devices
+func (d devices) List() []string {
+	list := make([]string, d.len)
+
+	for id, i := range d.lookup {
+		list[i] = id
+	}
+
+	return list
+}
+
+// Has checks whether the specified ID is in the set of requested devices
+func (d devices) Has(id string) bool {
+	if id == "" {
+		return false
+	}
+
+	_, exist := d.lookup[id]
+	return exist
+}

internal/discover/filter.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/sirupsen/logrus"
+
+// Filter defines an interface for filtering discovered entities
+type Filter interface {
+	DeviceIsSelected(device Device) bool
+}
+
+// filtered represents a filtered discoverer
+type filtered struct {
+	Discover
+	logger *logrus.Logger
+	filter Filter
+}
+
+// newFilteredDisoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
+func newFilteredDisoverer(logger *logrus.Logger, applyTo Discover, filter Filter) Discover {
+	return filtered{
+		Discover: applyTo,
+		logger:   logger,
+		filter:   filter,
+	}
+}
+
+// Devices returns a filtered list of devices based on the specified filter.
+func (d filtered) Devices() ([]Device, error) {
+	devices, err := d.Discover.Devices()
+	if err != nil {
+		return nil, err
+	}
+
+	if d.filter == nil {
+		return devices, nil
+	}
+
+	var selected []Device
+	for _, device := range devices {
+		if d.filter.DeviceIsSelected(device) {
+			selected = append(selected, device)
+		}
+		d.logger.Debugf("skipping device %v", device)
+	}
+
+	return selected, nil
+}

internal/discover/gds.go

@@ -45,7 +45,10 @@ func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
 
 	cufile := NewMounts(
 		logger,
-		lookup.NewFileLocator(logger, root),
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
 		root,
 		[]string{"/etc/cufile.json"},
 	)

internal/discover/graphics.go

@@ -18,13 +18,43 @@ package discover
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/sirupsen/logrus"
 )
 
 // NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
-func NewGraphicsDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, cfg *Config) (Discover, error) {
+	root := cfg.Root
+
+	mounts, err := NewGraphicsMountsDiscoverer(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
+	}
+
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
+
+	discover := Merge(
+		Merge(drmDeviceNodes, drmByPathSymlinks),
+		mounts,
+	)
+
+	return discover, nil
+}
+
+// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
+func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
 	locator, err := lookup.NewLibraryLocator(logger, root)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct library locator: %v", err)
@@ -40,17 +70,18 @@ func NewGraphicsDiscoverer(logger *logrus.Logger, root string) (Discover, error)
 
 	jsonMounts := NewMounts(
 		logger,
-		lookup.NewFileLocator(logger, root),
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+			lookup.WithSearchPaths("/etc", "/usr/share"),
+		),
 		root,
 		[]string{
-			// TODO: We should handle this more cleanly
-			"/etc/glvnd/egl_vendor.d/10_nvidia.json",
-			"/etc/vulkan/icd.d/nvidia_icd.json",
-			"/etc/vulkan/implicit_layer.d/nvidia_layers.json",
-			"/usr/share/glvnd/egl_vendor.d/10_nvidia.json",
-			"/usr/share/vulkan/icd.d/nvidia_icd.json",
-			"/usr/share/vulkan/implicit_layer.d/nvidia_layers.json",
-			"/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"glvnd/egl_vendor.d/10_nvidia.json",
+			"vulkan/icd.d/nvidia_icd.json",
+			"vulkan/implicit_layer.d/nvidia_layers.json",
+			"egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"egl/egl_external_platform.d/10_nvidia_wayland.json",
 		},
 	)
 
@@ -61,3 +92,185 @@ func NewGraphicsDiscoverer(logger *logrus.Logger, root string) (Discover, error)
 
 	return discover, nil
 }
+
+type drmDevicesByPath struct {
+	None
+	logger                  *logrus.Logger
+	lookup                  lookup.Locator
+	nvidiaCTKExecutablePath string
+	root                    string
+	devicesFrom             Discover
+}
+
+// newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
+func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover {
+	d := drmDevicesByPath{
+		logger:                  logger,
+		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
+		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+		root:                    cfg.Root,
+		devicesFrom:             devices,
+	}
+
+	return &d
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d drmDevicesByPath) Hooks() ([]Hook, error) {
+	devices, err := d.devicesFrom.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices for by-path symlinks: %v", err)
+	}
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	links, err := d.getSpecificLinkArgs(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	hookPath := nvidiaCTKDefaultFilePath
+	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
+	if err != nil {
+		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
+	} else if len(targets) == 0 {
+		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
+	} else {
+		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
+		hookPath = targets[0]
+	}
+	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
+
+	args := []string{hookPath, "hook", "create-symlinks"}
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	h := Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      hookPath,
+		Args:      args,
+	}
+
+	return []Hook{h}, nil
+}
+
+// getSpecificLinkArgs returns the required specic links that need to be created
+func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error) {
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[filepath.Base(d.HostPath)] = true
+	}
+
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.root),
+	)
+	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
+	if err != nil {
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
+	}
+
+	var links []string
+	for _, c := range candidates {
+		device, err := os.Readlink(c)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", c)
+			continue
+		}
+
+		if selectedDevices[filepath.Base(device)] {
+			d.logger.Debugf("adding device symlink %v -> %v", c, device)
+			links = append(links, fmt.Sprintf("%v::%v", device, c))
+		}
+	}
+
+	return links, nil
+}
+
+// newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
+func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, root string) (Discover, error) {
+	allDevices := NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(logger, root),
+		root,
+		[]string{
+			"/dev/dri/card*",
+			"/dev/dri/renderD*",
+		},
+	)
+
+	filter, err := newDRMDeviceFilter(logger, devices, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
+	}
+
+	// We return a discoverer that applies the DRM device filter created above to all discovered DRM device nodes.
+	d := newFilteredDisoverer(
+		logger,
+		allDevices,
+		filter,
+	)
+
+	return d, err
+}
+
+// newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
+func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, root string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read GPU information: %v", err)
+	}
+
+	var selectedBusIds []string
+	for _, f := range gpuInformationPaths {
+		info, err := proc.ParseGPUInformationFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %v: %v", f, err)
+		}
+		uuid := info[proc.GPUInfoGPUUUID]
+		busID := info[proc.GPUInfoBusLocation]
+		minor := info[proc.GPUInfoDeviceMinor]
+
+		if devices.Has(minor) || devices.Has(uuid) || devices.Has(busID) {
+			selectedBusIds = append(selectedBusIds, busID)
+		}
+	}
+
+	filter := make(selectDeviceByPath)
+	for _, busID := range selectedBusIds {
+		drmDeviceNodes, err := drm.GetDeviceNodesByBusID(busID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", busID, err)
+		}
+		for _, drmDeviceNode := range drmDeviceNodes {
+			filter[filepath.Join(drmDeviceNode)] = true
+		}
+	}
+
+	return filter, nil
+}
+
+// selectDeviceByPath is a filter that allows devices to be selected by the path
+type selectDeviceByPath map[string]bool
+
+var _ Filter = (*selectDeviceByPath)(nil)
+
+// DeviceIsSelected determines whether the device's path has been selected
+func (s selectDeviceByPath) DeviceIsSelected(device Device) bool {
+	return s[device.Path]
+}
+
+// MountIsSelected is always true
+func (s selectDeviceByPath) MountIsSelected(Mount) bool {
+	return true
+}
+
+// HookIsSelected is always true
+func (s selectDeviceByPath) HookIsSelected(Hook) bool {
+	return true
+}

internal/discover/ldconfig.go

@@ -67,27 +67,43 @@ func (d ldconfig) Hooks() ([]Hook, error) {
 }
 
 // CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
-func CreateLDCacheUpdateHook(logger *logrus.Logger, lookup lookup.Locator, execuable string, defaultPath string, libraries []string) Hook {
+func CreateLDCacheUpdateHook(logger *logrus.Logger, lookup lookup.Locator, executable string, defaultPath string, libraries []string) Hook {
+	var args []string
+	for _, f := range uniqueFolders(libraries) {
+		args = append(args, "--folder", f)
+	}
+
+	hook := CreateNvidiaCTKHook(
+		logger,
+		lookup,
+		executable,
+		defaultPath,
+		"update-ldcache",
+		args...,
+	)
+
+	return hook
+
+}
+
+// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
+func CreateNvidiaCTKHook(logger *logrus.Logger, lookup lookup.Locator, executable string, defaultPath string, hookName string, additionalArgs ...string) Hook {
 	hookPath := defaultPath
-	targets, err := lookup.Locate(execuable)
+	targets, err := lookup.Locate(executable)
 	if err != nil {
-		logger.Warnf("Failed to locate %v: %v", execuable, err)
+		logger.Warnf("Failed to locate %v: %v", executable, err)
 	} else if len(targets) == 0 {
-		logger.Warnf("%v not found", execuable)
+		logger.Warnf("%v not found", executable)
 	} else {
-		logger.Debugf("Found %v candidates: %v", execuable, targets)
+		logger.Debugf("Found %v candidates: %v", executable, targets)
 		hookPath = targets[0]
 	}
 	logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
 
-	args := []string{filepath.Base(hookPath), "hook", "update-ldcache"}
-	for _, f := range uniqueFolders(libraries) {
-		args = append(args, "--folder", f)
-	}
 	return Hook{
 		Lifecycle: cdi.CreateContainerHook,
 		Path:      hookPath,
-		Args:      args,
+		Args:      append([]string{filepath.Base(hookPath), "hook", hookName}, additionalArgs...),
 	}
 }
 

internal/discover/symlinks.go

@@ -117,7 +117,7 @@ func (d symlinks) getSpecificLinkArgs() ([]string, error) {
 		}
 
 		linkPath := filepath.Join(filepath.Dir(m.Path), link)
-		links = append(links, "--link", fmt.Sprintf("%v:%v", target, linkPath))
+		links = append(links, "--link", fmt.Sprintf("%v::%v", target, linkPath))
 		linkProcessed[link] = true
 	}
 

internal/edits/device.go

@@ -17,13 +17,9 @@
 package edits
 
 import (
-	"fmt"
-
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
-
-	"github.com/opencontainers/runc/libcontainer/devices"
 )
 
 type device discover.Device
@@ -46,21 +42,18 @@ func (d device) toEdits() (*cdi.ContainerEdits, error) {
 // toSpec converts a discovered Device to a CDI Spec Device. Note
 // that missing info is filled in when edits are applied by querying the Device node.
 func (d device) toSpec() (*specs.DeviceNode, error) {
-	// NOTE: This mirrors what cri-o does.
-	// https://github.com/cri-o/cri-o/blob/ca3bb80a3dda0440659fcf8da8ed6f23211de94e/internal/config/device/device.go#L93
-	// This can be removed once https://github.com/container-orchestrated-devices/container-device-interface/issues/72 is addressed
-	dev, err := devices.DeviceFromPath(d.HostPath, "rwm")
-	if err != nil {
-		return nil, fmt.Errorf("failed to query device node %v: %v", d.HostPath, err)
+	// The HostPath field was added in the v0.5.0 CDI specification.
+	// The cdi package uses strict unmarshalling when loading specs from file causing failures for
+	// unexpected fields.
+	// Since the behaviour for HostPath == "" and HostPath == Path are equivalent, we clear HostPath
+	// if it is equal to Path to ensure compatibility with the widest range of specs.
+	hostPath := d.HostPath
+	if hostPath == d.Path {
+		hostPath = ""
 	}
 	s := specs.DeviceNode{
+		HostPath: hostPath,
 		Path:     d.Path,
-		Type:     string(dev.Type),
-		Major:    dev.Major,
-		Minor:    dev.Minor,
-		FileMode: &dev.FileMode,
-		UID:      &dev.Uid,
-		GID:      &dev.Gid,
 	}
 
 	return &s, nil

internal/edits/device_test.go

@@ -0,0 +1,69 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeviceToSpec(t *testing.T) {
+	testCases := []struct {
+		device   discover.Device
+		expected *specs.DeviceNode
+	}{
+		{
+			device: discover.Device{
+				Path: "/foo",
+			},
+			expected: &specs.DeviceNode{
+				Path: "/foo",
+			},
+		},
+		{
+			device: discover.Device{
+				Path:     "/foo",
+				HostPath: "/foo",
+			},
+			expected: &specs.DeviceNode{
+				Path: "/foo",
+			},
+		},
+		{
+			device: discover.Device{
+				Path:     "/foo",
+				HostPath: "/not/foo",
+			},
+			expected: &specs.DeviceNode{
+				Path:     "/foo",
+				HostPath: "/not/foo",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			spec, err := device(tc.device).toSpec()
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expected, spec)
+		})
+	}
+}

internal/edits/edits.go

@@ -22,6 +22,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	ociSpecs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
@@ -34,6 +35,20 @@ type edits struct {
 // NewSpecEdits creates a SpecModifier that defines the required OCI spec edits (as CDI ContainerEdits) from the specified
 // discoverer.
 func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	c, err := FromDiscoverer(d)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing container edits: %v", err)
+	}
+	e := edits{
+		ContainerEdits: *c,
+		logger:         logger,
+	}
+
+	return &e, nil
+}
+
+// FromDiscoverer creates CDI container edits for the specified discoverer.
+func FromDiscoverer(d discover.Discover) (*cdi.ContainerEdits, error) {
 	devices, err := d.Devices()
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover devices: %v", err)
@@ -49,7 +64,7 @@ func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier,
 		return nil, fmt.Errorf("failed to discover hooks: %v", err)
 	}
 
-	c := cdi.ContainerEdits{}
+	c := NewContainerEdits()
 	for _, d := range devices {
 		edits, err := device(d).toEdits()
 		if err != nil {
@@ -66,12 +81,15 @@ func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier,
 		c.Append(hook(h).toEdits())
 	}
 
-	e := edits{
-		ContainerEdits: c,
-		logger:         logger,
-	}
+	return c, nil
+}
 
-	return &e, nil
+// NewContainerEdits is a utility function to create a CDI ContainerEdits struct.
+func NewContainerEdits() *cdi.ContainerEdits {
+	c := cdi.ContainerEdits{
+		ContainerEdits: &specs.ContainerEdits{},
+	}
+	return &c
 }
 
 // Modify applies the defined edits to the incoming OCI spec

internal/edits/edits_test.go

@@ -0,0 +1,31 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFromDiscovererAllowsMountsToIterate(t *testing.T) {
+	edits, err := FromDiscoverer(discover.None{})
+	require.NoError(t, err)
+
+	require.Empty(t, edits.Mounts)
+}

internal/info/drm/drm_devices.go

@@ -0,0 +1,39 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package drm
+
+import (
+	"fmt"
+	"path/filepath"
+)
+
+// GetDeviceNodesByBusID returns the DRM devices associated with the specified PCI bus ID
+func GetDeviceNodesByBusID(busID string) ([]string, error) {
+	drmRoot := filepath.Join("/sys/bus/pci/devices", busID, "drm")
+	matches, err := filepath.Glob(fmt.Sprintf("%s/*", drmRoot))
+	if err != nil {
+		return nil, err
+	}
+
+	var drmDeviceNodes []string
+	for _, m := range matches {
+		drmDeviceNode := filepath.Join("/dev/dri", filepath.Base(m))
+		drmDeviceNodes = append(drmDeviceNodes, drmDeviceNode)
+	}
+
+	return drmDeviceNodes, nil
+}

internal/info/proc/information_files.go

@@ -0,0 +1,89 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package proc
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// GPUInfoField represents the field name for information specified in a GPU's information file
+type GPUInfoField string
+
+// The following constants define the fields of interest from the GPU information file
+const (
+	GPUInfoModel       = GPUInfoField("Model")
+	GPUInfoGPUUUID     = GPUInfoField("GPU UUID")
+	GPUInfoBusLocation = GPUInfoField("Bus Location")
+	GPUInfoDeviceMinor = GPUInfoField("Device Minor")
+)
+
+// GPUInfo stores the information for a GPU as determined from its associated information file
+type GPUInfo map[GPUInfoField]string
+
+// GetInformationFilePaths returns the list of information files associated with NVIDIA GPUs.
+func GetInformationFilePaths(root string) ([]string, error) {
+	return filepath.Glob(filepath.Join(root, "/proc/driver/nvidia/gpus/*/information"))
+}
+
+// ParseGPUInformationFile parses the specified GPU information file and constructs a GPUInfo structure
+func ParseGPUInformationFile(path string) (GPUInfo, error) {
+	infoFile, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %v: %v", path, err)
+	}
+	defer infoFile.Close()
+
+	return gpuInfoFrom(infoFile), nil
+}
+
+// gpuInfoFrom parses a GPUInfo struct from the specified reader
+// An information file has the following strucutre:
+// $ cat /proc/driver/nvidia/gpus/0000\:06\:00.0/information
+// Model:           Tesla V100-SXM2-16GB
+// IRQ:             408
+// GPU UUID:        GPU-edfee158-11c1-52b8-0517-92f30e7fac88
+// Video BIOS:      88.00.41.00.01
+// Bus Type:        PCIe
+// DMA Size:        47 bits
+// DMA Mask:        0x7fffffffffff
+// Bus Location:    0000:06:00.0
+// Device Minor:    0
+// GPU Excluded:    No
+func gpuInfoFrom(reader io.Reader) GPUInfo {
+	info := make(GPUInfo)
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		parts := strings.SplitN(line, ":", 2)
+		if len(parts) != 2 {
+			continue
+		}
+
+		field := GPUInfoField(parts[0])
+		value := strings.TrimSpace(parts[1])
+
+		info[field] = value
+	}
+
+	return info
+}

internal/lookup/device.go

@@ -19,7 +19,6 @@ package lookup
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/sirupsen/logrus"
 )
@@ -31,13 +30,12 @@ const (
 // NewCharDeviceLocator creates a Locator that can be used to find char devices at the specified root. A logger is
 // also specified.
 func NewCharDeviceLocator(logger *logrus.Logger, root string) Locator {
-	l := file{
-		logger:   logger,
-		prefixes: []string{root, filepath.Join(root, devRoot)},
-		filter:   assertCharDevice,
-	}
-
-	return &l
+	return NewFileLocator(
+		WithLogger(logger),
+		WithRoot(root),
+		WithSearchPaths("", devRoot),
+		WithFilter(assertCharDevice),
+	)
 }
 
 // assertCharDevice checks whether the specified path is a char device and returns an error if this is not the case.

internal/lookup/device_test.go

@@ -0,0 +1,55 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCharDeviceLocator(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		root             string
+		expectedPrefixes []string
+	}{
+		{
+			root:             "",
+			expectedPrefixes: []string{"", "/dev"},
+		},
+		{
+			root:             "/",
+			expectedPrefixes: []string{"/", "/dev"},
+		},
+		{
+			root:             "/some/root",
+			expectedPrefixes: []string{"/some/root", "/some/root/dev"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			f := NewCharDeviceLocator(logger, tc.root).(*file)
+
+			require.EqualValues(t, tc.expectedPrefixes, f.prefixes)
+		})
+	}
+}

internal/lookup/dir.go

@@ -26,13 +26,11 @@ import (
 // NewDirectoryLocator creates a Locator that can be used to find directories at the specified root. A logger
 // is also specified.
 func NewDirectoryLocator(logger *log.Logger, root string) Locator {
-	l := file{
-		logger:   logger,
-		prefixes: []string{root},
-		filter:   assertDirectory,
-	}
-
-	return &l
+	return NewFileLocator(
+		WithLogger(logger),
+		WithRoot(root),
+		WithFilter(assertDirectory),
+	)
 }
 
 // assertDirectory checks wither the specified path is a directory.

internal/lookup/executable.go

@@ -19,7 +19,6 @@ package lookup
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -33,17 +32,21 @@ type executable struct {
 func NewExecutableLocator(logger *log.Logger, root string) Locator {
 	paths := GetPaths(root)
 
-	var prefixes []string
-	for _, dir := range paths {
-		prefixes = append(prefixes, filepath.Join(root, dir))
-	}
+	return newExecutableLocator(logger, root, paths...)
+}
+
+func newExecutableLocator(logger *log.Logger, root string, paths ...string) *executable {
+	f := newFileLocator(
+		WithLogger(logger),
+		WithRoot(root),
+		WithSearchPaths(paths...),
+		WithFilter(assertExecutable),
+	)
+
 	l := executable{
-		file: file{
-			logger:   logger,
-			prefixes: prefixes,
-			filter:   assertExecutable,
-		},
+		file: *f,
 	}
+
 	return &l
 }
 

internal/lookup/executable_test.go

@@ -0,0 +1,77 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecutableLocator(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		root             string
+		paths            []string
+		expectedPrefixes []string
+	}{
+		{
+			root:             "",
+			expectedPrefixes: []string{""},
+		},
+		{
+			root:             "",
+			paths:            []string{"/"},
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "",
+			paths:            []string{"/", "/bin"},
+			expectedPrefixes: []string{"/", "/bin"},
+		},
+		{
+			root:             "/",
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "/",
+			paths:            []string{"/"},
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "/",
+			paths:            []string{"/", "/bin"},
+			expectedPrefixes: []string{"/", "/bin"},
+		},
+		{
+			root:             "/some/path",
+			paths:            []string{"/", "/bin"},
+			expectedPrefixes: []string{"/some/path", "/some/path/bin"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			e := newExecutableLocator(logger, tc.root, tc.paths...)
+
+			require.EqualValues(t, tc.expectedPrefixes, e.prefixes)
+		})
+	}
+}

internal/lookup/file.go

@@ -28,26 +28,98 @@ import (
 // prefixes. The validity of a file is determined by a filter function.
 type file struct {
 	logger   *log.Logger
+	root     string
 	prefixes []string
 	filter   func(string) error
 }
 
-// NewFileLocator creates a Locator that can be used to find files at the specified root. A logger
-// can also be specified.
-func NewFileLocator(logger *log.Logger, root string) Locator {
-	l := newFileLocator(logger, root)
+// Option defines a function for passing options to the NewFileLocator() call
+type Option func(*file)
 
-	return &l
+// WithRoot sets the root for the file locator
+func WithRoot(root string) Option {
+	return func(f *file) {
+		f.root = root
+	}
 }
 
-func newFileLocator(logger *log.Logger, root string) file {
-	return file{
-		logger:   logger,
-		prefixes: []string{root},
-		filter:   assertFile,
+// WithLogger sets the logger for the file locator
+func WithLogger(logger *log.Logger) Option {
+	return func(f *file) {
+		f.logger = logger
 	}
 }
 
+// WithSearchPaths sets the search paths for the file locator.
+func WithSearchPaths(paths ...string) Option {
+	return func(f *file) {
+		f.prefixes = paths
+	}
+}
+
+// WithFilter sets the filter for the file locator
+// The filter is called for each candidate file and candidates that return nil are considered.
+func WithFilter(assert func(string) error) Option {
+	return func(f *file) {
+		f.filter = assert
+	}
+}
+
+// NewFileLocator creates a Locator that can be used to find files with the specified options.
+func NewFileLocator(opts ...Option) Locator {
+	return newFileLocator(opts...)
+}
+
+func newFileLocator(opts ...Option) *file {
+	f := &file{}
+	for _, opt := range opts {
+		opt(f)
+	}
+	if f.logger == nil {
+		f.logger = log.StandardLogger()
+	}
+	if f.filter == nil {
+		f.filter = assertFile
+	}
+	// Since the `Locate` implementations rely on the root already being specified we update
+	// the prefixes to include the root.
+	f.prefixes = getSearchPrefixes(f.root, f.prefixes...)
+
+	return f
+}
+
+// getSearchPrefixes generates a list of unique paths to be searched by a file locator.
+//
+// For each of the unique prefixes <p> specified, the path <root><p> is searched, where <root> is the
+// specified root. If no prefixes are specified, <root> is returned as the only search prefix.
+//
+// Note that an empty root is equivalent to searching relative to the current working directory, and
+// if the root filesystem should be searched instead, root should be specified as "/" explicitly.
+//
+// Also, a prefix of "" forces the root to be included in returned set of paths. This means that if
+// the root in addition to another prefix must be searched the function should be called with:
+//
+//	getSearchPrefixes("/root", "", "another/path")
+//
+// and will result in the search paths []{"/root", "/root/another/path"} being returned.
+func getSearchPrefixes(root string, prefixes ...string) []string {
+	seen := make(map[string]bool)
+	var uniquePrefixes []string
+	for _, p := range prefixes {
+		if seen[p] {
+			continue
+		}
+		seen[p] = true
+		uniquePrefixes = append(uniquePrefixes, filepath.Join(root, p))
+	}
+
+	if len(uniquePrefixes) == 0 {
+		uniquePrefixes = append(uniquePrefixes, root)
+	}
+
+	return uniquePrefixes
+}
+
 var _ Locator = (*file)(nil)
 
 // Locate attempts to find files with names matching the specified pattern.

internal/lookup/file_test.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetSearchPrefixes(t *testing.T) {
+	testCases := []struct {
+		root             string
+		prefixes         []string
+		expectedPrefixes []string
+	}{
+		{
+			root:             "",
+			expectedPrefixes: []string{""},
+		},
+		{
+			root:             "/",
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "/some/root",
+			expectedPrefixes: []string{"/some/root"},
+		},
+		{
+			root:             "",
+			prefixes:         []string{"foo", "bar"},
+			expectedPrefixes: []string{"foo", "bar"},
+		},
+		{
+			root:             "/",
+			prefixes:         []string{"foo", "bar"},
+			expectedPrefixes: []string{"/foo", "/bar"},
+		},
+		{
+			root:             "/",
+			prefixes:         []string{"/foo", "/bar"},
+			expectedPrefixes: []string{"/foo", "/bar"},
+		},
+		{
+			root:             "/some/root",
+			prefixes:         []string{"foo", "bar"},
+			expectedPrefixes: []string{"/some/root/foo", "/some/root/bar"},
+		},
+		{
+			root:             "",
+			prefixes:         []string{"foo", "bar", "bar", "foo"},
+			expectedPrefixes: []string{"foo", "bar"},
+		},
+		{
+			root:             "/some/root",
+			prefixes:         []string{"foo", "bar", "foo", "bar"},
+			expectedPrefixes: []string{"/some/root/foo", "/some/root/bar"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			prefixes := getSearchPrefixes(tc.root, tc.prefixes...)
+			require.EqualValues(t, tc.expectedPrefixes, prefixes)
+		})
+	}
+}

internal/lookup/symlinks.go

@@ -35,8 +35,9 @@ type symlink struct {
 // NewSymlinkChainLocator creats a locator that can be used for locating files through symlinks.
 // A logger can also be specified.
 func NewSymlinkChainLocator(logger *logrus.Logger, root string) Locator {
+	f := newFileLocator(WithLogger(logger), WithRoot(root))
 	l := symlinkChain{
-		file: newFileLocator(logger, root),
+		file: *f,
 	}
 
 	return &l
@@ -45,8 +46,9 @@ func NewSymlinkChainLocator(logger *logrus.Logger, root string) Locator {
 // NewSymlinkLocator creats a locator that can be used for locating files through symlinks.
 // A logger can also be specified.
 func NewSymlinkLocator(logger *logrus.Logger, root string) Locator {
+	f := newFileLocator(WithLogger(logger), WithRoot(root))
 	l := symlink{
-		file: newFileLocator(logger, root),
+		file: *f,
 	}
 
 	return &l

internal/modifier/cdi.go

@@ -80,7 +80,7 @@ func getDevicesFromSpec(ociSpec oci.Spec) ([]string, error) {
 	}
 
 	uniqueDevices := make(map[string]struct{})
-	for _, name := range append(envDevices, annotationDevices...) {
+	for _, name := range append(envDevices.List(), annotationDevices...) {
 		if !cdi.IsQualifiedName(name) {
 			name = cdi.QualifiedName("nvidia.com", "gpu", name)
 		}

internal/modifier/csv.go

@@ -55,7 +55,7 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		return nil, err
 	}
 
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}

internal/modifier/discover_test.go

@@ -37,8 +37,10 @@ func TestDiscoverModifier(t *testing.T) {
 		expectedSpec  *specs.Spec
 	}{
 		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
+			description:  "empty discoverer does not modify spec",
+			spec:         &specs.Spec{},
+			discover:     &discover.DiscoverMock{},
+			expectedSpec: &specs.Spec{},
 		},
 		{
 			description: "failed hooks discoverer returns error",

internal/modifier/gds.go

@@ -43,7 +43,7 @@ func NewGDSModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		return nil, err
 	}
 
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}

internal/modifier/graphics.go

@@ -18,7 +18,6 @@ package modifier
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
@@ -40,28 +39,36 @@ func NewGraphicsModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.
 		return nil, err
 	}
 
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
-		logger.Infof("No modification required; no devices requested")
+	if required, reason := requiresGraphicsModifier(image); !required {
+		logger.Infof("No graphics modifier required: %v", reason)
 		return nil, nil
 	}
 
-	var hasGraphics bool
-	for _, c := range strings.Split(image["NVIDIA_DRIVER_CAPABILITIES"], ",") {
-		if c == "graphics" || c == "all" {
-			hasGraphics = true
-			break
-		}
+	config := &discover.Config{
+		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
+		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
 	}
-
-	if !hasGraphics {
-		logger.Debugf("Capability %q not selected", "graphics")
-		return nil, nil
-	}
-
-	d, err := discover.NewGraphicsDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root)
+	d, err := discover.NewGraphicsDiscoverer(
+		logger,
+		image.DevicesFromEnvvars(visibleDevicesEnvvar),
+		config,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct discoverer: %v", err)
 	}
 
 	return NewModifierFromDiscoverer(logger, d)
 }
+
+// requiresGraphicsModifier determines whether a graphics modifier is required.
+func requiresGraphicsModifier(cudaImage image.CUDA) (bool, string) {
+	if devices := cudaImage.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
+		return false, "no devices requested"
+	}
+
+	if !cudaImage.GetDriverCapabilities().Any(image.DriverCapabilityGraphics, image.DriverCapabilityDisplay) {
+		return false, "no required capabilities requested"
+	}
+
+	return true, ""
+}

internal/modifier/graphics_test.go

@@ -0,0 +1,96 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGraphicsModifier(t *testing.T) {
+	testCases := []struct {
+		description      string
+		cudaImage        image.CUDA
+		expectedRequired bool
+	}{
+		{
+			description: "empty image does not create modifier",
+		},
+		{
+			description: "devices with no capabilities does not create modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES": "all",
+			},
+		},
+		{
+			description: "devices with no non-graphics does not create modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES":     "all",
+				"NVIDIA_DRIVER_CAPABILITIES": "compute",
+			},
+		},
+		{
+			description: "devices with all capabilities creates modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES":     "all",
+				"NVIDIA_DRIVER_CAPABILITIES": "all",
+			},
+			expectedRequired: true,
+		},
+		{
+			description: "devices with graphics capability creates modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES":     "all",
+				"NVIDIA_DRIVER_CAPABILITIES": "graphics",
+			},
+			expectedRequired: true,
+		},
+		{
+			description: "devices with compute,graphics capability creates modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES":     "all",
+				"NVIDIA_DRIVER_CAPABILITIES": "compute,graphics",
+			},
+			expectedRequired: true,
+		},
+		{
+			description: "devices with display capability creates modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES":     "all",
+				"NVIDIA_DRIVER_CAPABILITIES": "display",
+			},
+			expectedRequired: true,
+		},
+		{
+			description: "devices with display,graphics capability creates modifier",
+			cudaImage: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES":     "all",
+				"NVIDIA_DRIVER_CAPABILITIES": "display,graphics",
+			},
+			expectedRequired: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			required, _ := requiresGraphicsModifier(tc.cudaImage)
+			require.EqualValues(t, tc.expectedRequired, required)
+		})
+	}
+}

internal/modifier/mofed.go

@@ -43,7 +43,7 @@ func NewMOFEDModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spe
 		return nil, err
 	}
 
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices) == 0 {
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}

internal/modifier/stable.go

@@ -17,11 +17,10 @@
 package modifier
 
 import (
-	"os"
-	"os/exec"
-	"path/filepath"
+	"fmt"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
@@ -44,21 +43,8 @@ type stableRuntimeModifier struct {
 // Modify applies the required modification to the incoming OCI spec, inserting the nvidia-container-runtime-hook
 // as a prestart hook.
 func (m stableRuntimeModifier) Modify(spec *specs.Spec) error {
-	path, err := exec.LookPath(config.NVIDIAContainerRuntimeHookExecutable)
-	if err != nil {
-		path = filepath.Join(config.DefaultExecutableDir, config.NVIDIAContainerRuntimeHookExecutable)
-		_, err = os.Stat(path)
-		if err != nil {
-			return err
-		}
-	}
-
-	m.logger.Infof("Using prestart hook path: %s", path)
-
-	args := []string{path}
-	if spec.Hooks == nil {
-		spec.Hooks = &specs.Hooks{}
-	} else if len(spec.Hooks.Prestart) != 0 {
+	// If an NVIDIA Container Runtime Hook already exists, we don't make any modifications to the spec.
+	if spec.Hooks != nil {
 		for _, hook := range spec.Hooks.Prestart {
 			if isNVIDIAContainerRuntimeHook(&hook) {
 				m.logger.Infof("Existing nvidia prestart hook (%v) found in OCI spec", hook.Path)
@@ -67,6 +53,21 @@ func (m stableRuntimeModifier) Modify(spec *specs.Spec) error {
 		}
 	}
 
+	// We create a locator and look for the NVIDIA Container Runtime Hook in the path.
+	candidates, err := lookup.NewExecutableLocator(m.logger, "").Locate(config.NVIDIAContainerRuntimeHookExecutable)
+	if err != nil {
+		return fmt.Errorf("failed to locate NVIDIA Container Runtime Hook: %v", err)
+	}
+	path := candidates[0]
+	if len(candidates) > 1 {
+		m.logger.Debugf("Using %v from multiple NVIDIA Container Runtime Hook candidates: %v", path, candidates)
+	}
+
+	m.logger.Infof("Using prestart hook path: %v", path)
+	args := []string{path}
+	if spec.Hooks == nil {
+		spec.Hooks = &specs.Hooks{}
+	}
 	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
 		Path: path,
 		Args: append(args, "prestart"),

internal/modifier/tegra.go

@@ -33,7 +33,7 @@ func NewTegraPlatformFiles(logger *logrus.Logger) (oci.SpecModifier, error) {
 
 	tegraSystemMounts := discover.NewMounts(
 		logger,
-		lookup.NewFileLocator(logger, ""),
+		lookup.NewFileLocator(lookup.WithLogger(logger)),
 		"",
 		[]string{
 			"/etc/nv_tegra_release",

internal/nvcaps/nvcaps.go

@@ -48,13 +48,13 @@ type MigCaps map[MigCap]MigMinor
 // NewGPUInstanceCap creates a MigCap for the specified MIG GPU instance.
 // A GPU instance is uniquely defined by the GPU minor number and GI instance ID.
 func NewGPUInstanceCap(gpu, gi int) MigCap {
-	return MigCap(fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/access", gpu, gi))
+	return MigCap(fmt.Sprintf("gpu%d/gi%d/access", gpu, gi))
 }
 
 // NewComputeInstanceCap creates a MigCap for the specified MIG Compute instance.
 // A GPU instance is uniquely defined by the GPU minor number, GI instance ID, and CI instance ID.
 func NewComputeInstanceCap(gpu, gi, ci int) MigCap {
-	return MigCap(fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/ci%d/access", gpu, gi, ci))
+	return MigCap(fmt.Sprintf("gpu%d/gi%d/ci%d/access", gpu, gi, ci))
 }
 
 // GetCapDevicePath returns the path to the cap device for the specified cap.

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -56,7 +56,7 @@ install -m 644 -t %{buildroot}/usr/share/containers/oci/hooks.d oci-nvidia-hook.
 ln -sf %{_bindir}/nvidia-container-runtime-hook %{_bindir}/nvidia-container-toolkit
 
 %postun
-rm -f %{_bindir}/nvidia-container-toolkit
+if [ -L %{_bindir}/nvidia-container-toolkit ] then; rm -f %{_bindir}/nvidia-container-toolkit; fi
 
 %files
 %license LICENSE

scripts/build-all-components.sh

@@ -21,14 +21,14 @@
 
 function assert_usage() {
     echo "Missing argument $1"
-    echo "$(basename ${BASH_SOURCE[0]}) TARGET"
+    echo "$(basename "${BASH_SOURCE[0]}") TARGET"
     exit 1
 }
 
 set -e
 
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
-PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+PROJECT_ROOT="$( cd "${SCRIPTS_DIR}"/.. && pwd )"
 
 if [[ $# -ne 1 ]]; then
     assert_usage "TARGET"
@@ -36,46 +36,51 @@ fi
 
 TARGET=$1
 
-: ${DIST_DIR:=${PROJECT_ROOT}/dist}
+: "${DIST_DIR:=${PROJECT_ROOT}/dist}"
 export DIST_DIR
 
 echo "Building ${TARGET} for all packages to ${DIST_DIR}"
 
-: ${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}
-: ${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}
-: ${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}
-: ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}
+: "${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}"
+: "${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}"
+: "${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}"
+: "${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}"
 
 
-${SCRIPTS_DIR}/get-component-versions.sh
+"${SCRIPTS_DIR}/get-component-versions.sh"
 
 # Build libnvidia-container
-make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET}
+make -C "${LIBNVIDIA_CONTAINER_ROOT}" -f mk/docker.mk "${TARGET}"
 
-if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} || -z ${LIBNVIDIA_CONTAINER_VERSION} ]]; then
+if [[ -z "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" || -z "${LIBNVIDIA_CONTAINER_VERSION}" ]]; then
 eval $(${SCRIPTS_DIR}/get-component-versions.sh)
 fi
 
 # Build nvidia-container-toolkit
-make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} \
+make -C "${NVIDIA_CONTAINER_TOOLKIT_ROOT}" \
     LIBNVIDIA_CONTAINER_VERSION="${LIBNVIDIA_CONTAINER_VERSION}" \
     LIBNVIDIA_CONTAINER_TAG="${LIBNVIDIA_CONTAINER_TAG}" \
-        ${TARGET}
+        "${TARGET}"
 
-# We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets
-# The LIB_TAG is also overridden to match the TOOLKIT_TAG.
-# Build nvidia-container-runtime
-make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} \
-    LIB_VERSION="${NVIDIA_CONTAINER_RUNTIME_VERSION}" \
-    LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-    TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
-    TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-        ${TARGET}
+if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
+    # We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets
+    # The LIB_TAG is also overridden to match the TOOLKIT_TAG.
+    # Build nvidia-container-runtime
+    make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} \
+        LIB_VERSION="${NVIDIA_CONTAINER_RUNTIME_VERSION}" \
+        LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+        TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+        TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+            ${TARGET}
 
-# Build nvidia-docker2
-make -C ${NVIDIA_DOCKER_ROOT} \
-    LIB_VERSION="${NVIDIA_DOCKER_VERSION}" \
-    LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-    TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
-    TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-        ${TARGET}
+    # Build nvidia-docker2
+    make -C ${NVIDIA_DOCKER_ROOT} \
+        LIB_VERSION="${NVIDIA_DOCKER_VERSION}" \
+        LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+        TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+        TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+            ${TARGET}
+
+else
+    echo "Skipping nvidia-container-runtime and nvidia-docker builds for release candidate"
+fi

scripts/build-packages.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@
 set -e
 
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
-PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
 
 # This list represents the distribution-architecture pairs that are actually published
 # to the relevant repositories. This targets forwarded to the build-all-components script
@@ -54,7 +53,7 @@ else
 fi
 
 echo "Updating components"
-${SCRIPTS_DIR}/update-components.sh
+"${SCRIPTS_DIR}/update-components.sh"
 if [[ -n $(git status -s third_party) && ${ALLOW_LOCAL_COMPONENT_CHANGES} != "true" ]]; then
     echo "ERROR: Building with local component changes."
     echo "Commit pending changes or rerun with ALLOW_LOCAL_COMPONENT_CHANGES='true'"
@@ -66,7 +65,7 @@ eval $(${SCRIPTS_DIR}/get-component-versions.sh)
 
 if [[ -n ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
 echo "Allowing mismatched versions for release candidate "
-: ${ALLOW_VERSION_MISMATCH:=true}
+: "${ALLOW_VERSION_MISMATCH:=true}"
 fi
 
 if [[ "${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" != "${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}" ]]; then
@@ -87,5 +86,5 @@ export NVIDIA_CONTAINER_RUNTIME_VERSION
 export NVIDIA_DOCKER_VERSION
 
 for target in ${targets[@]}; do
-    ${SCRIPTS_DIR}/build-all-components.sh ${target}
+    "${SCRIPTS_DIR}/build-all-components.sh" "${target}"
 done

scripts/extract-packages.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function assert_usage() {
+    echo "Incorrect arguments: $*" >&2
+    echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG DIST-ARCH" >&2
+    echo -e "\\tPACKAGE_IMAGE: container image holding packages [e.g. registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit]" >&2
+    echo -e "\\tPACKAGE_TAG: tag for container image holding packages. [e.g. 1a2b3c4-packaging]" >&2
+    echo -e "\\tDIST: The distribution." >&2
+    echo -e "\\tARCH: The architecture." >&2
+    exit 1
+}
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
+
+if [[ $# -ne 2 ]]; then
+    assert_usage "$@"
+fi
+
+PACKAGE_IMAGE=$1
+DISTARCH=$2
+DIST=${DISTARCH%-*}
+ARCH=${DISTARCH##*-}
+
+if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
+    echo "ERROR: Distro and Architecture must be specified." >&2
+    assert_usage "$@"
+fi
+
+# TODO: accept ARTIFACTS_DIR as a command-line argument
+: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
+
+# For release-candidates we skip certain packages.
+# For example, we don't release release candidates of nvidia-container-runtime and nvidia-docker2
+# since these only bump the nvidia-container-toolkit dependency.
+function skip-for-release-candidate() {
+    if [[ "${VERSION/rc./}" == "${VERSION}" ]]; then
+        return 1
+    fi
+
+    local package_name=$1
+    if [[ "${package_name/"nvidia-docker2"/}" != "${package_name}" ]]; then
+        return 0
+    fi
+    if [[ "${package_name/"nvidia-container-runtime"/}" != "${package_name}" ]]; then
+        return 0
+    fi
+    return 1
+}
+
+# extract-file copies a file from a specified image.
+# If regctl is available this is used, otherwise a docker container is run and the file is copied from
+# there.
+function copy-file() {
+    local image=$1
+    local path_in_image=$2
+    local path_on_host=$3
+    if command -v regctl; then
+        regctl image get-file "${image}" "${path_in_image}" "${path_on_host}"
+    else
+        # Note this will only work for destinations where the `path_on_host` is in `pwd`
+        docker run --rm \
+        -v "$(pwd):$(pwd)" \
+        -w "$(pwd)" \
+        -u "$(id -u):$(id -g)" \
+        --entrypoint="bash" \
+            "${image}" \
+            -c "cp ${path_in_image} ${path_on_host}"
+    fi
+}
+
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+
+# extract-all extracts all package for the specified dist-arch combination from the package image.
+# The manifest.txt file in the image is used to detemine the applicable files for the combination.
+# Files are extracted to ${ARTIFACTS_DIR}/artifacts/packages/${dist}/${arch}
+function extract-all() {
+    local dist=$1
+    local arch=$2
+
+    echo "Extracting packages for ${dist}-${arch} from ${PACKAGE_IMAGE}"
+
+    mkdir -p "${ARTIFACTS_DIR}"
+    copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
+
+    # Extract every file for the specified dist-arch combiniation in MANIFEST.txt
+    grep "/${dist}/${arch}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
+        package_name="$(basename "$f")"
+        # For release-candidates, we skip certain packages
+        if skip-for-release-candidate "${package_name}"; then
+            echo "Skipping $f for release-candidate ${VERSION}"
+            continue
+        fi
+        target="${ARTIFACTS_DIR}/packages/${dist}/${arch}/${package_name}"
+        mkdir -p "$(dirname "$target")"
+        copy-file "${PACKAGE_IMAGE}" "${f}" "${target}"
+    done
+}
+
+extract-all "${DIST}" "${ARCH}"

scripts/packages-sign-all.sh

@@ -9,7 +9,7 @@ set -x -e
 
 function deb-sign {
 	local last_found
-	for r in ${*}; do
+	for r in "$@"; do
 		if [ -f "./${r}" ]; then
 			last_found=${r}
 		fi
@@ -27,12 +27,12 @@ function deb-sign {
 			--no-emit-version \
 			--no-comments \
 			--personal-digest-preferences sha512 \
-			--local-user ${GPG_LOCAL_USER} \
+			--local-user "${GPG_LOCAL_USER}" \
 		> InRelease
 }
 
 function rpm-sign {
-	for r in ${*}; do
+	for r in "$@"; do
 		if [ -f "./${r}" ]; then
 			rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}"
 		fi
@@ -42,7 +42,7 @@ function rpm-sign {
 		--armor \
 		--no-emit-version \
 		--no-comments --personal-digest-preferences sha512 \
-		--local-user ${GPG_LOCAL_USER} \
+		--local-user "${GPG_LOCAL_USER}" \
 	repodata/repomd.xml
 }
 
@@ -83,15 +83,15 @@ function sign() {
 		return
 	fi
 
-	cd ${dst}
+	cd "${dst}"
 	if [[ -f "/etc/debian_version" ]]; then
-		[[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS}
+		[[ "${pkg_type}" == "deb" ]] && deb-sign ${ALL_DEBS}
 	else
-		[[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS}
+		[[ "${pkg_type}" == "rpm" ]] && rpm-sign ${ALL_RPMS}
 	fi
 	cd -
 }
 
 for target in ${TARGETS[@]}; do
-    sign ${target} $(pwd)
+    sign "${target}" "$(pwd)"
 done

scripts/release-kitmaker-artifactory.sh

@@ -0,0 +1,216 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function assert_usage() {
+cat >&2 << EOF
+Incorrect arguments: $*
+$(basename "${BASH_SOURCE[0]}") DIST-ARCH ARTIFACTORY_URL
+    DIST: The distribution.
+    ARCH: The architecture.
+    ARTIFACTORY_URL must contain repo path for package, including hostname.
+
+Environment Variables
+    ARTIFACTORY_TOKEN: must contain an auth token. [required]
+EOF
+    exit 1
+}
+
+set -e
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
+COMPONENT_NAME="nvidia-container-toolkit"
+
+if [[ $# -ne 2 ]]; then
+    assert_usage "$@"
+fi
+
+source "${SCRIPTS_DIR}"/utils.sh
+
+DISTARCH=$1
+DIST=${DISTARCH%-*}
+ARCH=${DISTARCH##*-}
+ARTIFACTORY_URL=$2
+
+CURL=${CURL:-curl}
+
+if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
+    echo "ERROR: Distro and Architecture must be specified." >&2
+    assert_usage "$@"
+fi
+
+# TODO: accept ARTIFACTS_DIR as a command-line argument
+: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
+
+if [[ ! -d "${ARTIFACTS_DIR}" ]]; then
+    echo "ERROR: ARTIFACTS_DIR does not exist." >&2
+    assert_usage "$@"
+fi
+
+if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
+    echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
+    assert_usage "$@"
+fi
+
+# TODO: accept KITMACKER_DIR as a command-line argument
+: "${KITMAKER_DIR="${PROJECT_ROOT}/artifacts/kitmaker"}"
+
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+
+# Returns the key=value property if the value isn't empty
+# Prepends with ";" if needed
+set_prop_value() {
+    local key=$1
+    local value=$2
+    if [ -n "${value}" ]; then
+        if [ -z "${PROPS}" ]; then
+            echo "${key}=${value}"
+        else
+            echo ";${key}=${value}"
+        fi
+    fi
+}
+
+process_props() {
+    local dist=$1
+    local arch=$2
+
+    PROPS+=$(set_prop_value "component_name" "${COMPONENT_NAME}")
+    PROPS+=$(set_prop_value "version" "${VERSION}")
+    PROPS+=$(set_prop_value "os" "${dist}")
+    PROPS+=$(set_prop_value "arch" "${arch}")
+    PROPS+=$(set_prop_value "platform" "${dist}-${arch}")
+    # TODO: Use `git describe` to get this information if it's not available.
+    PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
+    PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
+
+    # Gitlab variables to expose
+    for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
+        if [ -n "${!var}" ]; then
+            PROPS+=$(set_prop_value "${var}" "${!var}")
+        fi
+    done
+
+    echo "Applying properties: ${PROPS}"
+}
+
+## NOT USED:
+## can substitute this function place of upload_file to modify properties of
+## existing file instead of uploading files.
+# Sets the properties on a path
+# Relies on global variables: ARTIFACTORY_TOKEN, ARTIFACTORY_URL
+set_props() {
+    local dist="$1"
+    local arch="$2"
+    local kitmakerfilename="$3"
+
+    # extract the Artifactory hostname
+    artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
+    local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/${kitmakerfilename}"
+
+    local PROPS
+    process_props "${DIST}" "${ARCH}"
+
+    echo "Setting ${image_path} with properties: ${PROPS}"
+    if ! ${CURL} -fs -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
+        -X PUT \
+        "https://${artifactory_host}/artifactory/api/storage/${image_path}?properties=${PROPS}&recursive=0" ; then
+        echo "ERROR: set props failed: ${image_path}"
+        exit 1
+    fi
+}
+
+# Uploads file to ARTIFACTS_DIR/<os>/<arch>/<filename>
+# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_URL
+upload_file() {
+    local dist=$1
+    local arch=$2
+    local file=$3
+
+    # extract the Artifactory hostname
+    artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
+    # get base part of the ARTIFACTORY_URL without hostname
+    local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/$(basename ${file})"
+
+    local PROPS
+    process_props "${dist}" "${arch}"
+
+    if [ ! -r "${file}" ]; then
+        echo "ERROR: File not found or not readable: ${file}"
+        exit 1
+    fi
+
+    # Collect sum
+    SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
+
+    echo "Uploading ${image_path} from ${file}"
+    if ! ${CURL} -f \
+        -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
+        -H "X-Checksum-Sha1: ${SHA1_SUM}" \
+        ${file:+-T ${file}} -X PUT \
+        "https://${artifactory_host}/${image_path};${PROPS}" ;
+        then
+        echo "ERROR: upload file failed: ${file}"
+        exit 1
+    fi
+}
+
+function push-kitmaker-artifactory() {
+    local dist=$1
+    local arch=$2
+    local archive=$3
+
+    upload_file "${dist}" "${arch}" "${archive}"
+}
+
+# kitmakerize-distro creates a tar.gz archive for the specified dist-arch combination.
+# The archive is created at ${KITMAKER_DIR}/${name}.tar.gz (where ${name} is the third positional argument)
+function kitmakerize-distro() {
+    local dist="$1"
+    local arch="$2"
+    local archive="$3"
+
+    local name=$(basename "${archive%%.tar.gz}")
+    ## Copy packages into directory layout for .tar.gz
+    # TODO: make scratch_dir configurable
+    local scratch_dir="$(dirname ${archive})/.scratch/${name}"
+    local packages_dir="${scratch_dir}/.packages/"
+
+    mkdir -p "${packages_dir}"
+
+    # Copy the extracted files to the .packages directory so that a kitmaker file can be created.
+    source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
+    cp -r "${source}/"* "${packages_dir}/"
+
+    ## Tar up the directory structure created above
+    tar zcvf "${archive}" -C "${scratch_dir}/.." "${name}"
+    echo "Created: ${archive}"
+    ls -l "${archive}"
+    echo "With contents:"
+    tar -tzvf "${archive}"
+    echo ""
+
+    # Clean up the scratch directories:
+    rm -f "${scratch_dir}/.packages/"*
+    rmdir "${scratch_dir}/.packages"
+    rmdir "${scratch_dir}"
+}
+
+: "${VERSION=$({NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION})}"
+kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${VERSION}"
+kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz"
+kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}"
+push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}"

scripts/release-packages-artifactory.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dependencies:
+#   regctl
+#
+
+function assert_usage() {
+cat >&2 << EOF
+Incorrect arguments: $*
+$(basename "${BASH_SOURCE[0]}") DIST-ARCH
+    DIST: The distribution.
+    ARCH: The architecture.
+
+Environment Variables
+    ARTIFACTORY_TOKEN: must contain an auth token. [required]
+    LIB_TAG: optional package tag.
+    CI_COMMIT_REF_NAME: provided by CI/CD system.
+    CI_COMMIT_SHA: provided by CI/CD system.
+EOF
+    exit 1
+}
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
+
+source "${SCRIPTS_DIR}"/utils.sh
+
+if [[ $# -ne 1 ]]; then
+    assert_usage "$@"
+fi
+
+DISTARCH=$1
+ARTIFACTORY_PATH=$2
+DIST=${DISTARCH%-*}
+ARCH=${DISTARCH##*-}
+
+CURL=${CURL:-curl}
+
+if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
+    echo "ERROR: Distro and Architecture must be specified." >&2
+    assert_usage "$@"
+fi
+
+if [[ -z "${ARTIFACTORY_PATH}" ]]; then
+    echo "ERROR: Package repo must be specified." >&2
+    assert_usage "$@"
+fi
+
+if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
+    echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
+    assert_usage "$@"
+fi
+
+# TODO: accept PACKAGES_DIR as a command-line argument
+: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
+: "${PACKAGES_DIR="${ARTIFACTS_DIR}/packages"}"
+
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+
+# Returns the key=value property if the value isn't empty
+# Prepends with ";" if needed
+set_prop_value() {
+    local key=$1
+    local value=$2
+    if [ -n "${value}" ]; then
+        if [ -z "${PROPS}" ]; then
+            echo "${key}=${value}"
+        else
+            echo ";${key}=${value}"
+        fi
+    fi
+}
+
+process_props() {
+    local dist=$1
+    local arch=$2
+    local file=$3
+    local component_name="${file%%.*}"
+    component_name="${component_name%-*}"
+    local pkg_type="$(package_type $dist)"
+
+    ## Component owner is free to define these
+    # PROPS+=$(set_prop_value "version" "${VERSION}")
+    # PROPS+=$(set_prop_value "lws_version" "${LWS_VER}")
+    # PROPS+=$(set_prop_value "platform" "${DISTARCH}")
+
+    # TODO: Use `git describe` to get this information if it's not available.
+    PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
+    PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
+
+    # PROPS+=$(set_prop_value "category" "utils")
+    # PROPS+=$(set_prop_value "platform" "${DISTARCH}")
+
+    # Gitlab variables to expose
+    for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
+        if [ -n "${!var}" ]; then
+            PROPS+=$(set_prop_value "${var}" "${!var}")
+        fi
+    done
+
+    # We also set the package-specific properties to allow this to be used for other artifactory repositories
+    PROPS+=$(set_prop_value "${pkg_type}.distribution" "${dist}")
+    PROPS+=$(set_prop_value "${pkg_type}.architecture" "${arch}")
+    PROPS+=$(set_prop_value "${pkg_type}.component" "${component_name}")
+}
+
+# Uploads file ARTIFACTORY_PATH
+# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_PATH
+upload_file() {
+    local dist=$1
+    local arch=$2
+    local file=$3
+
+    # TODO: These should be set by envvars
+    local artifactory_host="urm.nvidia.com"
+    local artifactory_repo="$(get_artifactory_repository $dist)"
+
+    if [ ! -r "${file}" ]; then
+        echo "ERROR: File not found or not readable: ${file}"
+        exit 1
+    fi
+
+    local PROPS
+    process_props "${dist}" "${arch}" "${file}"
+
+    # Collect sum
+    SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
+
+    url="https://${artifactory_host}/artifactory/${artifactory_repo}/${dist}/${arch}/$(basename "${file}")"
+    # NOTE: The URL to set the properties through the API is:
+    # "https://${artifactory_host}/artifactory/api/storage/${artifactory_repo}/${dist}/${arch}/$(basename ${file})"
+
+    echo "Uploading ${file} to ${url}"
+    if ! ${CURL} -f \
+          -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
+          -H "X-Checksum-Sha1: ${SHA1_SUM}" \
+          ${file:+-T ${file}} -X PUT \
+            "${url};${PROPS}" ;
+        then
+            echo "ERROR: upload file failed: ${file}"
+            exit 1
+    fi
+}
+
+function push-artifactory() {
+    local dist="$1"
+    local arch="$2"
+
+    source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
+
+    find "${source}" -maxdepth 1 | while read -r f ; do
+        upload_file "$dist" "$arch" "$f"
+    done
+}
+
+# TODO: use this to adapt as a general purpose command-line tool
+# case "${COMMAND}" in
+#   set)
+#     set_props
+#     ;;
+#   upload)
+#     if [ -z "${UPLOAD_FILE}" ]; then
+#       echo "ERROR: Upload package filename must be set using -f"
+#       usage
+#     fi
+#
+#     upload_file
+#     ;;
+#   *)
+#     echo "ERROR: Invalid command ${COMMAND}"
+#     usage
+#     ;;
+# esac
+
+push-artifactory "${DIST}" "${ARCH}"

scripts/utils.sh

@@ -0,0 +1,36 @@
+
+# package_type returns the packaging type (deb or rpm) for the specfied distribution.
+# An error is returned if the ditribution is unsupported.
+function package_type() {
+    local pkg_type
+    case ${1} in
+    amazonlinux*) pkg_type=rpm
+        ;;
+    centos*) pkg_type=rpm
+        ;;
+    debian*) pkg_type=deb
+        ;;
+    fedora*) pkg_type=rpm
+        ;;
+    opensuse-leap*) pkg_type=rpm
+        ;;
+    ubuntu*) pkg_type=deb
+        ;;
+    *) exit 1
+        ;;
+    esac
+    echo "${pkg_type}"
+}
+
+function get_artifactory_repository() {
+    local pkg_type=$(package_type $1)
+
+    case ${pkg_type} in
+    deb) echo "sw-gpu-cloudnative-debian-local"
+        ;;
+    rpm) echo "sw-gpu-cloudnative-rpm-local"
+        ;;
+    *) echo "sw-gpu-cloudnative-generic-local"
+        ;;
+    esac
+}

tools/container/crio/crio.go

@@ -20,23 +20,52 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
+	"github.com/pelletier/go-toml"
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 )
 
 const (
+	restartModeSystemd = "systemd"
+	restartModeNone    = "none"
+
+	defaultConfigMode = "hook"
+
+	// Hook-based settings
 	defaultHooksDir     = "/usr/share/containers/oci/hooks.d"
 	defaultHookFilename = "oci-nvidia-hook.json"
+
+	// Config-based settings
+	defaultConfig        = "/etc/crio/crio.conf"
+	defaultRuntimeClass  = "nvidia"
+	defaultSetAsDefault  = true
+	defaultRestartMode   = restartModeSystemd
+	defaultHostRootMount = "/host"
 )
 
-var hooksDirFlag string
-var hookFilenameFlag string
-var tooklitDirArg string
+// options stores the configuration from the command linek or environment variables
+type options struct {
+	configMode string
+
+	hooksDir     string
+	hookFilename string
+	runtimeDir   string
+
+	config        string
+	runtimeClass  string
+	setAsDefault  bool
+	restartMode   string
+	hostRootMount string
+}
 
 func main() {
+	options := options{}
+
 	// Create the top-level CLI
 	c := cli.NewApp()
 	c.Name = "crio"
@@ -47,17 +76,22 @@ func main() {
 	// Create the 'setup' subcommand
 	setup := cli.Command{}
 	setup.Name = "setup"
-	setup.Usage = "Create the cri-o hook required to run NVIDIA GPU containers"
+	setup.Usage = "Configure cri-o for NVIDIA GPU containers"
 	setup.ArgsUsage = "<toolkit_dirname>"
-	setup.Action = Setup
-	setup.Before = ParseArgs
+	setup.Action = func(c *cli.Context) error {
+		return Setup(c, &options)
+	}
+	setup.Before = func(c *cli.Context) error {
+		return ParseArgs(c, &options)
+	}
 
 	// Create the 'cleanup' subcommand
 	cleanup := cli.Command{}
 	cleanup.Name = "cleanup"
-	cleanup.Usage = "Remove the NVIDIA cri-o hook"
-	cleanup.Action = Cleanup
-
+	cleanup.Usage = "Remove the NVIDIA-specific cri-o configuration"
+	cleanup.Action = func(c *cli.Context) error {
+		return Cleanup(c, &options)
+	}
 	// Register the subcommands with the top-level CLI
 	c.Commands = []*cli.Command{
 		&setup,
@@ -74,7 +108,7 @@ func main() {
 			Aliases:     []string{"d"},
 			Usage:       "path to the cri-o hooks directory",
 			Value:       defaultHooksDir,
-			Destination: &hooksDirFlag,
+			Destination: &options.hooksDir,
 			EnvVars:     []string{"CRIO_HOOKS_DIR"},
 			DefaultText: defaultHooksDir,
 		},
@@ -83,10 +117,54 @@ func main() {
 			Aliases:     []string{"f"},
 			Usage:       "filename of the cri-o hook that will be created / removed in the hooks directory",
 			Value:       defaultHookFilename,
-			Destination: &hookFilenameFlag,
+			Destination: &options.hookFilename,
 			EnvVars:     []string{"CRIO_HOOK_FILENAME"},
 			DefaultText: defaultHookFilename,
 		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the configuration mode to use. One of [hook | config]",
+			Value:       defaultConfigMode,
+			Destination: &options.configMode,
+			EnvVars:     []string{"CRIO_CONFIG_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "Path to the cri-o config file",
+			Value:       defaultConfig,
+			Destination: &options.config,
+			EnvVars:     []string{"CRIO_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-class",
+			Usage:       "The name of the runtime class to set for the nvidia-container-runtime",
+			Value:       defaultRuntimeClass,
+			Destination: &options.runtimeClass,
+			EnvVars:     []string{"CRIO_RUNTIME_CLASS"},
+		},
+		// The flags below are only used by the 'setup' command.
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Usage:       "Set nvidia-container-runtime as the default runtime",
+			Value:       defaultSetAsDefault,
+			Destination: &options.setAsDefault,
+			EnvVars:     []string{"CRIO_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+		&cli.StringFlag{
+			Name:        "restart-mode",
+			Usage:       "Specify how cri-o should be restarted;  If 'none' is selected, it will not be restarted [systemd | none]",
+			Value:       defaultRestartMode,
+			Destination: &options.restartMode,
+			EnvVars:     []string{"CRIO_RESTART_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "Specify the path to the host root to be used when restarting crio using systemd",
+			Value:       defaultHostRootMount,
+			Destination: &options.hostRootMount,
+			EnvVars:     []string{"HOST_ROOT_MOUNT"},
+		},
 	}
 
 	// Update the subcommand flags with the common subcommand flags
@@ -100,16 +178,30 @@ func main() {
 }
 
 // Setup installs the prestart hook required to launch GPU-enabled containers
-func Setup(c *cli.Context) error {
+func Setup(c *cli.Context, o *options) error {
 	log.Infof("Starting 'setup' for %v", c.App.Name)
 
-	err := os.MkdirAll(hooksDirFlag, 0755)
+	switch o.configMode {
+	case "hook":
+		return setupHook(o)
+	case "config":
+		return setupConfig(o)
+	default:
+		return fmt.Errorf("invalid config-mode '%v'", o.configMode)
+	}
+}
+
+// setupHook installs the prestart hook required to launch GPU-enabled containers
+func setupHook(o *options) error {
+	log.Infof("Installing prestart hook")
+
+	err := os.MkdirAll(o.hooksDir, 0755)
 	if err != nil {
-		return fmt.Errorf("error creating hooks directory %v: %v", hooksDirFlag, err)
+		return fmt.Errorf("error creating hooks directory %v: %v", o.hooksDir, err)
 	}
 
-	hookPath := getHookPath(hooksDirFlag, hookFilenameFlag)
-	err = createHook(tooklitDirArg, hookPath)
+	hookPath := getHookPath(o.hooksDir, o.hookFilename)
+	err = createHook(o.runtimeDir, hookPath)
 	if err != nil {
 		return fmt.Errorf("error creating hook: %v", err)
 	}
@@ -117,11 +209,52 @@ func Setup(c *cli.Context) error {
 	return nil
 }
 
+// setupConfig updates the cri-o config for the NVIDIA container runtime
+func setupConfig(o *options) error {
+	log.Infof("Updating config file")
+
+	cfg, err := crio.LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = UpdateConfig(cfg, o)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = crio.FlushConfig(o.config, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = RestartCrio(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart crio: %v", err)
+	}
+
+	return nil
+}
+
 // Cleanup removes the specified prestart hook
-func Cleanup(c *cli.Context) error {
+func Cleanup(c *cli.Context, o *options) error {
 	log.Infof("Starting 'cleanup' for %v", c.App.Name)
 
-	hookPath := getHookPath(hooksDirFlag, hookFilenameFlag)
+	switch o.configMode {
+	case "hook":
+		return cleanupHook(o)
+	case "config":
+		return cleanupConfig(o)
+	default:
+		return fmt.Errorf("invalid config-mode '%v'", o.configMode)
+	}
+}
+
+// cleanupHook removes the prestart hook
+func cleanupHook(o *options) error {
+	log.Infof("Removing prestart hook")
+
+	hookPath := getHookPath(o.hooksDir, o.hookFilename)
 	err := os.Remove(hookPath)
 	if err != nil {
 		return fmt.Errorf("error removing hook '%v': %v", hookPath, err)
@@ -130,15 +263,42 @@ func Cleanup(c *cli.Context) error {
 	return nil
 }
 
+// cleanupConfig removes the NVIDIA container runtime from the cri-o config
+func cleanupConfig(o *options) error {
+	log.Infof("Reverting config file modifications")
+
+	cfg, err := crio.LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = RevertConfig(cfg, o)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = crio.FlushConfig(o.config, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = RestartCrio(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart crio: %v", err)
+	}
+
+	return nil
+}
+
 // ParseArgs parses the command line arguments to the CLI
-func ParseArgs(c *cli.Context) error {
+func ParseArgs(c *cli.Context, o *options) error {
 	args := c.Args()
 
 	log.Infof("Parsing arguments: %v", args.Slice())
 	if c.NArg() != 1 {
 		return fmt.Errorf("incorrect number of arguments")
 	}
-	tooklitDirArg = args.Get(0)
+	o.runtimeDir = args.Get(0)
 	log.Infof("Successfully parsed arguments")
 
 	return nil
@@ -152,7 +312,7 @@ func createHook(toolkitDir string, hookPath string) error {
 	defer hook.Close()
 
 	encoder := json.NewEncoder(hook)
-	err = encoder.Encode(generateOciHook(tooklitDirArg))
+	err = encoder.Encode(generateOciHook(toolkitDir))
 	if err != nil {
 		return fmt.Errorf("error writing hook file '%v': %v", hookPath, err)
 	}
@@ -183,3 +343,45 @@ func generateOciHook(toolkitDir string) podmanHook {
 	}
 	return hook
 }
+
+// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
+func UpdateConfig(config *toml.Tree, o *options) error {
+	runtimePath := filepath.Join(o.runtimeDir, "nvidia-container-runtime")
+	return crio.UpdateConfig(config, o.runtimeClass, runtimePath, o.setAsDefault)
+}
+
+// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
+func RevertConfig(config *toml.Tree, o *options) error {
+	return crio.RevertConfig(config, o.runtimeClass)
+}
+
+// RestartCrio restarts crio depending on the value of restartModeFlag
+func RestartCrio(o *options) error {
+	switch o.restartMode {
+	case restartModeNone:
+		log.Warnf("Skipping restart of crio due to --restart-mode=%v", o.restartMode)
+		return nil
+	case restartModeSystemd:
+		return RestartCrioSystemd(o.hostRootMount)
+	default:
+		return fmt.Errorf("invalid restart mode specified: %v", o.restartMode)
+	}
+}
+
+// RestartCrioSystemd restarts cri-o using systemctl
+func RestartCrioSystemd(hostRootMount string) error {
+	log.Infof("Restarting cri-o using systemd and host root mounted at %v", hostRootMount)
+
+	command := "chroot"
+	args := []string{hostRootMount, "systemctl", "restart", "crio"}
+
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error restarting crio using systemd: %v", err)
+	}
+
+	return nil
+}

tools/container/docker/docker.go

@@ -250,10 +250,15 @@ func LoadConfig(config string) (map[string]interface{}, error) {
 
 // UpdateConfig updates the docker config to include the nvidia runtimes
 func UpdateConfig(config map[string]interface{}, o *options) error {
-	defaultRuntime := o.getDefaultRuntime()
-	runtimes := o.runtimes()
+	for runtimeName, runtimePath := range o.getRuntimeBinaries() {
+		setAsDefault := runtimeName == o.getDefaultRuntime()
+		err := docker.UpdateConfig(config, runtimeName, runtimePath, setAsDefault)
+		if err != nil {
+			return fmt.Errorf("failed to update runtime %q: %v", runtimeName, err)
+		}
+	}
 
-	return docker.UpdateConfig(config, defaultRuntime, runtimes)
+	return nil
 }
 
 //RevertConfig reverts the docker config to remove the nvidia runtime
@@ -392,19 +397,6 @@ func (o options) getDefaultRuntime() string {
 	return o.runtimeName
 }
 
-// runtimes returns the docker runtime definitions for the supported nvidia runtimes
-// for the given options. This includes the path with the options runtimeDir applied
-func (o options) runtimes() map[string]interface{} {
-	runtimes := make(map[string]interface{})
-	for r, bin := range o.getRuntimeBinaries() {
-		runtimes[r] = map[string]interface{}{
-			"path": bin,
-			"args": []string{},
-		}
-	}
-	return runtimes
-}
-
 // getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
 // renaming of the `nvidia` runtime as per the --runtime-class command line flag.
 func (o options) getRuntimeBinaries() map[string]string {

versions.mk

@@ -14,7 +14,7 @@
 
 LIB_NAME := nvidia-container-toolkit
 LIB_VERSION := 1.12.0
-LIB_TAG := rc.1
+LIB_TAG := rc.3
 
 # Specify the nvidia-docker2 and nvidia-container-runtime package versions.
 # Note: The build tooling uses `LIB_TAG` above as the version tag.
@@ -26,6 +26,6 @@ NVIDIA_CONTAINER_RUNTIME_VERSION := 3.12.0
 LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack
 
 CUDA_VERSION := 11.8.0
-GOLANG_VERSION := 1.17.8
+GOLANG_VERSION := 1.18.8
 
 GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "")