Remove NVIDIA experimental runtime from toolkit container

Signed-off-by: Evan Lezar <elezar@nvidia.com>

CHANGELOG.md

@@ -19,6 +19,7 @@
 * [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
 
 * [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
 
 ## v1.13.1
 

build/container/Dockerfile.centos

@@ -26,11 +26,6 @@ FROM golang:${GOLANG_VERSION} as build
 # /artifacts/bin
 ARG GOPATH=/artifacts
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
-
 WORKDIR /build
 COPY . .
 

build/container/Dockerfile.ubuntu

@@ -26,11 +26,6 @@ FROM golang:${GOLANG_VERSION} as build
 # /artifacts/bin
 ARG GOPATH=/artifacts
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
-
 WORKDIR /build
 COPY . .
 

internal/config/config_test.go

@@ -96,7 +96,6 @@ func TestGetConfig(t *testing.T) {
 				"nvidia-container-cli.load-kmods = false",
 				"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
-				"nvidia-container-runtime.experimental = true",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
@@ -150,7 +149,6 @@ func TestGetConfig(t *testing.T) {
 				"ldconfig = \"/foo/bar/ldconfig\"",
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
-				"experimental = true",
 				"discover-mode = \"not-legacy\"",
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",

test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so

@@ -1 +0,0 @@
-# This is a dummy lib file to test nvidia-runtime.experimental

test/container/toolkit_test.sh

@@ -46,13 +46,6 @@ testing::toolkit::install() {
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-hook.real"
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
 
-	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental.real"
-
-	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-	grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
-
 	test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
 
 	# Ensure that the config file has the required contents.

tools/container/README.md

@@ -15,28 +15,23 @@ docker setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`.
 
 Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
 
-**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
 
 | Flags                                                       | Installed Runtimes              | Default Runtime       |
 |-------------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| **NONE SPECIFIED**                                          | `nvidia`                        | `nvidia`              |
-| `--runtime-name nvidia`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--runtime-name nvidia`                                     | `nvidia`                        | `nvidia`              |
-| `--runtime-name NAME`                                       | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--runtime-name NAME`                                       | `NAME`                          | `NAME`                |
-| `--runtime-name nvidia-experimental`                        | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default`                                          | `nvidia`                        | `nvidia`              |
-| `--set-as-default`                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name nvidia`                    | `nvidia`                        | `nvidia`              |
-| `--set-as-default --runtime-name nvidia`                    | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name NAME`                      | `NAME`                          | `NAME`                |
-| `--set-as-default --runtime-name NAME`                      | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default=false`                                    | `nvidia`                        | **NOT SET**           |
-| `--set-as-default --runtime-name nvidia-experimental`       | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default=false --runtime-name NAME`                | `NAME`                          | **NOT SET**           |
-| `--set-as-default=false`                                    | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia`              | `nvidia`                        | **NOT SET**           |
-| `--set-as-default=false --runtime-name NAME`                | `NAME`, `nvidia-experimental`   | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia`              | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET**           |
 
 These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
 
@@ -48,7 +43,7 @@ containerd setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`.
 
 Adding the `--set-as-default` flag as follows:
 ```bash
@@ -59,19 +54,15 @@ containerd setup \
 ```
 will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
 
-**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
 
 | Flags                                                  | Installed Runtime Classes       | Default Runtime Class |
 |--------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                     | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| **NONE SPECIFIED**                                     | `nvidia`                        | **NOT SET**           |
-| `--runtime-class NAME`                                 | `NAME`, `nvidia-experimental`   | **NOT SET**           |
+| `--runtime-class NAME`                                 | `NAME`                          | **NOT SET**           |
-| `--runtime-class nvidia`                               | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--runtime-class nvidia`                               | `nvidia`                        | **NOT SET**           |
-| `--runtime-class nvidia-experimental`                  | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default`                                     | `nvidia`                        | `nvidia`              |
-| `--set-as-default`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-class NAME`                | `NAME`                          | `NAME`                |
-| `--set-as-default --runtime-class NAME`                | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default --runtime-class nvidia`              | `nvidia`                        | `nvidia`              |
-| `--set-as-default --runtime-class nvidia`              | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
 
 These combinations also hold for the environment variables that map to the command line flags.

tools/container/containerd/config_v1_test.go

@@ -56,13 +56,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
 			expectedDefaultRuntimeName:   nil,
 			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
 		},
-		{
-			legacyConfig:                 true,
-			setAsDefault:                 true,
-			runtimeName:                  "nvidia-experimental",
-			expectedDefaultRuntimeName:   nil,
-			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime.experimental",
-		},
 		{
 			legacyConfig:                 false,
 			setAsDefault:                 false,
@@ -82,13 +75,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
 			expectedDefaultRuntimeName:   "NAME",
 			expectedDefaultRuntimeBinary: nil,
 		},
-		{
-			legacyConfig:                 false,
-			setAsDefault:                 true,
-			runtimeName:                  "nvidia-experimental",
-			expectedDefaultRuntimeName:   "nvidia-experimental",
-			expectedDefaultRuntimeBinary: nil,
-		},
 	}
 
 	for i, tc := range testCases {
@@ -163,17 +149,6 @@ func TestUpdateV1Config(t *testing.T) {
 										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -221,75 +196,6 @@ func TestUpdateV1Config(t *testing.T) {
 										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(1),
-				"plugins": map[string]interface{}{
-					"cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -387,18 +293,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
 										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -459,90 +353,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
 										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(1),
-				"plugins": map[string]interface{}{
-					"cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"runc": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/runc-binary",
-									},
-								},
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -625,7 +435,6 @@ func TestRevertV1Config(t *testing.T) {
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
 								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
 								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
 								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
 							},
@@ -642,7 +451,6 @@ func TestRevertV1Config(t *testing.T) {
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
 								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
 								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
 								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
 							},

tools/container/containerd/config_v2_test.go

@@ -49,11 +49,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: nil,
 		},
-		{
-			setAsDefault:               false,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: nil,
-		},
 		{
 			setAsDefault:               true,
 			runtimeName:                "nvidia",
@@ -64,11 +59,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: "NAME",
 		},
-		{
-			setAsDefault:               true,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: "nvidia-experimental",
-		},
 	}
 
 	for i, tc := range testCases {
@@ -124,16 +114,6 @@ func TestUpdateV2Config(t *testing.T) {
 										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -178,70 +158,6 @@ func TestUpdateV2Config(t *testing.T) {
 										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(2),
-				"plugins": map[string]interface{}{
-					"io.containerd.grpc.v1.cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runtime_type",
-									"runtime_root":                    "",
-									"runtime_engine":                  "",
-									"privileged_without_host_devices": false,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runtime_type",
 									"runtime_root":                    "",
@@ -337,17 +253,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -405,85 +310,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
 									},
 								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
-								"nvidia-cdi": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
-									},
-								},
-								"nvidia-legacy": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			runtimeName: "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"version": int64(2),
-				"plugins": map[string]interface{}{
-					"io.containerd.grpc.v1.cri": map[string]interface{}{
-						"containerd": map[string]interface{}{
-							"runtimes": map[string]interface{}{
-								"runc": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/runc-binary",
-									},
-								},
-								"nvidia": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
-									},
-								},
-								"nvidia-experimental": map[string]interface{}{
-									"runtime_type":                    "runc_runtime_type",
-									"runtime_root":                    "runc_runtime_root",
-									"runtime_engine":                  "runc_runtime_engine",
-									"privileged_without_host_devices": true,
-									"container_annotations":           []string{"cdi.k8s.io/*"},
-									"options": map[string]interface{}{
-										"runc-option": "value",
-										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.experimental",
-									},
-								},
 								"nvidia-cdi": map[string]interface{}{
 									"runtime_type":                    "runc_runtime_type",
 									"runtime_root":                    "runc_runtime_root",
@@ -563,7 +389,6 @@ func TestRevertV2Config(t *testing.T) {
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
 								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
 							},
 						},
 					},
@@ -578,7 +403,6 @@ func TestRevertV2Config(t *testing.T) {
 						"containerd": map[string]interface{}{
 							"runtimes": map[string]interface{}{
 								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
-								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
 							},
 							"default_runtime_name": "nvidia",
 						},

tools/container/docker/docker.go

@@ -142,7 +142,7 @@ func main() {
 		},
 		&cli.BoolFlag{
 			Name:        "set-as-default",
-			Usage:       "Set the `nvidia` runtime as the default runtime. If --runtime-name is specified as `nvidia-experimental` the experimental runtime is set as the default runtime instead",
+			Usage:       "Set the `nvidia` runtime as the default runtime.",
 			Value:       defaultSetAsDefault,
 			Destination: &options.SetAsDefault,
 			EnvVars:     []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"},

tools/container/docker/docker_test.go

@@ -43,11 +43,6 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: "NAME",
 		},
-		{
-			setAsDefault:               true,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: "nvidia-experimental",
-		},
 		{
 			setAsDefault:               true,
 			runtimeName:                "nvidia",
@@ -92,10 +87,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -117,35 +108,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
-					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
-						"args": []string{},
-					},
-					"nvidia-legacy": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config:       map[string]interface{}{},
-			setAsDefault: false,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -173,10 +135,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -207,10 +165,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -235,38 +189,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
-					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
-						"args": []string{},
-					},
-					"nvidia-legacy": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"default-runtime": "runc",
-			},
-			setAsDefault: true,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"default-runtime": "nvidia-experimental",
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -299,10 +221,6 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
@@ -361,7 +279,7 @@ func TestRevertConfig(t *testing.T) {
 		{
 			config: map[string]interface{}{
 				"runtimes": map[string]interface{}{
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
@@ -376,25 +294,6 @@ func TestRevertConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
-				},
-			},
-			expectedConfig: map[string]interface{}{},
-		},
-		{
-			config: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
-						"args": []string{},
-					},
 					"nvidia-cdi": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},

tools/container/operator/operator.go

@@ -20,7 +20,6 @@ import "path/filepath"
 
 const (
 	defaultRuntimeName = "nvidia"
-	experimentalRuntimeName = "nvidia-experimental"
 
 	defaultRoot = "/usr/bin"
 )
@@ -59,7 +58,7 @@ func GetRuntimes(opts ...Option) Runtimes {
 	runtimes := make(Runtimes)
 	runtimes.add(c.nvidiaRuntime())
 
-	modes := []string{"experimental", "cdi", "legacy"}
+	modes := []string{"cdi", "legacy"}
 	for _, mode := range modes {
 		runtimes.add(c.modeRuntime(mode))
 	}
@@ -85,7 +84,6 @@ func (r *Runtimes) add(runtime Runtime) {
 // If name is equal to one of the predefined runtimes, `nvidia` is used as the runtime name instead.
 func (c config) nvidiaRuntime() Runtime {
 	predefinedRuntimes := map[string]struct{}{
-		"nvidia-experimental": {},
 		"nvidia-cdi":    {},
 		"nvidia-legacy": {},
 	}

tools/container/operator/operator_test.go

@@ -37,10 +37,6 @@ func TestOptions(t *testing.T) {
 					name: "nvidia",
 					Path: "/usr/bin/nvidia-container-runtime",
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -60,10 +56,6 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime",
 					SetAsDefault: true,
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -84,10 +76,6 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime",
 					SetAsDefault: true,
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -108,10 +96,6 @@ func TestOptions(t *testing.T) {
 					Path:         "/usr/bin/nvidia-container-runtime",
 					SetAsDefault: true,
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",
@@ -130,56 +114,6 @@ func TestOptions(t *testing.T) {
 					name: "NAME",
 					Path: "/usr/bin/nvidia-container-runtime",
 				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
-				"nvidia-cdi": Runtime{
-					name: "nvidia-cdi",
-					Path: "/usr/bin/nvidia-container-runtime.cdi",
-				},
-				"nvidia-legacy": Runtime{
-					name: "nvidia-legacy",
-					Path: "/usr/bin/nvidia-container-runtime.legacy",
-				},
-			},
-		},
-		{
-			setAsDefault:           true,
-			nvidiaRuntimeName:      "nvidia-experimental",
-			expectedDefaultRuntime: "nvidia-experimental",
-			expectedRuntimes: Runtimes{
-				"nvidia": Runtime{
-					name: "nvidia",
-					Path: "/usr/bin/nvidia-container-runtime",
-				},
-				"nvidia-experimental": Runtime{
-					name:         "nvidia-experimental",
-					Path:         "/usr/bin/nvidia-container-runtime.experimental",
-					SetAsDefault: true,
-				},
-				"nvidia-cdi": Runtime{
-					name: "nvidia-cdi",
-					Path: "/usr/bin/nvidia-container-runtime.cdi",
-				},
-				"nvidia-legacy": Runtime{
-					name: "nvidia-legacy",
-					Path: "/usr/bin/nvidia-container-runtime.legacy",
-				},
-			},
-		},
-		{
-			setAsDefault:      false,
-			nvidiaRuntimeName: "nvidia-experimental",
-			expectedRuntimes: Runtimes{
-				"nvidia": Runtime{
-					name: "nvidia",
-					Path: "/usr/bin/nvidia-container-runtime",
-				},
-				"nvidia-experimental": Runtime{
-					name: "nvidia-experimental",
-					Path: "/usr/bin/nvidia-container-runtime.experimental",
-				},
 				"nvidia-cdi": Runtime{
 					name: "nvidia-cdi",
 					Path: "/usr/bin/nvidia-container-runtime.cdi",

tools/container/toolkit/runtime.go

@@ -19,16 +19,12 @@ package main
 import (
 	"fmt"
 	"path/filepath"
-	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
-	log "github.com/sirupsen/logrus"
 )
 
 const (
 	nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
-
-	nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental"
 )
 
 // installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
@@ -36,9 +32,6 @@ const (
 func installContainerRuntimes(toolkitDir string, driverRoot string) error {
 	runtimes := operator.GetRuntimes()
 	for _, runtime := range runtimes {
-		if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource {
-			continue
-		}
 		r := newNvidiaContainerRuntimeInstaller(runtime.Path)
 
 		_, err := r.install(toolkitDir)
@@ -46,30 +39,6 @@ func installContainerRuntimes(toolkitDir string, driverRoot string) error {
 			return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
 		}
 	}
-
-	// Install the experimental runtime and treat failures as non-fatal.
-	err := installExperimentalRuntime(toolkitDir, driverRoot)
-	if err != nil {
-		log.Warningf("Could not install experimental runtime: %v", err)
-	}
-
-	return nil
-}
-
-// installExperimentalRuntime ensures that the experimental NVIDIA Container runtime is installed
-func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
-	libraryRoot, err := findLibraryRoot(driverRoot)
-	if err != nil {
-		log.Warningf("Error finding library path for root %v: %v", driverRoot, err)
-	}
-	log.Infof("Using library root %v", libraryRoot)
-
-	e := newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot)
-	_, err = e.install(toolkitDir)
-	if err != nil {
-		return fmt.Errorf("error installing experimental NVIDIA Container Runtime: %v", err)
-	}
-
 	return nil
 }
 
@@ -87,22 +56,6 @@ func newNvidiaContainerRuntimeInstaller(source string) *executable {
 	return newRuntimeInstaller(source, target, nil)
 }
 
-func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {
-	source := nvidiaExperimentalContainerRuntimeSource
-	wrapperName := filepath.Base(source)
-	dotfileName := wrapperName + ".real"
-	target := executableTarget{
-		dotfileName: dotfileName,
-		wrapperName: wrapperName,
-	}
-
-	env := make(map[string]string)
-	if libraryRoot != "" {
-		env["LD_LIBRARY_PATH"] = strings.Join([]string{libraryRoot, "$LD_LIBRARY_PATH"}, ":")
-	}
-	return newRuntimeInstaller(source, target, env)
-}
-
 func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {
 	preLines := []string{
 		"",

tools/container/toolkit/runtime_test.go

@@ -55,36 +55,3 @@ func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
 	exepectedContents := strings.Join(expectedLines, "\n")
 	require.Equal(t, exepectedContents, buf.String())
 }
-
-func TestExperimentalContainerRuntimeInstallerWrapper(t *testing.T) {
-	r := newNvidiaContainerRuntimeExperimentalInstaller("/some/root/usr/lib64")
-
-	const shebang = "#! /bin/sh"
-	const destFolder = "/dest/folder"
-	const dotfileName = "source.real"
-
-	buf := &bytes.Buffer{}
-
-	err := r.writeWrapperTo(buf, destFolder, dotfileName)
-	require.NoError(t, err)
-
-	expectedLines := []string{
-		shebang,
-		"",
-		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
-		"if [ \"${?}\" != \"0\" ]; then",
-		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
-		"	exec runc \"$@\"",
-		"fi",
-		"",
-		"LD_LIBRARY_PATH=/some/root/usr/lib64:$LD_LIBRARY_PATH \\",
-		"PATH=/dest/folder:$PATH \\",
-		"XDG_CONFIG_HOME=/dest/folder/.config \\",
-		"source.real \\",
-		"\t\"$@\"",
-		"",
-	}
-
-	exepectedContents := strings.Join(expectedLines, "\n")
-	require.Equal(t, exepectedContents, buf.String())
-}