diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fa2c96..2c5fd23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library * [toolkit-container] Bump CUDA base image version to 12.2.0. +* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode. ## v1.13.1 diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos index 4875084c..7a72d507 100644 --- a/build/container/Dockerfile.centos +++ b/build/container/Dockerfile.centos @@ -26,11 +26,6 @@ FROM golang:${GOLANG_VERSION} as build # /artifacts/bin ARG GOPATH=/artifacts -# Install the experiemental nvidia-container-runtime -# NOTE: This will be integrated into the nvidia-container-toolkit package / repo -ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental -RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION} - WORKDIR /build COPY . . diff --git a/build/container/Dockerfile.ubuntu b/build/container/Dockerfile.ubuntu index 2efb95bf..ce248c95 100644 --- a/build/container/Dockerfile.ubuntu +++ b/build/container/Dockerfile.ubuntu @@ -26,11 +26,6 @@ FROM golang:${GOLANG_VERSION} as build # /artifacts/bin ARG GOPATH=/artifacts -# Install the experiemental nvidia-container-runtime -# NOTE: This will be integrated into the nvidia-container-toolkit package / repo -ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental -RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION} - WORKDIR /build COPY . . diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0c60ca99..0e709f9b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -96,7 +96,6 @@ func TestGetConfig(t *testing.T) { "nvidia-container-cli.load-kmods = false", "nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"", "nvidia-container-runtime.debug = \"/foo/bar\"", - "nvidia-container-runtime.experimental = true", "nvidia-container-runtime.discover-mode = \"not-legacy\"", "nvidia-container-runtime.log-level = \"debug\"", "nvidia-container-runtime.runtimes = [\"/some/runtime\",]", @@ -150,7 +149,6 @@ func TestGetConfig(t *testing.T) { "ldconfig = \"/foo/bar/ldconfig\"", "[nvidia-container-runtime]", "debug = \"/foo/bar\"", - "experimental = true", "discover-mode = \"not-legacy\"", "log-level = \"debug\"", "runtimes = [\"/some/runtime\",]", diff --git a/test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so b/test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so deleted file mode 100644 index 5f0613d7..00000000 --- a/test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so +++ /dev/null @@ -1 +0,0 @@ -# This is a dummy lib file to test nvidia-runtime.experimental diff --git a/test/container/toolkit_test.sh b/test/container/toolkit_test.sh index d11b0a58..46d2e206 100644 --- a/test/container/toolkit_test.sh +++ b/test/container/toolkit_test.sh @@ -46,13 +46,6 @@ testing::toolkit::install() { test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-hook.real" test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real" - test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental" - test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental.real" - - grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental" - grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental" - grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental" - test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml" # Ensure that the config file has the required contents. diff --git a/tools/container/README.md b/tools/container/README.md index 16e4c055..c438f4b4 100644 --- a/tools/container/README.md +++ b/tools/container/README.md @@ -15,28 +15,23 @@ docker setup \ /run/nvidia/toolkit ``` -Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim. +Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`. -**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use. - The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations. | Flags | Installed Runtimes | Default Runtime | |-------------------------------------------------------------|:--------------------------------|:----------------------| -| **NONE SPECIFIED** | `nvidia`, `nvidia-experimental` | `nvidia` | -| `--runtime-name nvidia` | `nvidia`, `nvidia-experimental` | `nvidia` | -| `--runtime-name NAME` | `NAME`, `nvidia-experimental` | `NAME` | -| `--runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` | -| `--set-as-default` | `nvidia`, `nvidia-experimental` | `nvidia` | -| `--set-as-default --runtime-name nvidia` | `nvidia`, `nvidia-experimental` | `nvidia` | -| `--set-as-default --runtime-name NAME` | `NAME`, `nvidia-experimental` | `NAME` | -| `--set-as-default --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` | -| `--set-as-default=false` | `nvidia`, `nvidia-experimental` | **NOT SET** | -| `--set-as-default=false --runtime-name NAME` | `NAME`, `nvidia-experimental` | **NOT SET** | -| `--set-as-default=false --runtime-name nvidia` | `nvidia`, `nvidia-experimental` | **NOT SET** | -| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET** | +| **NONE SPECIFIED** | `nvidia` | `nvidia` | +| `--runtime-name nvidia` | `nvidia` | `nvidia` | +| `--runtime-name NAME` | `NAME` | `NAME` | +| `--set-as-default` | `nvidia` | `nvidia` | +| `--set-as-default --runtime-name nvidia` | `nvidia` | `nvidia` | +| `--set-as-default --runtime-name NAME` | `NAME` | `NAME` | +| `--set-as-default=false` | `nvidia` | **NOT SET** | +| `--set-as-default=false --runtime-name NAME` | `NAME` | **NOT SET** | +| `--set-as-default=false --runtime-name nvidia` | `nvidia` | **NOT SET** | These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`. @@ -48,7 +43,7 @@ containerd setup \ /run/nvidia/toolkit ``` -Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim. +Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. Adding the `--set-as-default` flag as follows: ```bash @@ -59,19 +54,15 @@ containerd setup \ ``` will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class. -**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use. - The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations. | Flags | Installed Runtime Classes | Default Runtime Class | |--------------------------------------------------------|:--------------------------------|:----------------------| -| **NONE SPECIFIED** | `nvidia`, `nvidia-experimental` | **NOT SET** | -| `--runtime-class NAME` | `NAME`, `nvidia-experimental` | **NOT SET** | -| `--runtime-class nvidia` | `nvidia`, `nvidia-experimental` | **NOT SET** | -| `--runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET** | -| `--set-as-default` | `nvidia`, `nvidia-experimental` | `nvidia` | -| `--set-as-default --runtime-class NAME` | `NAME`, `nvidia-experimental` | `NAME` | -| `--set-as-default --runtime-class nvidia` | `nvidia`, `nvidia-experimental` | `nvidia` | -| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` | +| **NONE SPECIFIED** | `nvidia` | **NOT SET** | +| `--runtime-class NAME` | `NAME` | **NOT SET** | +| `--runtime-class nvidia` | `nvidia` | **NOT SET** | +| `--set-as-default` | `nvidia` | `nvidia` | +| `--set-as-default --runtime-class NAME` | `NAME` | `NAME` | +| `--set-as-default --runtime-class nvidia` | `nvidia` | `nvidia` | These combinations also hold for the environment variables that map to the command line flags. diff --git a/tools/container/containerd/config_v1_test.go b/tools/container/containerd/config_v1_test.go index 72d285ea..3430e781 100644 --- a/tools/container/containerd/config_v1_test.go +++ b/tools/container/containerd/config_v1_test.go @@ -56,13 +56,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) { expectedDefaultRuntimeName: nil, expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime", }, - { - legacyConfig: true, - setAsDefault: true, - runtimeName: "nvidia-experimental", - expectedDefaultRuntimeName: nil, - expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime.experimental", - }, { legacyConfig: false, setAsDefault: false, @@ -82,13 +75,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) { expectedDefaultRuntimeName: "NAME", expectedDefaultRuntimeBinary: nil, }, - { - legacyConfig: false, - setAsDefault: true, - runtimeName: "nvidia-experimental", - expectedDefaultRuntimeName: "nvidia-experimental", - expectedDefaultRuntimeBinary: nil, - }, } for i, tc := range testCases { @@ -163,17 +149,6 @@ func TestUpdateV1Config(t *testing.T) { "Runtime": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runtime_type", "runtime_root": "", @@ -221,75 +196,6 @@ func TestUpdateV1Config(t *testing.T) { "Runtime": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "nvidia-experimental", - expectedConfig: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - "Runtime": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runtime_type", "runtime_root": "", @@ -387,18 +293,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) { "Runtime": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runc_runtime_type", "runtime_root": "runc_runtime_root", @@ -459,90 +353,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) { "Runtime": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "nvidia-experimental", - expectedConfig: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/runc-binary", - }, - }, - "nvidia": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - "Runtime": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runc_runtime_type", "runtime_root": "runc_runtime_root", @@ -624,10 +434,9 @@ func TestRevertV1Config(t *testing.T) { "cri": map[string]interface{}{ "containerd": map[string]interface{}{ "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), - "nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"), - "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), - "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), + "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), + "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), + "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), }, }, }, @@ -641,10 +450,9 @@ func TestRevertV1Config(t *testing.T) { "cri": map[string]interface{}{ "containerd": map[string]interface{}{ "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), - "nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"), - "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), - "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), + "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), + "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), + "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), }, "default_runtime": defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"), "default_runtime_name": "nvidia", diff --git a/tools/container/containerd/config_v2_test.go b/tools/container/containerd/config_v2_test.go index b5d29e91..3081b8a8 100644 --- a/tools/container/containerd/config_v2_test.go +++ b/tools/container/containerd/config_v2_test.go @@ -49,11 +49,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) { runtimeName: "NAME", expectedDefaultRuntimeName: nil, }, - { - setAsDefault: false, - runtimeName: "nvidia-experimental", - expectedDefaultRuntimeName: nil, - }, { setAsDefault: true, runtimeName: "nvidia", @@ -64,11 +59,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) { runtimeName: "NAME", expectedDefaultRuntimeName: "NAME", }, - { - setAsDefault: true, - runtimeName: "nvidia-experimental", - expectedDefaultRuntimeName: "nvidia-experimental", - }, } for i, tc := range testCases { @@ -124,16 +114,6 @@ func TestUpdateV2Config(t *testing.T) { "BinaryName": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runtime_type", "runtime_root": "", @@ -178,70 +158,6 @@ func TestUpdateV2Config(t *testing.T) { "BinaryName": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "nvidia-experimental", - expectedConfig: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runtime_type", "runtime_root": "", @@ -337,17 +253,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) { "BinaryName": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runc_runtime_type", "runtime_root": "runc_runtime_root", @@ -405,85 +310,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) { "BinaryName": "/test/runtime/dir/nvidia-container-runtime", }, }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "nvidia-experimental", - expectedConfig: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/runc-binary", - }, - }, - "nvidia": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-experimental": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental", - }, - }, "nvidia-cdi": map[string]interface{}{ "runtime_type": "runc_runtime_type", "runtime_root": "runc_runtime_root", @@ -562,8 +388,7 @@ func TestRevertV2Config(t *testing.T) { "io.containerd.grpc.v1.cri": map[string]interface{}{ "containerd": map[string]interface{}{ "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"), - "nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"), + "nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"), }, }, }, @@ -577,8 +402,7 @@ func TestRevertV2Config(t *testing.T) { "io.containerd.grpc.v1.cri": map[string]interface{}{ "containerd": map[string]interface{}{ "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"), - "nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"), + "nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"), }, "default_runtime_name": "nvidia", }, diff --git a/tools/container/docker/docker.go b/tools/container/docker/docker.go index a32b0744..c2dddb7e 100644 --- a/tools/container/docker/docker.go +++ b/tools/container/docker/docker.go @@ -142,7 +142,7 @@ func main() { }, &cli.BoolFlag{ Name: "set-as-default", - Usage: "Set the `nvidia` runtime as the default runtime. If --runtime-name is specified as `nvidia-experimental` the experimental runtime is set as the default runtime instead", + Usage: "Set the `nvidia` runtime as the default runtime.", Value: defaultSetAsDefault, Destination: &options.SetAsDefault, EnvVars: []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"}, diff --git a/tools/container/docker/docker_test.go b/tools/container/docker/docker_test.go index f33d904a..d43c7203 100644 --- a/tools/container/docker/docker_test.go +++ b/tools/container/docker/docker_test.go @@ -43,11 +43,6 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) { runtimeName: "NAME", expectedDefaultRuntimeName: "NAME", }, - { - setAsDefault: true, - runtimeName: "nvidia-experimental", - expectedDefaultRuntimeName: "nvidia-experimental", - }, { setAsDefault: true, runtimeName: "nvidia", @@ -92,10 +87,6 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, @@ -117,35 +108,6 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, - "nvidia-cdi": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.cdi", - "args": []string{}, - }, - "nvidia-legacy": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.legacy", - "args": []string{}, - }, - }, - }, - }, - { - config: map[string]interface{}{}, - setAsDefault: false, - runtimeName: "nvidia-experimental", - expectedConfig: map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime", - "args": []string{}, - }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, @@ -173,10 +135,6 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, @@ -207,10 +165,6 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, @@ -235,38 +189,6 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, - "nvidia-cdi": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.cdi", - "args": []string{}, - }, - "nvidia-legacy": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.legacy", - "args": []string{}, - }, - }, - }, - }, - { - config: map[string]interface{}{ - "default-runtime": "runc", - }, - setAsDefault: true, - runtimeName: "nvidia-experimental", - expectedConfig: map[string]interface{}{ - "default-runtime": "nvidia-experimental", - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime", - "args": []string{}, - }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, @@ -299,10 +221,6 @@ func TestUpdateConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, @@ -361,7 +279,7 @@ func TestRevertConfig(t *testing.T) { { config: map[string]interface{}{ "runtimes": map[string]interface{}{ - "nvidia-experimental": map[string]interface{}{ + "nvidia": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, @@ -376,25 +294,6 @@ func TestRevertConfig(t *testing.T) { "path": "/test/runtime/dir/nvidia-container-runtime", "args": []string{}, }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, - }, - }, - expectedConfig: map[string]interface{}{}, - }, - { - config: map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime", - "args": []string{}, - }, - "nvidia-experimental": map[string]interface{}{ - "path": "/test/runtime/dir/nvidia-container-runtime.experimental", - "args": []string{}, - }, "nvidia-cdi": map[string]interface{}{ "path": "/test/runtime/dir/nvidia-container-runtime.cdi", "args": []string{}, diff --git a/tools/container/operator/operator.go b/tools/container/operator/operator.go index 5f020d0b..780aa5c7 100644 --- a/tools/container/operator/operator.go +++ b/tools/container/operator/operator.go @@ -19,8 +19,7 @@ package operator import "path/filepath" const ( - defaultRuntimeName = "nvidia" - experimentalRuntimeName = "nvidia-experimental" + defaultRuntimeName = "nvidia" defaultRoot = "/usr/bin" ) @@ -59,7 +58,7 @@ func GetRuntimes(opts ...Option) Runtimes { runtimes := make(Runtimes) runtimes.add(c.nvidiaRuntime()) - modes := []string{"experimental", "cdi", "legacy"} + modes := []string{"cdi", "legacy"} for _, mode := range modes { runtimes.add(c.modeRuntime(mode)) } @@ -85,9 +84,8 @@ func (r *Runtimes) add(runtime Runtime) { // If name is equal to one of the predefined runtimes, `nvidia` is used as the runtime name instead. func (c config) nvidiaRuntime() Runtime { predefinedRuntimes := map[string]struct{}{ - "nvidia-experimental": {}, - "nvidia-cdi": {}, - "nvidia-legacy": {}, + "nvidia-cdi": {}, + "nvidia-legacy": {}, } name := c.nvidiaRuntimeName if _, isPredefinedRuntime := predefinedRuntimes[name]; isPredefinedRuntime { diff --git a/tools/container/operator/operator_test.go b/tools/container/operator/operator_test.go index e2f4abf7..f1e53994 100644 --- a/tools/container/operator/operator_test.go +++ b/tools/container/operator/operator_test.go @@ -37,10 +37,6 @@ func TestOptions(t *testing.T) { name: "nvidia", Path: "/usr/bin/nvidia-container-runtime", }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - }, "nvidia-cdi": Runtime{ name: "nvidia-cdi", Path: "/usr/bin/nvidia-container-runtime.cdi", @@ -60,10 +56,6 @@ func TestOptions(t *testing.T) { Path: "/usr/bin/nvidia-container-runtime", SetAsDefault: true, }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - }, "nvidia-cdi": Runtime{ name: "nvidia-cdi", Path: "/usr/bin/nvidia-container-runtime.cdi", @@ -84,10 +76,6 @@ func TestOptions(t *testing.T) { Path: "/usr/bin/nvidia-container-runtime", SetAsDefault: true, }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - }, "nvidia-cdi": Runtime{ name: "nvidia-cdi", Path: "/usr/bin/nvidia-container-runtime.cdi", @@ -108,10 +96,6 @@ func TestOptions(t *testing.T) { Path: "/usr/bin/nvidia-container-runtime", SetAsDefault: true, }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - }, "nvidia-cdi": Runtime{ name: "nvidia-cdi", Path: "/usr/bin/nvidia-container-runtime.cdi", @@ -130,56 +114,6 @@ func TestOptions(t *testing.T) { name: "NAME", Path: "/usr/bin/nvidia-container-runtime", }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - }, - "nvidia-cdi": Runtime{ - name: "nvidia-cdi", - Path: "/usr/bin/nvidia-container-runtime.cdi", - }, - "nvidia-legacy": Runtime{ - name: "nvidia-legacy", - Path: "/usr/bin/nvidia-container-runtime.legacy", - }, - }, - }, - { - setAsDefault: true, - nvidiaRuntimeName: "nvidia-experimental", - expectedDefaultRuntime: "nvidia-experimental", - expectedRuntimes: Runtimes{ - "nvidia": Runtime{ - name: "nvidia", - Path: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - SetAsDefault: true, - }, - "nvidia-cdi": Runtime{ - name: "nvidia-cdi", - Path: "/usr/bin/nvidia-container-runtime.cdi", - }, - "nvidia-legacy": Runtime{ - name: "nvidia-legacy", - Path: "/usr/bin/nvidia-container-runtime.legacy", - }, - }, - }, - { - setAsDefault: false, - nvidiaRuntimeName: "nvidia-experimental", - expectedRuntimes: Runtimes{ - "nvidia": Runtime{ - name: "nvidia", - Path: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": Runtime{ - name: "nvidia-experimental", - Path: "/usr/bin/nvidia-container-runtime.experimental", - }, "nvidia-cdi": Runtime{ name: "nvidia-cdi", Path: "/usr/bin/nvidia-container-runtime.cdi", diff --git a/tools/container/toolkit/runtime.go b/tools/container/toolkit/runtime.go index 884dc4bf..220d7ff5 100644 --- a/tools/container/toolkit/runtime.go +++ b/tools/container/toolkit/runtime.go @@ -19,16 +19,12 @@ package main import ( "fmt" "path/filepath" - "strings" "github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator" - log "github.com/sirupsen/logrus" ) const ( nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime" - - nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental" ) // installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables @@ -36,9 +32,6 @@ const ( func installContainerRuntimes(toolkitDir string, driverRoot string) error { runtimes := operator.GetRuntimes() for _, runtime := range runtimes { - if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource { - continue - } r := newNvidiaContainerRuntimeInstaller(runtime.Path) _, err := r.install(toolkitDir) @@ -46,30 +39,6 @@ func installContainerRuntimes(toolkitDir string, driverRoot string) error { return fmt.Errorf("error installing NVIDIA container runtime: %v", err) } } - - // Install the experimental runtime and treat failures as non-fatal. - err := installExperimentalRuntime(toolkitDir, driverRoot) - if err != nil { - log.Warningf("Could not install experimental runtime: %v", err) - } - - return nil -} - -// installExperimentalRuntime ensures that the experimental NVIDIA Container runtime is installed -func installExperimentalRuntime(toolkitDir string, driverRoot string) error { - libraryRoot, err := findLibraryRoot(driverRoot) - if err != nil { - log.Warningf("Error finding library path for root %v: %v", driverRoot, err) - } - log.Infof("Using library root %v", libraryRoot) - - e := newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot) - _, err = e.install(toolkitDir) - if err != nil { - return fmt.Errorf("error installing experimental NVIDIA Container Runtime: %v", err) - } - return nil } @@ -87,22 +56,6 @@ func newNvidiaContainerRuntimeInstaller(source string) *executable { return newRuntimeInstaller(source, target, nil) } -func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable { - source := nvidiaExperimentalContainerRuntimeSource - wrapperName := filepath.Base(source) - dotfileName := wrapperName + ".real" - target := executableTarget{ - dotfileName: dotfileName, - wrapperName: wrapperName, - } - - env := make(map[string]string) - if libraryRoot != "" { - env["LD_LIBRARY_PATH"] = strings.Join([]string{libraryRoot, "$LD_LIBRARY_PATH"}, ":") - } - return newRuntimeInstaller(source, target, env) -} - func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable { preLines := []string{ "", diff --git a/tools/container/toolkit/runtime_test.go b/tools/container/toolkit/runtime_test.go index 14afb14b..61fa8b9e 100644 --- a/tools/container/toolkit/runtime_test.go +++ b/tools/container/toolkit/runtime_test.go @@ -55,36 +55,3 @@ func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) { exepectedContents := strings.Join(expectedLines, "\n") require.Equal(t, exepectedContents, buf.String()) } - -func TestExperimentalContainerRuntimeInstallerWrapper(t *testing.T) { - r := newNvidiaContainerRuntimeExperimentalInstaller("/some/root/usr/lib64") - - const shebang = "#! /bin/sh" - const destFolder = "/dest/folder" - const dotfileName = "source.real" - - buf := &bytes.Buffer{} - - err := r.writeWrapperTo(buf, destFolder, dotfileName) - require.NoError(t, err) - - expectedLines := []string{ - shebang, - "", - "cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1", - "if [ \"${?}\" != \"0\" ]; then", - " echo \"nvidia driver modules are not yet loaded, invoking runc directly\"", - " exec runc \"$@\"", - "fi", - "", - "LD_LIBRARY_PATH=/some/root/usr/lib64:$LD_LIBRARY_PATH \\", - "PATH=/dest/folder:$PATH \\", - "XDG_CONFIG_HOME=/dest/folder/.config \\", - "source.real \\", - "\t\"$@\"", - "", - } - - exepectedContents := strings.Join(expectedLines, "\n") - require.Equal(t, exepectedContents, buf.String()) -}