UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2024-11-21 07:47:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove NVIDIA experimental runtime from toolkit container

Browse Source 
Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar 2022-11-07 12:22:30 +01:00
parent 4ca8d4173a
commit 0938576618
15 changed files with 32 additions and 677 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -19,6 +19,7 @@
* [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
* [toolkit-container] Bump CUDA base image version to 12.2.0.
* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
## v1.13.1

5
build/container/Dockerfile.centos
View File

@ -26,11 +26,6 @@ FROM golang:${GOLANG_VERSION} as build
# /artifacts/bin
ARG GOPATH=/artifacts
# Install the experiemental nvidia-container-runtime
# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
WORKDIR /build
COPY . .

5
build/container/Dockerfile.ubuntu
View File

@ -26,11 +26,6 @@ FROM golang:${GOLANG_VERSION} as build
# /artifacts/bin
ARG GOPATH=/artifacts
# Install the experiemental nvidia-container-runtime
# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
WORKDIR /build
COPY . .

2
internal/config/config_test.go
View File

@ -96,7 +96,6 @@ func TestGetConfig(t *testing.T) {
"nvidia-container-cli.load-kmods = false",
"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
"nvidia-container-runtime.debug = \"/foo/bar\"",
"nvidia-container-runtime.experimental = true",
"nvidia-container-runtime.discover-mode = \"not-legacy\"",
"nvidia-container-runtime.log-level = \"debug\"",
"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
@ -150,7 +149,6 @@ func TestGetConfig(t *testing.T) {
"ldconfig = \"/foo/bar/ldconfig\"",
"[nvidia-container-runtime]",
"debug = \"/foo/bar\"",
"experimental = true",
"discover-mode = \"not-legacy\"",
"log-level = \"debug\"",
"runtimes = [\"/some/runtime\",]",

1
test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so
View File

@ -1 +0,0 @@
# This is a dummy lib file to test nvidia-runtime.experimental

7
test/container/toolkit_test.sh
View File

@ -46,13 +46,6 @@ testing::toolkit::install() {
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-hook.real"
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental.real"
grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
# Ensure that the config file has the required contents.

43
tools/container/README.md
View File

@ -15,28 +15,23 @@ docker setup \
/run/nvidia/toolkit
```
Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`.
Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
| Flags | Installed Runtimes | Default Runtime |
|-------------------------------------------------------------|:--------------------------------|:----------------------|
| **NONE SPECIFIED** | `nvidia`, `nvidia-experimental` | `nvidia` |
| `--runtime-name nvidia` | `nvidia`, `nvidia-experimental` | `nvidia` |
| `--runtime-name NAME` | `NAME`, `nvidia-experimental` | `NAME` |
| `--runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
| `--set-as-default` | `nvidia`, `nvidia-experimental` | `nvidia` |
| `--set-as-default --runtime-name nvidia` | `nvidia`, `nvidia-experimental` | `nvidia` |
| `--set-as-default --runtime-name NAME` | `NAME`, `nvidia-experimental` | `NAME` |
| `--set-as-default --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
| `--set-as-default=false` | `nvidia`, `nvidia-experimental` | **NOT SET** |
| `--set-as-default=false --runtime-name NAME` | `NAME`, `nvidia-experimental` | **NOT SET** |
| `--set-as-default=false --runtime-name nvidia` | `nvidia`, `nvidia-experimental` | **NOT SET** |
| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET** |
| **NONE SPECIFIED** | `nvidia` | `nvidia` |
| `--runtime-name nvidia` | `nvidia` | `nvidia` |
| `--runtime-name NAME` | `NAME` | `NAME` |
| `--set-as-default` | `nvidia` | `nvidia` |
| `--set-as-default --runtime-name nvidia` | `nvidia` | `nvidia` |
| `--set-as-default --runtime-name NAME` | `NAME` | `NAME` |
| `--set-as-default=false` | `nvidia` | **NOT SET** |
| `--set-as-default=false --runtime-name NAME` | `NAME` | **NOT SET** |
| `--set-as-default=false --runtime-name nvidia` | `nvidia` | **NOT SET** |
These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
@ -48,7 +43,7 @@ containerd setup \
/run/nvidia/toolkit
```
Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime.experimental` OCI-compliant runtime shim.
Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`.
Adding the `--set-as-default` flag as follows:
```bash
@ -59,19 +54,15 @@ containerd setup \
```
will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
| Flags | Installed Runtime Classes | Default Runtime Class |
|--------------------------------------------------------|:--------------------------------|:----------------------|
| **NONE SPECIFIED** | `nvidia`, `nvidia-experimental` | **NOT SET** |
| `--runtime-class NAME` | `NAME`, `nvidia-experimental` | **NOT SET** |
| `--runtime-class nvidia` | `nvidia`, `nvidia-experimental` | **NOT SET** |
| `--runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET** |
| `--set-as-default` | `nvidia`, `nvidia-experimental` | `nvidia` |
| `--set-as-default --runtime-class NAME` | `NAME`, `nvidia-experimental` | `NAME` |
| `--set-as-default --runtime-class nvidia` | `nvidia`, `nvidia-experimental` | `nvidia` |
| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
| **NONE SPECIFIED** | `nvidia` | **NOT SET** |
| `--runtime-class NAME` | `NAME` | **NOT SET** |
| `--runtime-class nvidia` | `nvidia` | **NOT SET** |
| `--set-as-default` | `nvidia` | `nvidia` |
| `--set-as-default --runtime-class NAME` | `NAME` | `NAME` |
| `--set-as-default --runtime-class nvidia` | `nvidia` | `nvidia` |
These combinations also hold for the environment variables that map to the command line flags.

204
tools/container/containerd/config_v1_test.go
View File

@ -56,13 +56,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
expectedDefaultRuntimeName: nil,
expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
},
{
legacyConfig: true,
setAsDefault: true,
runtimeName: "nvidia-experimental",
expectedDefaultRuntimeName: nil,
expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime.experimental",
},
{
legacyConfig: false,
setAsDefault: false,
@ -82,13 +75,6 @@ func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
expectedDefaultRuntimeName: "NAME",
expectedDefaultRuntimeBinary: nil,
},
{
legacyConfig: false,
setAsDefault: true,
runtimeName: "nvidia-experimental",
expectedDefaultRuntimeName: "nvidia-experimental",
expectedDefaultRuntimeBinary: nil,
},
}
for i, tc := range testCases {
@ -163,17 +149,6 @@ func TestUpdateV1Config(t *testing.T) {
"Runtime": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
@ -221,75 +196,6 @@ func TestUpdateV1Config(t *testing.T) {
"Runtime": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi",
},
},
"nvidia-legacy": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy",
},
},
},
},
},
},
},
},
{
runtimeName: "nvidia-experimental",
expectedConfig: map[string]interface{}{
"version": int64(1),
"plugins": map[string]interface{}{
"cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
"Runtime": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
@ -387,18 +293,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
"Runtime": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
@ -459,90 +353,6 @@ func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
"Runtime": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi",
},
},
"nvidia-legacy": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy",
},
},
},
},
},
},
},
},
{
runtimeName: "nvidia-experimental",
expectedConfig: map[string]interface{}{
"version": int64(1),
"plugins": map[string]interface{}{
"cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"runc": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/runc-binary",
},
},
"nvidia": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
"Runtime": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
"Runtime": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
@ -624,10 +434,9 @@ func TestRevertV1Config(t *testing.T) {
"cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
},
},
},
@ -641,10 +450,9 @@ func TestRevertV1Config(t *testing.T) {
"cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
"nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
},
"default_runtime": defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
"default_runtime_name": "nvidia",

180
tools/container/containerd/config_v2_test.go
View File

@ -49,11 +49,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
runtimeName: "NAME",
expectedDefaultRuntimeName: nil,
},
{
setAsDefault: false,
runtimeName: "nvidia-experimental",
expectedDefaultRuntimeName: nil,
},
{
setAsDefault: true,
runtimeName: "nvidia",
@ -64,11 +59,6 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
runtimeName: "NAME",
expectedDefaultRuntimeName: "NAME",
},
{
setAsDefault: true,
runtimeName: "nvidia-experimental",
expectedDefaultRuntimeName: "nvidia-experimental",
},
}
for i, tc := range testCases {
@ -124,16 +114,6 @@ func TestUpdateV2Config(t *testing.T) {
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
@ -178,70 +158,6 @@ func TestUpdateV2Config(t *testing.T) {
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
},
},
"nvidia-legacy": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
},
},
},
},
},
},
},
},
{
runtimeName: "nvidia-experimental",
expectedConfig: map[string]interface{}{
"version": int64(2),
"plugins": map[string]interface{}{
"io.containerd.grpc.v1.cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
"runtime_engine": "",
"privileged_without_host_devices": false,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runtime_type",
"runtime_root": "",
@ -337,17 +253,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
@ -405,85 +310,6 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
},
},
"nvidia-legacy": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
},
},
},
},
},
},
},
},
{
runtimeName: "nvidia-experimental",
expectedConfig: map[string]interface{}{
"version": int64(2),
"plugins": map[string]interface{}{
"io.containerd.grpc.v1.cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"runc": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/runc-binary",
},
},
"nvidia": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
},
},
"nvidia-experimental": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
"runtime_engine": "runc_runtime_engine",
"privileged_without_host_devices": true,
"container_annotations": []string{"cdi.k8s.io/*"},
"options": map[string]interface{}{
"runc-option": "value",
"BinaryName": "/test/runtime/dir/nvidia-container-runtime.experimental",
},
},
"nvidia-cdi": map[string]interface{}{
"runtime_type": "runc_runtime_type",
"runtime_root": "runc_runtime_root",
@ -562,8 +388,7 @@ func TestRevertV2Config(t *testing.T) {
"io.containerd.grpc.v1.cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
},
},
},
@ -577,8 +402,7 @@ func TestRevertV2Config(t *testing.T) {
"io.containerd.grpc.v1.cri": map[string]interface{}{
"containerd": map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime.experimental"),
"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
},
"default_runtime_name": "nvidia",
},

2
tools/container/docker/docker.go
View File

@ -142,7 +142,7 @@ func main() {
},
&cli.BoolFlag{
Name: "set-as-default",
Usage: "Set the `nvidia` runtime as the default runtime. If --runtime-name is specified as `nvidia-experimental` the experimental runtime is set as the default runtime instead",
Usage: "Set the `nvidia` runtime as the default runtime.",
Value: defaultSetAsDefault,
Destination: &options.SetAsDefault,
EnvVars: []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"},

103
tools/container/docker/docker_test.go
View File

@ -43,11 +43,6 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
runtimeName: "NAME",
expectedDefaultRuntimeName: "NAME",
},
{
setAsDefault: true,
runtimeName: "nvidia-experimental",
expectedDefaultRuntimeName: "nvidia-experimental",
},
{
setAsDefault: true,
runtimeName: "nvidia",
@ -92,10 +87,6 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
@ -117,35 +108,6 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
{
config: map[string]interface{}{},
setAsDefault: false,
runtimeName: "nvidia-experimental",
expectedConfig: map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
@ -173,10 +135,6 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
@ -207,10 +165,6 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
@ -235,38 +189,6 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
},
"nvidia-legacy": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
"args": []string{},
},
},
},
},
{
config: map[string]interface{}{
"default-runtime": "runc",
},
setAsDefault: true,
runtimeName: "nvidia-experimental",
expectedConfig: map[string]interface{}{
"default-runtime": "nvidia-experimental",
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
@ -299,10 +221,6 @@ func TestUpdateConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},
@ -361,7 +279,7 @@ func TestRevertConfig(t *testing.T) {
{
config: map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia-experimental": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
@ -376,25 +294,6 @@ func TestRevertConfig(t *testing.T) {
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
},
},
expectedConfig: map[string]interface{}{},
},
{
config: map[string]interface{}{
"runtimes": map[string]interface{}{
"nvidia": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime",
"args": []string{},
},
"nvidia-experimental": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.experimental",
"args": []string{},
},
"nvidia-cdi": map[string]interface{}{
"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
"args": []string{},

10
tools/container/operator/operator.go
View File

@ -19,8 +19,7 @@ package operator
import "path/filepath"
const (
defaultRuntimeName = "nvidia"
experimentalRuntimeName = "nvidia-experimental"
defaultRuntimeName = "nvidia"
defaultRoot = "/usr/bin"
)
@ -59,7 +58,7 @@ func GetRuntimes(opts ...Option) Runtimes {
runtimes := make(Runtimes)
runtimes.add(c.nvidiaRuntime())
modes := []string{"experimental", "cdi", "legacy"}
modes := []string{"cdi", "legacy"}
for _, mode := range modes {
runtimes.add(c.modeRuntime(mode))
}
@ -85,9 +84,8 @@ func (r *Runtimes) add(runtime Runtime) {
// If name is equal to one of the predefined runtimes, `nvidia` is used as the runtime name instead.
func (c config) nvidiaRuntime() Runtime {
predefinedRuntimes := map[string]struct{}{
"nvidia-experimental": {},
"nvidia-cdi": {},
"nvidia-legacy": {},
"nvidia-cdi": {},
"nvidia-legacy": {},
}
name := c.nvidiaRuntimeName
if _, isPredefinedRuntime := predefinedRuntimes[name]; isPredefinedRuntime {

66
tools/container/operator/operator_test.go
View File

@ -37,10 +37,6 @@ func TestOptions(t *testing.T) {
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
@ -60,10 +56,6 @@ func TestOptions(t *testing.T) {
Path: "/usr/bin/nvidia-container-runtime",
SetAsDefault: true,
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
@ -84,10 +76,6 @@ func TestOptions(t *testing.T) {
Path: "/usr/bin/nvidia-container-runtime",
SetAsDefault: true,
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
@ -108,10 +96,6 @@ func TestOptions(t *testing.T) {
Path: "/usr/bin/nvidia-container-runtime",
SetAsDefault: true,
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
@ -130,56 +114,6 @@ func TestOptions(t *testing.T) {
name: "NAME",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: true,
nvidiaRuntimeName: "nvidia-experimental",
expectedDefaultRuntime: "nvidia-experimental",
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
SetAsDefault: true,
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",
},
"nvidia-legacy": Runtime{
name: "nvidia-legacy",
Path: "/usr/bin/nvidia-container-runtime.legacy",
},
},
},
{
setAsDefault: false,
nvidiaRuntimeName: "nvidia-experimental",
expectedRuntimes: Runtimes{
"nvidia": Runtime{
name: "nvidia",
Path: "/usr/bin/nvidia-container-runtime",
},
"nvidia-experimental": Runtime{
name: "nvidia-experimental",
Path: "/usr/bin/nvidia-container-runtime.experimental",
},
"nvidia-cdi": Runtime{
name: "nvidia-cdi",
Path: "/usr/bin/nvidia-container-runtime.cdi",

47
tools/container/toolkit/runtime.go
View File

@ -19,16 +19,12 @@ package main
import (
"fmt"
"path/filepath"
"strings"
"github.com/NVIDIA/nvidia-container-toolkit/tools/container/operator"
log "github.com/sirupsen/logrus"
)
const (
nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
nvidiaExperimentalContainerRuntimeSource = "nvidia-container-runtime.experimental"
)
// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
@ -36,9 +32,6 @@ const (
func installContainerRuntimes(toolkitDir string, driverRoot string) error {
runtimes := operator.GetRuntimes()
for _, runtime := range runtimes {
if filepath.Base(runtime.Path) == nvidiaExperimentalContainerRuntimeSource {
continue
}
r := newNvidiaContainerRuntimeInstaller(runtime.Path)
_, err := r.install(toolkitDir)
@ -46,30 +39,6 @@ func installContainerRuntimes(toolkitDir string, driverRoot string) error {
return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
}
}
// Install the experimental runtime and treat failures as non-fatal.
err := installExperimentalRuntime(toolkitDir, driverRoot)
if err != nil {
log.Warningf("Could not install experimental runtime: %v", err)
}
return nil
}
// installExperimentalRuntime ensures that the experimental NVIDIA Container runtime is installed
func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
libraryRoot, err := findLibraryRoot(driverRoot)
if err != nil {
log.Warningf("Error finding library path for root %v: %v", driverRoot, err)
}
log.Infof("Using library root %v", libraryRoot)
e := newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot)
_, err = e.install(toolkitDir)
if err != nil {
return fmt.Errorf("error installing experimental NVIDIA Container Runtime: %v", err)
}
return nil
}
@ -87,22 +56,6 @@ func newNvidiaContainerRuntimeInstaller(source string) *executable {
return newRuntimeInstaller(source, target, nil)
}
func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {
source := nvidiaExperimentalContainerRuntimeSource
wrapperName := filepath.Base(source)
dotfileName := wrapperName + ".real"
target := executableTarget{
dotfileName: dotfileName,
wrapperName: wrapperName,
}
env := make(map[string]string)
if libraryRoot != "" {
env["LD_LIBRARY_PATH"] = strings.Join([]string{libraryRoot, "$LD_LIBRARY_PATH"}, ":")
}
return newRuntimeInstaller(source, target, env)
}
func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {
preLines := []string{
"",

33
tools/container/toolkit/runtime_test.go
View File

@ -55,36 +55,3 @@ func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
exepectedContents := strings.Join(expectedLines, "\n")
require.Equal(t, exepectedContents, buf.String())
}
func TestExperimentalContainerRuntimeInstallerWrapper(t *testing.T) {
r := newNvidiaContainerRuntimeExperimentalInstaller("/some/root/usr/lib64")
const shebang = "#! /bin/sh"
const destFolder = "/dest/folder"
const dotfileName = "source.real"
buf := &bytes.Buffer{}
err := r.writeWrapperTo(buf, destFolder, dotfileName)
require.NoError(t, err)
expectedLines := []string{
shebang,
"",
"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
"if [ \"${?}\" != \"0\" ]; then",
" echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
" exec runc \"$@\"",
"fi",
"",
"LD_LIBRARY_PATH=/some/root/usr/lib64:$LD_LIBRARY_PATH \\",
"PATH=/dest/folder:$PATH \\",
"XDG_CONFIG_HOME=/dest/folder/.config \\",
"source.real \\",
"\t\"$@\"",
"",
}
exepectedContents := strings.Join(expectedLines, "\n")
require.Equal(t, exepectedContents, buf.String())
}