Merge pull request #445 from NVIDIA/dependabot/go_modules/release-1.14/golang.org/x/sys-0.19.0

Bump golang.org/x/sys from 0.18.0 to 0.19.0

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker:stable
+  image: docker
   services:
-    - name: docker:stable-dind
+    - name: docker:dind
       command: ["--experimental"]
 
 variables:
@@ -23,6 +23,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -34,51 +35,68 @@ stages:
   - scan
   - release
 
-# Define the distribution targets
-.dist-amazonlinux2:
-  variables:
-    DIST: amazonlinux2
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
 
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
+.main-or-manual:
+  rules:
+    - !reference [.pipeline-trigger-rules, rules]
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: manual
+
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
+  rules:
+    - !reference [.main-or-manual, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
+
+# Define the distribution targets
 .dist-centos7:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: centos7
-    CVE_UPDATES: "cyrus-sasl-lib"
 
 .dist-centos8:
   variables:
     DIST: centos8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-debian10:
-  variables:
-    DIST: debian10
-
-.dist-debian9:
-  variables:
-    DIST: debian9
-
-.dist-opensuse-leap15.1:
-  variables:
-    DIST: opensuse-leap15.1
 
 .dist-ubi8:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubi8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-ubuntu16.04:
-  variables:
-    DIST: ubuntu16.04
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
 
 .dist-ubuntu20.04:
   variables:
     DIST: ubuntu20.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
 
 .dist-packaging:
   variables:
@@ -98,6 +116,8 @@ stages:
     ARCH: arm64
 
 .arch-ppc64le:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     ARCH: ppc64le
 
@@ -138,7 +158,7 @@ test-packaging:
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.3.10
+    - export REGCTL_VERSION=v0.4.5
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
@@ -194,6 +214,8 @@ test-packaging:
 .release:external:
   extends:
     - .release
+  variables:
+    FORCE_PUBLISH_IMAGES: "yes"
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -210,13 +232,6 @@ release:staging-centos7:
   needs:
     - image-centos7
 
-release:staging-centos8:
-  extends:
-    - .release:staging
-    - .dist-centos8
-  needs:
-    - image-centos8
-
 release:staging-ubi8:
   extends:
     - .release:staging
@@ -224,16 +239,6 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
 release:staging-ubuntu20.04:
   extends:
     - .release:staging

.github/workflows/pre-sanity.yml

@@ -0,0 +1,22 @@
+name: Run pre sanity
+
+# run this workflow for each commit
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+             
+      - name: Build dev image
+        run: make .build-image
+      
+      - name: Build
+        run: make docker-build
+
+      - name: Tests
+        run: make docker-coverage
+
+      - name: Checks
+        run: make docker-check

.gitignore

@@ -1,9 +1,13 @@
 dist
+artifacts
 *.swp
 *.swo
 /coverage.out*
 /test/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
+/nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk
 /shared-*
+/release-*

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,7 +94,7 @@ unit-tests:
     - .multi-arch-build
     - .package-artifacts
   stage: package-build
-  timeout: 2h 30m
+  timeout: 3h
   script:
     - ./scripts/build-packages.sh ${DIST}-${ARCH}
 
@@ -102,25 +102,35 @@ unit-tests:
     name: ${ARTIFACTS_NAME}
     paths:
       - ${ARTIFACTS_ROOT}
+  needs:
+    - job: package-meta-packages
+      artifacts: true
 
 # Define the package build targets
-package-amazonlinux2-aarch64:
+package-meta-packages:
   extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-aarch64
+    - .package-artifacts
+  stage: package-build
+  variables:
+    SKIP_LIBNVIDIA_CONTAINER: "yes"
+    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
+  parallel:
+    matrix:
+      - PACKAGING: [deb, rpm]
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+  script:
+    - ./scripts/build-packages.sh ${PACKAGING}
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
 
-package-amazonlinux2-x86_64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-package-centos7-ppc64le:
+package-centos7-aarch64:
   extends:
     - .package-build
     - .dist-centos7
-    - .arch-ppc64le
+    - .arch-aarch64
 
 package-centos7-x86_64:
   extends:
@@ -128,54 +138,12 @@ package-centos7-x86_64:
     - .dist-centos7
     - .arch-x86_64
 
-package-centos8-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-aarch64
-
 package-centos8-ppc64le:
   extends:
     - .package-build
     - .dist-centos8
     - .arch-ppc64le
 
-package-centos8-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-x86_64
-
-package-debian10-amd64:
-  extends:
-    - .package-build
-    - .dist-debian10
-    - .arch-amd64
-
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-opensuse-leap15.1-x86_64:
-  extends:
-    - .package-build
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -216,7 +184,7 @@ package-ubuntu18.04-ppc64le:
   before_script:
     - !reference [.buildx-setup, before_script]
 
-    - apk add --no-cache bash make
+    - apk add --no-cache bash make git
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
@@ -228,39 +196,17 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
-    - package-centos7-ppc64le
     - package-centos7-x86_64
 
-image-centos8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos8
-  needs:
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
 image-ubi8:
   extends:
     - .image-build
     - .package-artifacts
     - .dist-ubi8
   needs:
-    # Note: The ubi8 image uses the centos8 packages
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    # Note: The ubi8 image uses the centos7 packages
+    - package-centos7-aarch64
+    - package-centos7-x86_64
 
 image-ubuntu20.04:
   extends:
@@ -270,7 +216,8 @@ image-ubuntu20.04:
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -279,21 +226,24 @@ image-packaging:
     - .package-artifacts
     - .dist-packaging
   needs:
-    - package-amazonlinux2-aarch64
-    - package-amazonlinux2-x86_64
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-    - package-centos8-aarch64
-    - package-centos8-ppc64le
-    - package-centos8-x86_64
-    - package-debian10-amd64
-    - package-debian9-amd64
-    - package-opensuse-leap15.1-x86_64
-    - package-ubuntu16.04-amd64
-    - package-ubuntu16.04-ppc64le
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-amd64
+    - job: package-ubuntu18.04-arm64
+    - job: package-amazonlinux2-aarch64
+      optional: true
+    - job: package-amazonlinux2-x86_64
+      optional: true
+    - job: package-centos7-aarch64
+      optional: true
+    - job: package-centos7-x86_64
+      optional: true
+    - job: package-centos8-ppc64le
+      optional: true
+    - job: package-debian10-amd64
+      optional: true
+    - job: package-opensuse-leap15.1-x86_64
+      optional: true
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # Define publish test helpers
 .test:toolkit:
@@ -325,34 +275,6 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu18.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-containerd-ubuntu18.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-crio-ubuntu18.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-docker-ubuntu18.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
 test-toolkit-ubuntu20.04:
   extends:
     - .test:toolkit

.gitmodules

@@ -1,10 +1,4 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
 	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
-	branch = main
-[submodule "third_party/nvidia-container-runtime"]
-	path = third_party/nvidia-container-runtime
-	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
-[submodule "third_party/nvidia-docker"]
-	path = third_party/nvidia-docker
-	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
+	branch = release-1.14

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,6 +35,9 @@ variables:
   # Define the public staging registry
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
+  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
 .image-pull:
   stage: image-build
@@ -48,8 +51,9 @@ variables:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PUSH_MULTIPLE_TAGS: "false"
   # We delay the job start to allow the public pipeline to generate the required images.
-  when: delayed
-  start_in: 30 minutes
+  rules:
+    - when: delayed
+      start_in: 30 minutes
   timeout: 30 minutes
   retry:
     max: 2
@@ -67,34 +71,24 @@ variables:
 
 image-centos7:
   extends:
-    - .image-pull
     - .dist-centos7
-
-image-centos8:
-  extends:
     - .image-pull
-    - .dist-centos8
 
 image-ubi8:
   extends:
-    - .image-pull
     - .dist-ubi8
-
-image-ubuntu18.04:
-  extends:
     - .image-pull
-    - .dist-ubuntu18.04
 
 image-ubuntu20.04:
   extends:
-    - .image-pull
     - .dist-ubuntu20.04
+    - .image-pull
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
   extends:
-    - .image-pull
     - .dist-packaging
+    - .image-pull
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -111,10 +105,10 @@ image-packaging:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
-  except:
-    variables:
-      - $SKIP_SCANS && $SKIP_SCANS == "yes"
+    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
+  rules:
+    - if: $SKIP_SCANS != "yes"
+    - when: manual
   before_script:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     # TODO: We should specify the architecture here and scan all architectures
@@ -126,6 +120,7 @@ image-packaging:
     - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
   script:
     - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+    - rm -f "${IMAGE_ARCHIVE}"
   artifacts:
     when: always
     expire_in: 1 week
@@ -139,89 +134,53 @@ image-packaging:
 # Define the scan targets
 scan-centos7-amd64:
   extends:
-    - .scan
     - .dist-centos7
     - .platform-amd64
+    - .scan
   needs:
     - image-centos7
 
-scan-centos7-arm64:
-  extends:
-    - .scan
-    - .dist-centos7
-    - .platform-arm64
-  needs:
-    - image-centos7
-    - scan-centos7-amd64
-
-scan-centos8-amd64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-amd64
-  needs:
-    - image-centos8
-
-scan-centos8-arm64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-arm64
-  needs:
-    - image-centos8
-    - scan-centos8-amd64
-
-scan-ubuntu18.04-amd64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-amd64
-  needs:
-    - image-ubuntu18.04
-
-scan-ubuntu18.04-arm64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-arm64
-  needs:
-    - image-ubuntu18.04
-    - scan-ubuntu18.04-amd64
-
 scan-ubuntu20.04-amd64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-amd64
+    - .scan
   needs:
     - image-ubuntu20.04
 
 scan-ubuntu20.04-arm64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-arm64
+    - .scan
   needs:
     - image-ubuntu20.04
     - scan-ubuntu20.04-amd64
 
 scan-ubi8-amd64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-amd64
+    - .scan
   needs:
     - image-ubi8
 
 scan-ubi8-arm64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-arm64
+    - .scan
   needs:
     - image-ubi8
     - scan-ubi8-amd64
 
+scan-packaging:
+  extends:
+    - .dist-packaging
+    - .scan
+  needs:
+    - image-packaging
+
 # Define external release helpers
 .release:ngc:
   extends:
@@ -232,12 +191,48 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
+.release:packages:
+  stage: release
   needs:
-    - image-ubuntu18.04
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+    ARTIFACTS_DIR: "${CI_PROJECT_DIR}/artifacts"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash git
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+    - rm -rf ${ARTIFACTS_DIR}
+
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
+
+release:archive:
+  extends:
+    - .release:external
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}"
+  script:
+    - apk add --no-cache bash git
+    - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}"
 
 release:staging-ubuntu20.04:
   extends:
@@ -250,25 +245,20 @@ release:staging-ubuntu20.04:
 # Release to NGC
 release:ngc-centos7:
   extends:
-    - .release:ngc
     - .dist-centos7
-
-release:ngc-centos8:
-  extends:
     - .release:ngc
-    - .dist-centos8
-
-release:ngc-ubuntu18.04:
-  extends:
-    - .release:ngc
-    - .dist-ubuntu18.04
 
 release:ngc-ubuntu20.04:
   extends:
-    - .release:ngc
     - .dist-ubuntu20.04
+    - .release:ngc
 
 release:ngc-ubi8:
   extends:
-    - .release:ngc
     - .dist-ubi8
+    - .release:ngc
+
+release:ngc-packaging:
+  extends:
+    - .dist-packaging
+    - .release:ngc

CHANGELOG.md

@@ -0,0 +1,392 @@
+# NVIDIA Container Toolkit Changelog
+
+## v1.14.6
+* Add support for extracting device major number from `/proc/devices` if `nvidia` is used as a device name over `nvidia-frontend`.
+* Add support for selecting IMEX channels using the NVIDIA_IMEX_CHANNELS environement variable.
+
+## v1.14.5
+* Fix `nvidia-ctk runtime configure --cdi.enabled` for Docker. This was incorrectly setting `experimental = true` instead
+  of setting `features.cdi = true`.
+
+## v1.14.4
+* Include `nvidia/nvoptix.bin` in list of graphics mounts.
+* Include `vulkan/icd.d/nvidia_layers.json` in list of graphics mounts.
+* Fixed bug in `nvidia-ctk config` command when using `--set`. The types of applied config options are now applied correctly.
+* Log explicitly requested runtime mode.
+* Remove package dependency on libseccomp.
+* Added detection of libnvdxgdmal.so.1 on WSL2.
+* Fix bug in determining default nvidia-container-runtime.user config value on SUSE-based systems.
+* Add `crun` to the list of configured low-level runtimes.
+* Add `--cdi.enabled` option to `nvidia-ctk runtime configure` command to enable CDI in containerd.
+* Added support for `nvidia-ctk runtime configure --enable-cdi` for the `docker` runtime. Note that this requires Docker >= 25.
+
+* [toolkit-container] Bump CUDA base image version to 12.3.1.
+* [libnvidia-container] Added detection of libnvdxgdmal.so.1 on WSL2.
+
+## v1.14.3
+* [toolkit-container] Bump CUDA base image version to 12.2.2.
+
+## v1.14.2
+* Fix bug on Tegra-based systems where symlinks were not created in containers.
+* Add --csv.ignore-pattern command line option to nvidia-ctk cdi generate command.
+
+## v1.14.1
+* Fixed bug where contents of `/etc/nvidia-container-runtime/config.toml` is ignored by the NVIDIA Container Runtime Hook.
+
+* [libnvidia-container] Use libelf.so on RPM-based systems due to removed mageia repositories hosting pmake and bmake.
+
+## v1.14.0
+* Promote v1.14.0-rc.3 to v1.14.0
+
+## v1.14.0-rc.3
+* Added support for generating OCI hook JSON file to `nvidia-ctk runtime configure` command.
+* Remove installation of OCI hook JSON from RPM package.
+* Refactored config for `nvidia-container-runtime-hook`.
+* Added a `nvidia-ctk config` command which supports setting config options using a `--set` flag.
+* Added `--library-search-path` option to `nvidia-ctk cdi generate` command in `csv` mode. This allows folders where
+  libraries are located to be specified explicitly.
+* Updated go-nvlib to support devices which are not present in the PCI device database. This allows the creation of dev/char symlinks on systems with such devices installed.
+* Added `UsesNVGPUModule` info function for more robust platform detection. This is required on Tegra-based systems where libnvidia-ml.so is also supported.
+
+* [toolkit-container] Set `NVIDIA_VISIBLE_DEVICES=void` to prevent injection of NVIDIA devices and drivers into the NVIDIA Container Toolkit container.
+
+## v1.14.0-rc.2
+* Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
+* Remove dependency on coreutils when installing package on RPM-based systems.
+* Create ouput folders if required when running `nvidia-ctk runtime configure`
+* Generate default config as post-install step.
+* Added support for detecting GSP firmware at custom paths when generating CDI specifications.
+* Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.
+
+* [libnvidia-container] Include Shared Compiler Library (libnvidia-gpucomp.so) in the list of compute libaries.
+
+* [toolkit-container] Ensure that common envvars have higher priority when configuring the container engines.
+* [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
+
+## v1.14.0-rc.1
+
+* Add support for updating containerd configs to the `nvidia-ctk runtime configure` command.
+* Create file in `etc/ld.so.conf.d` with permissions `644` to support non-root containers.
+* Generate CDI specification files with `644` permissions to allow rootless applications (e.g. podman)
+* Add `nvidia-ctk cdi list` command to show the known CDI devices.
+* Add support for generating merged devices (e.g. `all` device) to the nvcdi API.
+* Use *.* pattern to locate libcuda.so when generating a CDI specification to support platforms where a patch version is not specified.
+* Update go-nvlib to skip devices that are not MIG capable when generating CDI specifications.
+* Add `nvidia-container-runtime-hook.path` config option to specify NVIDIA Container Runtime Hook path explicitly.
+* Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
+* Add option to load kernel modules when creating device nodes
+* Add option to create device nodes when creating `/dev/char` symlinks
+
+* [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
+
+* [toolkit-container] Allow same envars for all runtime configs
+
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
+## v1.12.0-rc.1
+
+* Add support for multiple Docker Swarm resources
+* Improve injection of Vulkan configurations and libraries
+* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
+* [libnvidia-container] Include NVVM compiler library in compute libs
+
+## v1.11.0
+
+* Promote v1.11.0-rc.3 to v1.11.0
+
+## v1.11.0-rc.3
+
+* Build fedora35 packages
+* Introduce an `nvidia-container-toolkit-base` package for better dependency management
+* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
+* Inject platform files into container on Tegra-based systems
+* [toolkit container] Update CUDA base images to 11.7.1
+* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
+
+## v1.11.0-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.0-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
+## v1.10.0
+
+* Promote v1.10.0-rc.3 to v1.10.0
+
+## v1.10.0-rc.3
+
+* Use default config instead of raising an error if config file cannot be found
+* Ignore NVIDIA_REQUIRE_JETPACK* environment variables for requirement checks
+* Fix bug in detection of Tegra systems where `/sys/devices/soc0/family` is ignored
+* Fix bug where links to devices were detected as devices
+* [libnvida-container] Fix bug introduced when adding libcudadebugger.so to list of libraries
+
+## v1.10.0-rc.2
+
+* Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
+* Switch to debug logging to reduce log verbosity
+* Support logging to logs requested in command line
+* Fix bug when launching containers with relative root path (e.g. using containerd)
+* Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
+* Fix failure to locate low-level runtime if PATH envvar is unset
+* Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
+* Use csv as default mode on Tegra systems without NVML
+* Add --version flag to all CLIs
+* [libnvidia-container] Bump libtirpc to 1.3.2
+* [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
+* [libnvidia-container] Add libcudadebugger.so to list of compute libraries
+
+## v1.10.0-rc.1
+
+* Include nvidia-ctk CLI in installed binaries
+* Add experimental option to NVIDIA Container Runtime
+
+## v1.9.0
+
+* [libnvidia-container] Add additional check for Tegra in /sys/.../family file in CLI
+* [libnvidia-container] Update jetpack-specific CLI option to only load Base CSV files by default
+* [libnvidia-container] Fix bug (from 1.8.0) when mounting GSP firmware into containers without /lib to /usr/lib symlinks
+* [libnvidia-container] Update nvml.h to CUDA 11.6.1 nvML_DEV 11.6.55
+* [libnvidia-container] Update switch statement to include new brands from latest nvml.h
+* [libnvidia-container] Process all --require flags on Jetson platforms
+* [libnvidia-container] Fix long-standing issue with running ldconfig on Debian systems
+
+## v1.8.1
+
+* [libnvidia-container] Fix bug in determining cgroup root when running in nested containers
+* [libnvidia-container] Fix permission issue when determining cgroup version
+
+## v1.8.0
+
+* Promote 1.8.0-rc.2-1 to 1.8.0
+
+## v1.8.0-rc.2
+
+* Remove support for building amazonlinux1 packages
+
+## v1.8.0-rc.1
+
+* [libnvidia-container] Add support for cgroupv2
+* Release toolkit-container images from nvidia-container-toolkit repository
+
+## v1.7.0
+
+* Promote 1.7.0-rc.1-1 to 1.7.0
+ * Bump Golang version to 1.16.4
+
+## v1.7.0-rc.1
+
+* Specify containerd runtime type as string in config tools to remove dependency on containerd package
+* Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
+
+## v1.6.0
+
+* Promote 1.6.0-rc.3-1 to 1.6.0
+ * Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
+
+## v1.6.0-rc.3
+
+* Add supported-driver-capabilities config option to the nvidia-container-toolkit
+* Move OCI and command line checks for runtime to internal oci package
+
+## v1.6.0-rc.2
+
+* Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+## v1.6.0-rc.1
+
+* Add AARCH64 package for Amazon Linux 2
+* Include nvidia-container-runtime into nvidia-container-toolkit package
+
+## v1.5.1
+
+* Fix bug where Docker Swarm device selection is ignored if NVIDIA_VISIBLE_DEVICES is also set
+* Improve unit testing by using require package and adding coverage reports
+* Remove unneeded go dependencies by running go mod tidy
+* Move contents of pkg directory to cmd for CLI tools
+* Ensure make binary target explicitly sets GOOS
+
+## v1.5.0
+
+* Add dependence on libnvidia-container-tools >= 1.4.0
+* Add golang check targets to Makefile
+* Add Jenkinsfile definition for build targets
+* Move docker.mk to docker folder
+
+## v1.4.2
+
+* Add dependence on libnvidia-container-tools >= 1.3.3
+
+## v1.4.1
+
+* Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
+* Add dependence on libnvidia-container-tools >= 1.3.2
+
+## v1.4.0
+
+* Add 'compute' capability to list of defaults
+* Add dependence on libnvidia-container-tools >= 1.3.1
+
+## v1.3.0
+
+* Promote 1.3.0-rc.2-1 to 1.3.0
+* Add dependence on libnvidia-container-tools >= 1.3.0
+
+## v1.3.0-rc.2
+
+* 2c180947 Add more tests for new semantics with device list from volume mounts
+* 7c003857 Refactor accepting device lists from volume mounts as a boolean
+
+## v1.3.0-rc.1
+
+* b50d86c1 Update build system to accept a TAG variable for things like rc.x
+* fe65573b Add common CI tests for things like golint, gofmt, unit tests, etc.
+* da6fbb34 Revert "Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*"
+* a7fb3330 Flip build-all targets to run automatically on merge requests
+* 8b248b66 Rename github.com/NVIDIA/container-toolkit to nvidia-container-toolkit
+* da36874e Add new config options to pull device list from mounted files instead of ENVVAR
+
+## v1.2.1
+
+* 4e6e0ed4 Add 'ngx' to list of*all* driver capabilities
+* 2f4af743 List config.toml as a config file in the RPM SPEC
+
+## v1.2.0
+
+*  8e0aab46 Fix repo listed in changelog for debian distributions
+*  320bb6e4 Update dependence on libnvidia-container to 1.2.0
+*  6cfc8097 Update package license to match source license
+*  e7dc3cbb Fix debian copyright file
+*  d3aee3e0 Add the 'ngx' driver capability
+
+## v1.1.2
+
+* c32237f3 Add support for parsing Linux Capabilities for older OCI specs
+
+## v1.1.1
+
+* d202aded Update dependence to libnvidia-container 1.1.1
+
+## v1.1.0
+
+* 4e4de762 Update build system to support multi-arch builds
+* fcc1d116 Add support for MIG (Multi-Instance GPUs)
+* d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
+* 60f165ad Add no-pivot option to toolkit
+
+## v1.0.5
+
+* Initial release. Replaces older package nvidia-container-runtime-hook. (Closes: #XXXXXX)

DEVELOPMENT.md

@@ -19,7 +19,7 @@ where `TARGET` is a make target that is valid for each of the sub-components.
 
 These include:
 * `ubuntu18.04-amd64`
-* `centos8-x86_64`
+* `centos7-x86_64`
 
 If no `TARGET` is specified, all valid release targets are built.
 

Makefile

@@ -39,7 +39,7 @@ CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
 CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate licenses $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
@@ -51,6 +51,7 @@ CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 else
 CLI_VERSION = $(VERSION)
 endif
+CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
 
 GOOS ?= linux
 
@@ -60,7 +61,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.gitCommit=$(GIT_COMMIT) -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-extldflags=-Wl,-z,lazy -s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...
@@ -102,6 +103,9 @@ misspell:
 vet:
 	go vet $(MODULE)/...
 
+licenses:
+	go-licenses csv $(MODULE)/...
+
 COVERAGE_FILE := coverage.out
 test: build cmds
 	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...

build/container/Dockerfile.centos

@@ -17,19 +17,29 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN yum install -y \
+    wget make git gcc \
+     && \
+    rm -rf /var/cache/yum/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -53,7 +63,7 @@ RUN [[ "${BASE_DIST}" != "centos8" ]] || \
     )
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_VISIBLE_DEVICES=void
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
 ARG ARTIFACTS_ROOT
@@ -67,9 +77,9 @@ ARG TARGETARCH
 ENV PACKAGE_ARCH ${TARGETARCH}
 RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
     yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
 
@@ -85,13 +95,6 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-COPY ./LICENSE /licenses/LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        yum update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.packaging

@@ -15,15 +15,27 @@
 ARG BASE_DIST
 ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
-ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
-
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
-COPY ./LICENSE /licenses/LICENSE
+# build-args are added to the manifest.txt file below.
+ARG BASE_DIST
+ARG PACKAGE_DIST
+ARG PACKAGE_VERSION
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
+ARG SOURCE_DATE_EPOCH
+ARG VERSION
+
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

build/container/Dockerfile.ubuntu

@@ -17,19 +17,28 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN apt-get update && \
+    apt-get install -y wget make git gcc \
+    && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -53,7 +62,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_VISIBLE_DEVICES=void
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
 ARG ARTIFACTS_ROOT
@@ -66,7 +75,7 @@ ARG PACKAGE_VERSION
 ARG TARGETARCH
 ENV PACKAGE_ARCH ${TARGETARCH}
 
-ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
+ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container/stable"
 ARG LIBNVIDIA_CONTAINER0_VERSION
 RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
         curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
@@ -75,9 +84,9 @@ RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
     fi
 
 RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
 
@@ -93,13 +102,6 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-COPY ./LICENSE /licenses/LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -14,6 +14,7 @@
 
 BUILD_MULTI_ARCH_IMAGES ?= false
 DOCKER ?= docker
+REGCTL ?= regctl
 
 BUILDX =
 ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
@@ -43,14 +44,14 @@ OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu18.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+DEFAULT_PUSH_TARGET := ubuntu20.04
+DISTRIBUTIONS := ubuntu20.04 ubi8 centos7
 
 META_TARGETS := packaging
 
 BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
 PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
 
 .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -74,6 +75,14 @@ endif
 push-%: DIST = $(*)
 push-short: DIST = $(DEFAULT_PUSH_TARGET)
 
+# Define the push targets
+$(PUSH_TARGETS): push-%:
+	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+
+push-short:
+	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+
+
 build-%: DIST = $(*)
 build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
 
@@ -94,7 +103,10 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
-		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
+		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
+		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
+		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
 
@@ -102,24 +114,20 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
 build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
 
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
-build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-ubi8: PACKAGE_DIST = centos7
 
-build-centos%: BASE_DIST = $(*)
-build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: PACKAGE_DIST = $(BASE_DIST)
-build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-centos7: BASE_DIST = $(*)
+build-centos7: DOCKERFILE_SUFFIX := centos
+build-centos7: PACKAGE_DIST = $(BASE_DIST)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
@@ -135,18 +143,9 @@ $(TEST_TARGETS): test-%:
 test-packaging: DIST = packaging
 test-packaging:
 	@echo "Testing package image contents"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/aarch64" || echo "Missing centos7/aarch64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/multi-arch.mk

@@ -16,19 +16,11 @@ PUSH_ON_BUILD ?= false
 DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD)
 DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64
 
-REGCTL ?= regctl
-$(PUSH_TARGETS): push-%:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE)
-
-push-short:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)
-
 # We only have x86_64 packages for centos7
 build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
 
+# We only generate amd64 image for ubuntu18.04
+build-ubuntu18.04: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
 # We only generate a single image for packaging targets
 build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64

build/container/native-only.mk

@@ -13,11 +13,3 @@
 # limitations under the License.
 
 DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
-
-$(PUSH_TARGETS): push-%:
-	$(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)"
-	$(DOCKER) push "$(OUT_IMAGE)"
-
-push-short:
-	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"
-	$(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"

cmd/nvidia-container-runtime-hook/capabilities.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"log"
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -9,11 +9,11 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
-)
 
-var envSwarmGPU *string
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
 
 const (
 	envCUDAVersion          = "CUDA_VERSION"
@@ -23,6 +23,7 @@ const (
 	envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
 	envNVMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
 	envNVMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
+	envNVImexChannels       = "NVIDIA_IMEX_CHANNELS"
 	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
@@ -38,15 +39,18 @@ type nvidiaConfig struct {
 	Devices            string
 	MigConfigDevices   string
 	MigMonitorDevices  string
+	ImexChannels       string
 	DriverCapabilities string
-	Requirements       []string
-	DisableRequire     bool
+	// Requirements defines the requirements DSL for the container to run.
+	// This is empty if no specific requirements are needed, or if requirements are
+	// explicitly disabled.
+	Requirements []string
 }
 
 type containerConfig struct {
 	Pid    int
 	Rootfs string
-	Env    map[string]string
+	Image  image.CUDA
 	Nvidia *nvidiaConfig
 }
 
@@ -132,7 +136,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -141,67 +145,58 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		for _, c := range caps {
+			if c == capSysAdmin {
+				return true
+			}
 		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
+		return false
 	}
 
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	process := specs.Process{
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
-func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
-	// Build a list of envvars to consider.
-	envVars := []string{envNVVisibleDevices}
-	if envSwarmGPU != nil {
-		// The Swarm envvar has higher precedence.
-		envVars = append([]string{*envSwarmGPU}, envVars...)
-	}
-
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
-	for _, envVar := range envVars {
-		if devs, ok := env[envVar]; ok {
-			devices = &devs
+func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
+	// We check if the image has at least one of the Swarm resource envvars defined and use this
+	// if specified.
+	var hasSwarmEnvvar bool
+	for _, envvar := range swarmResourceEnvvars {
+		if _, exists := image[envvar]; exists {
+			hasSwarmEnvvar = true
 			break
 		}
 	}
 
-	// Environment variable unset with legacy image: default to "all".
-	if devices == nil && legacyImage {
-		all := "all"
-		return &all
+	var devices []string
+	if hasSwarmEnvvar {
+		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
+	} else {
+		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
 	}
 
-	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
+	if len(devices) == 0 {
 		return nil
 	}
 
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		empty := ""
-		return &empty
-	}
+	devicesString := strings.Join(devices, ",")
 
-	// Any other value.
-	return devices
+	return &devicesString
 }
 
 func getDevicesFromMounts(mounts []Mount) *string {
@@ -241,7 +236,7 @@ func getDevicesFromMounts(mounts []Mount) *string {
 	return &ret
 }
 
-func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
+func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
 		devices := getDevicesFromMounts(mounts)
@@ -251,7 +246,7 @@ func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, p
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(env, legacyImage)
+	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
 	if devices == nil {
 		return nil
 	}
@@ -279,10 +274,19 @@ func getMigMonitorDevices(env map[string]string) *string {
 	return nil
 }
 
-func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
+func getImexChannels(env map[string]string) *string {
+	if chans, ok := env[envNVImexChannels]; ok {
+		return &chans
+	}
+	return nil
+}
+
+func (c *HookConfig) getDriverCapabilities(env map[string]string, legacyImage bool) image.DriverCapabilities {
 	// We use the default driver capabilities by default. This is filtered to only include the
 	// supported capabilities
-	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
+	supportedDriverCapabilities := image.NewDriverCapabilities(c.SupportedDriverCapabilities)
+
+	capabilities := supportedDriverCapabilities.Intersection(image.DefaultDriverCapabilities)
 
 	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
 
@@ -293,9 +297,9 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr
 
 	if capsEnvSpecified && len(capsEnv) > 0 {
 		// If the envvironment variable is specified and is non-empty, use the capabilities value
-		envCapabilities := DriverCapabilities(capsEnv)
+		envCapabilities := image.NewDriverCapabilities(capsEnv)
 		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
-		if envCapabilities != all && capabilities != envCapabilities {
+		if !envCapabilities.IsAll() && len(capabilities) != len(envCapabilities) {
 			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
 		}
 	}
@@ -307,7 +311,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	legacyImage := image.IsLegacy()
 
 	var devices string
-	if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil {
+	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
 		devices = *d
 	} else {
 		// 'nil' devices means this is not a GPU container.
@@ -330,22 +334,25 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+	var imexChannels string
+	if c := getImexChannels(image); c != nil {
+		imexChannels = *c
+	}
+
+	driverCapabilities := hookConfig.getDriverCapabilities(image, legacyImage).String()
 
 	requirements, err := image.GetRequirements()
 	if err != nil {
 		log.Panicln("failed to get requirements", err)
 	}
 
-	disableRequire := image.HasDisableRequire()
-
 	return &nvidiaConfig{
 		Devices:            devices,
 		MigConfigDevices:   migConfigDevices,
 		MigMonitorDevices:  migMonitorDevices,
+		ImexChannels:       imexChannels,
 		DriverCapabilities: driverCapabilities,
 		Requirements:       requirements,
-		DisableRequire:     disableRequire,
 	}
 }
 
@@ -363,17 +370,19 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	image, err := image.New(
+		image.WithEnv(s.Process.Env),
+		image.WithDisableRequire(hook.DisableRequire),
+	)
 	if err != nil {
 		log.Panicln(err)
 	}
 
 	privileged := isPrivileged(s)
-	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    image,
+		Image:  image,
 		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"fmt"
 	"path/filepath"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -36,9 +38,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -50,9 +51,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -68,7 +68,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Legacy image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
+				envNVVisibleDevices: "void",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -82,9 +82,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -96,9 +95,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -111,9 +109,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -126,9 +123,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -141,9 +137,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -158,9 +153,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -176,9 +170,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -207,9 +200,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -225,7 +217,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Modern image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
+				envNVVisibleDevices: "void",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -239,9 +231,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -253,9 +244,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -268,9 +258,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -283,9 +272,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -298,9 +286,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -315,9 +302,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -333,9 +319,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -347,9 +332,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -363,9 +347,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigConfigDevices:   "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -389,9 +372,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigMonitorDevices:  "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -416,7 +398,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			},
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 			},
 		},
 		{
@@ -431,7 +413,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			},
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 			},
 		},
 		{
@@ -445,7 +427,39 @@ func TestGetNvidiaConfig(t *testing.T) {
 			},
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, swarmResource overrides device selection",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SwarmResource:               "DOCKER_SWARM_RESOURCE",
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "GPU1,GPU2",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, comma separated swarmResource is split and overrides device selection",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SwarmResource:               "NOT_DOCKER_SWARM_RESOURCE,DOCKER_SWARM_RESOURCE",
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "GPU1,GPU2",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 			},
 		},
 	}
@@ -456,7 +470,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			getConfig := func() {
 				hookConfig := tc.hookConfig
 				if hookConfig == nil {
-					defaultConfig := getDefaultHookConfig()
+					defaultConfig, _ := getDefaultHookConfig()
 					hookConfig = &defaultConfig
 				}
 				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
@@ -485,7 +499,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
 
 			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
-			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
 		})
 	}
 }
@@ -668,10 +681,10 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				env := map[string]string{
 					envNVVisibleDevices: tc.envvarDevices,
 				}
-				hookConfig := getDefaultHookConfig()
+				hookConfig, _ := getDefaultHookConfig()
 				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
+				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)
 			}
 
 			// For all other tests, just grab the devices and check the results
@@ -688,13 +701,13 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
 	gpuID := "GPU-12345"
 	anotherGPUID := "GPU-67890"
+	thirdGPUID := "MIG-12345"
 
 	var tests = []struct {
-		description     string
-		envSwarmGPU     *string
-		env             map[string]string
-		legacyImage     bool
-		expectedDevices *string
+		description          string
+		swarmResourceEnvvars []string
+		env                  map[string]string
+		expectedDevices      *string
 	}{
 		{
 			description: "empty env returns nil for non-legacy image",
@@ -729,13 +742,15 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
 			env: map[string]string{
 				envNVVisibleDevices: gpuID,
+				envCUDAVersion:      "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description:     "empty env returns all for legacy image",
-			legacyImage:     true,
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envCUDAVersion: "legacy",
+			},
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
@@ -781,86 +796,116 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
 			description: "empty env returns all for legacy image",
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
 		// enabled
 		{
-			description: "empty env returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "empty env returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 		},
 		{
-			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "",
 			},
 		},
 		{
-			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "void",
 			},
 		},
 		{
-			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "none",
 			},
 			expectedDevices: &empty,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
 			},
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS is selected if present",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS is selected if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
+		{
+			description:          "All available swarm resource envvars are selected and override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS":            thirdGPUID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: func() *string {
+				result := fmt.Sprintf("%s,%s", thirdGPUID, anotherGPUID)
+				return &result
+			}(),
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
 	}
 
 	for i, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			envSwarmGPU = tc.envSwarmGPU
-			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
+			devices := getDevicesFromEnvvar(image.CUDA(tc.env), tc.swarmResourceEnvvars)
 			if tc.expectedDevices == nil {
 				require.Nil(t, devices, "%d: %v", i, tc)
 				return
@@ -874,7 +919,7 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 
 func TestGetDriverCapabilities(t *testing.T) {
 
-	supportedCapabilities := "compute,utility,display,video"
+	supportedCapabilities := "compute,display,utility,video"
 
 	testCases := []struct {
 		description           string
@@ -909,7 +954,7 @@ func TestGetDriverCapabilities(t *testing.T) {
 			},
 			legacyImage:           true,
 			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
 		},
 		{
 			description:           "Env unset for legacy image is 'all'",
@@ -932,7 +977,7 @@ func TestGetDriverCapabilities(t *testing.T) {
 			env:                   map[string]string{},
 			legacyImage:           false,
 			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
 		},
 		{
 			description: "Env is all for modern image",
@@ -950,7 +995,7 @@ func TestGetDriverCapabilities(t *testing.T) {
 			},
 			legacyImage:           false,
 			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
 		},
 		{
 			description: "Invalid capabilities panic",
@@ -970,11 +1015,14 @@ func TestGetDriverCapabilities(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			var capabilites DriverCapabilities
+			var capabilites string
+
+			c := HookConfig{
+				SupportedDriverCapabilities: tc.supportedCapabilities,
+			}
 
 			getDriverCapabilities := func() {
-				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
-				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
+				capabilites = c.getDriverCapabilities(tc.env, tc.legacyImage).String()
 			}
 
 			if tc.expectedPanic {

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
+
+const (
+	configPath = "/etc/nvidia-container-runtime/config.toml"
+	driverPath = "/run/nvidia/driver"
+)
+
+var defaultPaths = [...]string{}
+
+// HookConfig : options for the nvidia-container-runtime-hook.
+type HookConfig config.Config
+
+func getDefaultHookConfig() (HookConfig, error) {
+	defaultCfg, err := config.GetDefault()
+	if err != nil {
+		return HookConfig{}, err
+	}
+
+	return *(*HookConfig)(defaultCfg), nil
+}
+
+// loadConfig loads the required paths for the hook config.
+func loadConfig() (*config.Config, error) {
+	var configPaths []string
+	var required bool
+	if len(*configflag) != 0 {
+		configPaths = append(configPaths, *configflag)
+		required = true
+	} else {
+		configPaths = append(configPaths, path.Join(driverPath, configPath), configPath)
+	}
+
+	for _, p := range configPaths {
+		cfg, err := config.New(
+			config.WithConfigFile(p),
+			config.WithRequired(true),
+		)
+		if err == nil {
+			return cfg.Config()
+		} else if os.IsNotExist(err) && !required {
+			continue
+		}
+		return nil, fmt.Errorf("couldn't open required configuration file: %v", err)
+	}
+
+	return config.GetDefault()
+}
+
+func getHookConfig() (*HookConfig, error) {
+	cfg, err := loadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config := (*HookConfig)(cfg)
+
+	allSupportedDriverCapabilities := image.SupportedDriverCapabilities
+	if config.SupportedDriverCapabilities == "all" {
+		config.SupportedDriverCapabilities = allSupportedDriverCapabilities.String()
+	}
+	configuredCapabilities := image.NewDriverCapabilities(config.SupportedDriverCapabilities)
+	// We ensure that the configured value is a subset of all supported capabilities
+	if !allSupportedDriverCapabilities.IsSuperset(configuredCapabilities) {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allSupportedDriverCapabilities.String())
+	}
+
+	return config, nil
+}
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c HookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}
+
+// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
+func (c *HookConfig) getSwarmResourceEnvvars() []string {
+	if c.SwarmResource == "" {
+		return nil
+	}
+
+	candidates := strings.Split(c.SwarmResource, ",")
+
+	var envvars []string
+	for _, c := range candidates {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) > 0 {
+			envvars = append(envvars, trimmed)
+		}
+	}
+
+	return envvars
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -28,16 +29,16 @@ func TestGetHookConfig(t *testing.T) {
 	testCases := []struct {
 		lines                      []string
 		expectedPanic              bool
-		expectedDriverCapabilities DriverCapabilities
+		expectedDriverCapabilities string
 	}{
 		{
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"all\"",
 			},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
@@ -47,19 +48,19 @@ func TestGetHookConfig(t *testing.T) {
 		},
 		{
 			lines:                      []string{},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"\"",
 			},
-			expectedDriverCapabilities: none,
+			expectedDriverCapabilities: "",
 		},
 		{
 			lines: []string{
-				"supported-driver-capabilities = \"utility,compute\"",
+				"supported-driver-capabilities = \"compute,utility\"",
 			},
-			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+			expectedDriverCapabilities: "compute,utility",
 		},
 	}
 
@@ -89,7 +90,8 @@ func TestGetHookConfig(t *testing.T) {
 
 			var config HookConfig
 			getHookConfig := func() {
-				config = getHookConfig()
+				c, _ := getHookConfig()
+				config = *c
 			}
 
 			if tc.expectedPanic {
@@ -103,3 +105,50 @@ func TestGetHookConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSwarmResourceEnvvars(t *testing.T) {
+	testCases := []struct {
+		value    string
+		expected []string
+	}{
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value:    " ",
+			expected: nil,
+		},
+		{
+			value:    "single",
+			expected: []string{"single"},
+		},
+		{
+			value:    "single ",
+			expected: []string{"single"},
+		},
+		{
+			value:    "one,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one ,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one, two",
+			expected: []string{"one", "two"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			c := &HookConfig{
+				SwarmResource: tc.value,
+			}
+
+			envvars := c.getSwarmResourceEnvvars()
+			require.EqualValues(t, tc.expected, envvars)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/main.go

@@ -13,7 +13,9 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
@@ -36,16 +38,12 @@ func exit() {
 	os.Exit(0)
 }
 
-func getCLIPath(config CLIConfig) string {
-	if config.Path != nil {
-		return *config.Path
+func getCLIPath(config config.ContainerCLIConfig) string {
+	if config.Path != "" {
+		return config.Path
 	}
 
-	var root string
-	if config.Root != nil {
-		root = *config.Root
-	}
-	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
+	if err := os.Setenv("PATH", lookup.GetPath(config.Root)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -71,25 +69,28 @@ func doPrestart() {
 	defer exit()
 	log.SetFlags(0)
 
-	hook := getHookConfig()
-	cli := hook.NvidiaContainerCLI
-
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+	hook, err := getHookConfig()
+	if err != nil || hook == nil {
+		log.Panicln("error getting hook config:", err)
 	}
+	cli := hook.NVIDIAContainerCLIConfig
 
-	container := getContainerConfig(hook)
+	container := getContainerConfig(*hook)
 	nvidia := container.Nvidia
 	if nvidia == nil {
 		// Not a GPU container, nothing to do.
 		return
 	}
 
+	if !hook.NVIDIAContainerRuntimeHookConfig.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntimeConfig.Mode, container.Image) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
+	}
+
 	rootfs := getRootfsPath(container)
 
 	args := []string{getCLIPath(cli)}
-	if cli.Root != nil {
-		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
+	if cli.Root != "" {
+		args = append(args, fmt.Sprintf("--root=%s", cli.Root))
 	}
 	if cli.LoadKmods {
 		args = append(args, "--load-kmods")
@@ -99,19 +100,19 @@ func doPrestart() {
 	}
 	if *debugflag {
 		args = append(args, "--debug=/dev/stderr")
-	} else if cli.Debug != nil {
-		args = append(args, fmt.Sprintf("--debug=%s", *cli.Debug))
+	} else if cli.Debug != "" {
+		args = append(args, fmt.Sprintf("--debug=%s", cli.Debug))
 	}
-	if cli.Ldcache != nil {
-		args = append(args, fmt.Sprintf("--ldcache=%s", *cli.Ldcache))
+	if cli.Ldcache != "" {
+		args = append(args, fmt.Sprintf("--ldcache=%s", cli.Ldcache))
 	}
-	if cli.User != nil {
-		args = append(args, fmt.Sprintf("--user=%s", *cli.User))
+	if cli.User != "" {
+		args = append(args, fmt.Sprintf("--user=%s", cli.User))
 	}
 	args = append(args, "configure")
 
-	if cli.Ldconfig != nil {
-		args = append(args, fmt.Sprintf("--ldconfig=%s", *cli.Ldconfig))
+	if cli.Ldconfig != "" {
+		args = append(args, fmt.Sprintf("--ldconfig=%s", cli.Ldconfig))
 	}
 	if cli.NoCgroups {
 		args = append(args, "--no-cgroups")
@@ -125,6 +126,9 @@ func doPrestart() {
 	if len(nvidia.MigMonitorDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
 	}
+	if len(nvidia.ImexChannels) > 0 {
+		args = append(args, fmt.Sprintf("--imex-channel=%s", nvidia.ImexChannels))
+	}
 
 	for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
 		if len(cap) == 0 {
@@ -133,10 +137,8 @@ func doPrestart() {
 		args = append(args, capabilityToCLI(cap))
 	}
 
-	if !hook.DisableRequire && !nvidia.DisableRequire {
-		for _, req := range nvidia.Requirements {
-			args = append(args, fmt.Sprintf("--require=%s", req))
-		}
+	for _, req := range nvidia.Requirements {
+		args = append(args, fmt.Sprintf("--require=%s", req))
 	}
 
 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
@@ -185,11 +187,11 @@ func main() {
 	}
 }
 
-// logInterceptor implements the info.Logger interface to allow for logging from this function.
-type logInterceptor struct{}
+// logInterceptor implements the logger.Interface to allow for logging from executable.
+type logInterceptor struct {
+	logger.NullLogger
+}
 
 func (l *logInterceptor) Infof(format string, args ...interface{}) {
 	log.Printf(format, args...)
 }
-
-func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/README.md

@@ -2,24 +2,86 @@
 
 The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
 
-## Standard Mode
+## Configuration
 
-In the standard mode configuration, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from project [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+The NVIDIA Container Runtime uses file-based configuration, with the config stored in `/etc/nvidia-container-runtime/config.toml`. The `/etc` path can be overridden using the `XDG_CONFIG_HOME` environment variable with the `${XDG_CONFIG_HOME}/nvidia-container-runtime/config.toml` file used instead if this environment variable is set.
 
-## Experimental Mode
+This config file may contain options for other components of the NVIDIA container stack and for the NVIDIA Container Runtime, the relevant config section is `nvidia-container-runtime`
 
-The NVIDIA Container Runtime can be configured in an experimental mode by setting the following options in the runtime's `config.toml` file:
+### Logging
+
+The `log-level` config option (default: `"info"`) specifies the log level to use and the `debug` option, if set, specifies a log file to which logs for the NVIDIA Container Runtime must be written.
+
+In addition to this, the NVIDIA Container Runtime considers the value of `--log` and `--log-format` flags that may be passed to it by a container runtime such as docker or containerd. If the `--debug` flag is present the log-level specified in the config file is overridden as `"debug"`.
+
+### Low-level Runtime Path
+
+The `runtimes` config option allows for the low-level runtime to be specified. The first entry in this list that is an existing executable file is used as the low-level runtime. If the entry is not a path, the `PATH` is searched for a matching executable. If the entry is a path this is checked instead.
+
+The default value for this setting is:
+```toml
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+```
+
+and if, for example, `crun` is to be used instead this can be changed to:
+```toml
+runtimes = [
+    "crun",
+]
+```
+
+### Runtime Mode
+
+The `mode` config option (default `"auto"`) controls the high-level behaviour of the runtime.
+
+#### Auto Mode
+
+When `mode` is set to `"auto"`, the runtime employs heuristics to determine which mode to use based on, for example, the platform where the runtime is being run.
+
+#### Legacy Mode
+
+When `mode` is set to `"legacy"`, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from the [libnvidia-container](https://github.com/NVIDIA/libnvidia-container) project.
+
+#### CSV Mode
+
+When `mode` is set to `"csv"`, CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` define the devices and mounts that are to be injected into a container when it is created. The search path for the files can be overridden by modifying the `nvidia-container-runtime.modes.csv.mount-spec-path` in the config as below:
 
 ```toml
 [nvidia-container-runtime]
-experimental = true
+    [nvidia-container-runtime.modes.csv]
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
 ```
 
-When this setting is enabled, the modifications made to the OCI specification are controlled by the `nvidia-container-runtime.discover-mode` option, with the following mode supported:
-
-* `"legacy"`: This mode mirrors the behaviour of the standard mode, inserting the NVIDIA Container Runtime Hook as a `prestart` hook into the container's OCI specification.
-* `"csv"`: This mode uses CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` to define the devices and mounts that are to be injected into a container when it is created.
+This mode is primarily targeted at Tegra-based systems without NVML available.
 
 ### Notes on using the docker CLI
 
-The `docker` CLI supports the `--gpus` flag to select GPUs for inclusion in a container. Since specifying this flag inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification. When experimental mode is activated, the NVIDIA Container Runtime detects the presence of the hook and raises an error. This requirement will be relaxed in the near future.
+Note that only the `"legacy"` NVIDIA Container Runtime mode is directly compatible with the `--gpus` flag implemented by the `docker` CLI (assuming the NVIDIA Container Runtime is not used). The reason for this is that `docker` inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification.
+
+
+If a different mode is explicitly set or detected, the NVIDIA Container Runtime Hook will raise the following error when `--gpus` is set:
+```
+$ docker run --rm --gpus all ubuntu:18.04
+docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'csv'
+invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.: unknown.
+```
+Here NVIDIA Container Runtime must be used explicitly. The recommended way to do this is to specify the `--runtime=nvidia` command line argument as part of the `docker run` commmand as follows:
+```
+$ docker run --rm --gpus all --runtime=nvidia ubuntu:18.04
+```
+
+Alternatively the NVIDIA Container Runtime can be set as the default runtime for docker. This can be done by modifying the `/etc/docker/daemon.json` file as follows:
+```json
+{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}
+```

cmd/nvidia-container-runtime/main.go

@@ -1,84 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer logger.Reset()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -4,15 +4,17 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,7 +43,7 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		log.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
@@ -53,11 +55,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,7 +79,7 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -91,9 +93,10 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -170,7 +173,8 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	logger, _ := testlog.NewNullLogger()
+	m := modifier.NewStableRuntimeModifier(logger, nvidiaHook)
 	return m.Modify(spec)
 }
 

cmd/nvidia-container-runtime/modifier/csv.go

@@ -1,166 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/cuda"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
-)
-
-// csvMode represents the modifications as performed by the csv runtime mode
-type csvMode struct {
-	logger     *logrus.Logger
-	discoverer discover.Discover
-}
-
-const (
-	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
-	visibleDevicesVoid   = "void"
-
-	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
-)
-
-// NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
-// The modifications are defined by CSV MountSpecs.
-func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	rawSpec, err := ociSpec.Load()
-	if err != nil {
-		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
-	// if no modification is required.
-	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
-	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
-		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
-		return nil, nil
-	}
-	logger.Infof("Constructing modifier from config: %+v", *cfg)
-
-	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
-	}
-
-	// TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the
-	// visible devices are checked.
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := checkRequirements(logger, &image); err != nil {
-		return nil, fmt.Errorf("requirements not met: %v", err)
-	}
-
-	csvFiles, err := csv.GetFileList(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.MountSpecPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
-	}
-
-	nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
-	if nvidiaRequireJetpack != "csv-mounts=all" {
-		csvFiles = csv.BaseFilesOnly(csvFiles)
-	}
-
-	csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
-	}
-
-	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
-	}
-
-	createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
-	}
-
-	d := discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
-
-	return newModifierFromDiscoverer(logger, d)
-}
-
-// newModifierFromDiscoverer created a modifier that aplies the discovered
-// modifications to an OCI spec if require by the runtime wrapper.
-func newModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
-	m := csvMode{
-		logger:     logger,
-		discoverer: d,
-	}
-	return &m, nil
-}
-
-// Modify applies the required modifications to the incomming OCI spec. These modifications
-// are applied in-place.
-func (m csvMode) Modify(spec *specs.Spec) error {
-	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
-	if err != nil {
-		return fmt.Errorf("failed to remove existing hooks: %v", err)
-	}
-
-	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
-	if err != nil {
-		return fmt.Errorf("failed to get required container edits: %v", err)
-	}
-
-	return specEdits.Modify(spec)
-}
-
-func checkRequirements(logger *logrus.Logger, image *image.CUDA) error {
-	if image.HasDisableRequire() {
-		// TODO: We could print the real value here instead
-		logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true)
-		return nil
-	}
-
-	imageRequirements, err := image.GetRequirements()
-	if err != nil {
-		//  TODO: Should we treat this as a failure, or just issue a warning?
-		return fmt.Errorf("failed to get image requirements: %v", err)
-	}
-
-	r := requirements.New(logger, imageRequirements)
-
-	cudaVersion, err := cuda.Version()
-	if err != nil {
-		logger.Warnf("Failed to get CUDA version: %v", err)
-	} else {
-		r.AddVersionProperty(requirements.CUDA, cudaVersion)
-	}
-
-	compteCapability, err := cuda.ComputeCapability(0)
-	if err != nil {
-		logger.Warnf("Failed to get CUDA Compute Capability: %v", err)
-	} else {
-		r.AddVersionProperty(requirements.ARCH, compteCapability)
-	}
-
-	return r.Assert()
-}

cmd/nvidia-container-runtime/modifier/csv_test.go

@@ -1,284 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewCSVModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description    string
-		cfg            *config.Config
-		spec           oci.Spec
-		visibleDevices string
-		expectedError  error
-		expectedNil    bool
-	}{
-		{
-			description: "spec load error returns error",
-			spec: &oci.SpecMock{
-				LoadFunc: func() (*specs.Spec, error) {
-					return nil, fmt.Errorf("load failed")
-				},
-			},
-			expectedError: fmt.Errorf("load failed"),
-		},
-		{
-			description:    "visible devices not set returns nil",
-			visibleDevices: "NOT_SET",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices empty returns nil",
-			visibleDevices: "",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices 'void' returns nil",
-			visibleDevices: "void",
-			expectedNil:    true,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			spec := tc.spec
-			if spec == nil {
-				spec = &oci.SpecMock{
-					LookupEnvFunc: func(s string) (string, bool) {
-						if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
-							return tc.visibleDevices, true
-						}
-						return "", false
-					},
-				}
-			}
-
-			m, err := NewCSVModifier(logger, tc.cfg, spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			if tc.expectedNil || tc.expectedError != nil {
-				require.Nil(t, m)
-			} else {
-				require.NotNil(t, m)
-			}
-		})
-	}
-}
-
-func TestExperimentalModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description   string
-		discover      *discover.DiscoverMock
-		spec          *specs.Spec
-		expectedError error
-		expectedSpec  *specs.Spec
-	}{
-		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
-		},
-		{
-			description: "failed hooks discoverer returns error",
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					return nil, fmt.Errorf("discover.Hooks error")
-				},
-			},
-			expectedError: fmt.Errorf("discover.Hooks error"),
-		},
-		{
-			description: "discovered hooks are injected into spec",
-			spec:        &specs.Spec{},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/a",
-							Args:      []string{"/hook/a", "arga"},
-						},
-						{
-							Lifecycle: "createContainer",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-					CreateContainer: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "existing hooks are maintained",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-runtime-hook",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-runtime-hook",
-							Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-toolkit",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-toolkit",
-							Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			m, err := newModifierFromDiscoverer(logger, tc.discover)
-			require.NoError(t, err)
-
-			err = m.Modify(tc.spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			require.EqualValues(t, tc.expectedSpec, tc.spec)
-		})
-	}
-}

cmd/nvidia-container-toolkit/capabilities.go

@@ -1,83 +0,0 @@
-package main
-
-import (
-	"log"
-	"strings"
-)
-
-const (
-	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
-	defaultDriverCapabilities = DriverCapabilities("utility,compute")
-
-	none = DriverCapabilities("")
-	all  = DriverCapabilities("all")
-)
-
-func capabilityToCLI(cap string) string {
-	switch cap {
-	case "compute":
-		return "--compute"
-	case "compat32":
-		return "--compat32"
-	case "graphics":
-		return "--graphics"
-	case "utility":
-		return "--utility"
-	case "video":
-		return "--video"
-	case "display":
-		return "--display"
-	case "ngx":
-		return "--ngx"
-	default:
-		log.Panicln("unknown driver capability:", cap)
-	}
-	return ""
-}
-
-// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
-// variable. Operations include default values, filtering, and handling meta values such as "all"
-type DriverCapabilities string
-
-// Intersection returns intersection between two sets of capabilities.
-func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
-	if capabilities == all {
-		return d
-	}
-	if d == all {
-		return capabilities
-	}
-
-	lookup := make(map[string]bool)
-	for _, c := range d.list() {
-		lookup[c] = true
-	}
-	var found []string
-	for _, c := range capabilities.list() {
-		if lookup[c] {
-			found = append(found, c)
-		}
-	}
-
-	intersection := DriverCapabilities(strings.Join(found, ","))
-	return intersection
-}
-
-// String returns the string representation of the driver capabilities
-func (d DriverCapabilities) String() string {
-	return string(d)
-}
-
-// list returns the driver capabilities as a list
-func (d DriverCapabilities) list() []string {
-	var caps []string
-	for _, c := range strings.Split(string(d), ",") {
-		trimmed := strings.TrimSpace(c)
-		if len(trimmed) == 0 {
-			continue
-		}
-		caps = append(caps, trimmed)
-	}
-
-	return caps
-}

cmd/nvidia-container-toolkit/capabilities_test.go

@@ -1,134 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestDriverCapabilitiesIntersection(t *testing.T) {
-	testCases := []struct {
-		capabilities          DriverCapabilities
-		supportedCapabilities DriverCapabilities
-		expectedIntersection  DriverCapabilities
-	}{
-		{
-			capabilities:          none,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          allDriverCapabilities,
-			supportedCapabilities: all,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: all,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: DriverCapabilities("cap1"),
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("cap0,cap1"),
-			supportedCapabilities: DriverCapabilities("cap1,cap0"),
-			expectedIntersection:  DriverCapabilities("cap0,cap1"),
-		},
-		{
-			capabilities:          defaultDriverCapabilities,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  defaultDriverCapabilities,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-		{
-			capabilities:          DriverCapabilities("cap1"),
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
-			require.EqualValues(t, tc.expectedIntersection, intersection)
-		})
-	}
-}
-
-func TestDriverCapabilitiesList(t *testing.T) {
-	testCases := []struct {
-		capabilities DriverCapabilities
-		expected     []string
-	}{
-		{
-			capabilities: DriverCapabilities(""),
-		},
-		{
-			capabilities: DriverCapabilities("  "),
-		},
-		{
-			capabilities: DriverCapabilities(","),
-		},
-		{
-			capabilities: DriverCapabilities(",cap"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap,"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap0,,cap1"),
-			expected:     []string{"cap0", "cap1"},
-		},
-		{
-			capabilities: DriverCapabilities("cap1,cap0,cap3"),
-			expected:     []string{"cap1", "cap0", "cap3"},
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			require.EqualValues(t, tc.expected, tc.capabilities.list())
-		})
-	}
-}

cmd/nvidia-container-toolkit/hook_config.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"log"
-	"os"
-	"path"
-	"reflect"
-
-	"github.com/BurntSushi/toml"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-)
-
-const (
-	configPath = "/etc/nvidia-container-runtime/config.toml"
-	driverPath = "/run/nvidia/driver"
-)
-
-var defaultPaths = [...]string{
-	path.Join(driverPath, configPath),
-	configPath,
-}
-
-// CLIConfig : options for nvidia-container-cli.
-type CLIConfig struct {
-	Root        *string  `toml:"root"`
-	Path        *string  `toml:"path"`
-	Environment []string `toml:"environment"`
-	Debug       *string  `toml:"debug"`
-	Ldcache     *string  `toml:"ldcache"`
-	LoadKmods   bool     `toml:"load-kmods"`
-	NoPivot     bool     `toml:"no-pivot"`
-	NoCgroups   bool     `toml:"no-cgroups"`
-	User        *string  `toml:"user"`
-	Ldconfig    *string  `toml:"ldconfig"`
-}
-
-// HookConfig : options for the nvidia-container-toolkit.
-type HookConfig struct {
-	DisableRequire                 bool               `toml:"disable-require"`
-	SwarmResource                  *string            `toml:"swarm-resource"`
-	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
-	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
-
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-func getDefaultHookConfig() HookConfig {
-	return HookConfig{
-		DisableRequire:                 false,
-		SwarmResource:                  nil,
-		AcceptEnvvarUnprivileged:       true,
-		AcceptDeviceListAsVolumeMounts: false,
-		SupportedDriverCapabilities:    allDriverCapabilities,
-		NvidiaContainerCLI: CLIConfig{
-			Root:        nil,
-			Path:        nil,
-			Environment: []string{},
-			Debug:       nil,
-			Ldcache:     nil,
-			LoadKmods:   true,
-			NoPivot:     false,
-			NoCgroups:   false,
-			User:        nil,
-			Ldconfig:    nil,
-		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
-	}
-}
-
-func getHookConfig() (config HookConfig) {
-	var err error
-
-	if len(*configflag) > 0 {
-		config = getDefaultHookConfig()
-		_, err = toml.DecodeFile(*configflag, &config)
-		if err != nil {
-			log.Panicln("couldn't open configuration file:", err)
-		}
-	} else {
-		for _, p := range defaultPaths {
-			config = getDefaultHookConfig()
-			_, err = toml.DecodeFile(p, &config)
-			if err == nil {
-				break
-			} else if !os.IsNotExist(err) {
-				log.Panicln("couldn't open default configuration file:", err)
-			}
-		}
-	}
-
-	if config.SupportedDriverCapabilities == all {
-		config.SupportedDriverCapabilities = allDriverCapabilities
-	}
-	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
-	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
-		configName := config.getConfigOption("SupportedDriverCapabilities")
-		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
-	}
-
-	return config
-}
-
-// getConfigOption returns the toml config option associated with the
-// specified struct field.
-func (c HookConfig) getConfigOption(fieldName string) string {
-	t := reflect.TypeOf(c)
-	f, ok := t.FieldByName(fieldName)
-	if !ok {
-		return fieldName
-	}
-	v, ok := f.Tag.Lookup("toml")
-	if !ok {
-		return fieldName
-	}
-	return v
-}

cmd/nvidia-ctk/README.md

@@ -1,3 +1,73 @@
 # NVIDIA Container Toolkit CLI
 
 The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.
+
+## Configure the NVIDIA Container Toolkit
+
+The `config` command of the `nvidia-ctk` CLI allows a user to display and manipulate the NVIDIA Container Toolkit
+configuration.
+
+For example, running the following command:
+```bash
+nvidia-ctk config default
+```
+will display the default config for the detected platform.
+
+Whereas
+```bash
+nvidia-ctk config
+```
+will display the effective NVIDIA Container Toolkit config using the configured config file, and running:
+
+Individual config options can be set by specifying these are key-value pairs to the `--set` argument:
+
+```bash
+nvidia-ctk config --set nvidia-container-cli.no-cgroups=true
+```
+
+By default, all commands output to `STDOUT`, but specifying the `--output` flag writes the config to the specified file.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,54 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/list"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
+		list.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,273 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+)
+
+const (
+	allDeviceName = "all"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	output             string
+	format             string
+	deviceNameStrategy string
+	driverRoot         string
+	nvidiaCTKPath      string
+	mode               string
+	vendor             string
+	class              string
+
+	librarySearchPaths cli.StringSlice
+
+	csv struct {
+		files          cli.StringSlice
+		ignorePatterns cli.StringSlice
+	}
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       spec.FormatYAML,
+			Destination: &opts.format,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Aliases:     []string{"discovery-mode"},
+			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       nvcdi.ModeAuto,
+			Destination: &opts.mode,
+		},
+		&cli.StringFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
+			Value:       nvcdi.DeviceNameStrategyIndex,
+			Destination: &opts.deviceNameStrategy,
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.driverRoot,
+		},
+		&cli.StringSliceFlag{
+			Name:        "library-search-path",
+			Usage:       "Specify the path to search for libraries when discovering the entities that should be included in the CDI specification.\n\tNote: This option only applies to CSV mode.",
+			Destination: &opts.librarySearchPaths,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-ctk-path",
+			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
+			Destination: &opts.nvidiaCTKPath,
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &opts.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &opts.class,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.file",
+			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CSV mode.",
+			Value:       cli.NewStringSlice(csv.DefaultFileList()...),
+			Destination: &opts.csv.files,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.ignore-pattern",
+			Usage:       "Specify a pattern the CSV mount specifications.",
+			Destination: &opts.csv.ignorePatterns,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	opts.format = strings.ToLower(opts.format)
+	switch opts.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", opts.format)
+	}
+
+	opts.mode = strings.ToLower(opts.mode)
+	switch opts.mode {
+	case nvcdi.ModeAuto:
+	case nvcdi.ModeCSV:
+	case nvcdi.ModeNvml:
+	case nvcdi.ModeWsl:
+	case nvcdi.ModeManagement:
+	default:
+		return fmt.Errorf("invalid discovery mode: %v", opts.mode)
+	}
+
+	_, err := nvcdi.NewDeviceNamer(opts.deviceNameStrategy)
+	if err != nil {
+		return err
+	}
+
+	opts.nvidiaCTKPath = config.ResolveNVIDIACTKPath(m.logger, opts.nvidiaCTKPath)
+
+	if outputFileFormat := formatFromFilename(opts.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			opts.format = outputFileFormat
+		} else if outputFileFormat != opts.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", opts.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(opts.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(opts.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := m.generateSpec(opts)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
+
+	if opts.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return spec.Save(opts.output)
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
+	}
+
+	return ""
+}
+
+func (m command) generateSpec(opts *options) (spec.Interface, error) {
+	deviceNamer, err := nvcdi.NewDeviceNamer(opts.deviceNameStrategy)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device namer: %v", err)
+	}
+
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(opts.driverRoot),
+		nvcdi.WithNVIDIACTKPath(opts.nvidiaCTKPath),
+		nvcdi.WithDeviceNamer(deviceNamer),
+		nvcdi.WithMode(string(opts.mode)),
+		nvcdi.WithLibrarySearchPaths(opts.librarySearchPaths.Value()),
+		nvcdi.WithCSVFiles(opts.csv.files.Value()),
+		nvcdi.WithCSVIgnorePatterns(opts.csv.ignorePatterns.Value()),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
+	}
+
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+
+	commonEdits, err := cdilib.GetCommonEdits()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
+	}
+
+	return spec.New(
+		spec.WithVendor(opts.vendor),
+		spec.WithClass(opts.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(opts.format),
+		spec.WithMergedDeviceOptions(
+			transform.WithName(allDeviceName),
+			transform.WithSkipIfExists(true),
+		),
+		spec.WithPermissions(0644),
+	)
+}

cmd/nvidia-ctk/cdi/list/list.go

@@ -0,0 +1,87 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package list
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct{}
+
+// NewCommand constructs a cdi list command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the command
+	c := cli.Command{
+		Name:  "list",
+		Usage: "List the available CDI devices",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	registry, err := cdi.NewCache(
+		cdi.WithAutoRefresh(false),
+		cdi.WithSpecDirs(cdi.DefaultSpecDirs...),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI cache: %v", err)
+	}
+
+	refreshErr := registry.Refresh()
+	devices := registry.ListDevices()
+	m.logger.Infof("Found %d CDI devices", len(devices))
+	if refreshErr != nil {
+		m.logger.Warningf("Refreshing the CDI registry returned the following error(s): %v", refreshErr)
+	}
+	for _, device := range devices {
+		fmt.Printf("%s\n", device)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,160 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+)
+
+type loadSaver interface {
+	Load() (spec.Interface, error)
+	Save(spec.Interface) error
+}
+
+type command struct {
+	logger logger.Interface
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from string
+	to   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transform.NewRootTransformer(
+		opts.from,
+		opts.to,
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/config/config.go

@@ -0,0 +1,213 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	createdefault "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/create-default"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// options stores the subcommand options
+type options struct {
+	flags.Options
+	sets cli.StringSlice
+}
+
+// NewCommand constructs an config command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'config' command
+	c := cli.Command{
+		Name:  "config",
+		Usage: "Interact with the NVIDIA Container Toolkit configuration",
+		Action: func(ctx *cli.Context) error {
+			return run(ctx, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config-file",
+			Aliases:     []string{"config", "c"},
+			Usage:       "Specify the config file to modify.",
+			Value:       config.GetConfigFilePath(),
+			Destination: &opts.Config,
+		},
+		&cli.StringSliceFlag{
+			Name:        "set",
+			Usage:       "Set a config value using the pattern key=value. If value is empty, this is equivalent to specifying the same key in unset. This flag can be specified multiple times",
+			Destination: &opts.sets,
+		},
+		&cli.BoolFlag{
+			Name:        "in-place",
+			Aliases:     []string{"i"},
+			Usage:       "Modify the config file in-place",
+			Destination: &opts.InPlace,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	c.Subcommands = []*cli.Command{
+		createdefault.NewCommand(m.logger),
+	}
+
+	return &c
+}
+
+func run(c *cli.Context, opts *options) error {
+	cfgToml, err := config.New(
+		config.WithConfigFile(opts.Config),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create config: %v", err)
+	}
+
+	for _, set := range opts.sets.Value() {
+		key, value, err := setFlagToKeyValue(set)
+		if err != nil {
+			return fmt.Errorf("invalid --set option %v: %w", set, err)
+		}
+		cfgToml.Set(key, value)
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	if err != nil {
+		return err
+	}
+	cfgToml.Save(output)
+	return nil
+}
+
+var errInvalidConfigOption = errors.New("invalid config option")
+var errUndefinedField = errors.New("undefined field")
+var errInvalidFormat = errors.New("invalid format")
+
+// setFlagToKeyValue converts a --set flag to a key-value pair.
+// The set flag is of the form key[=value], with the value being optional if key refers to a
+// boolean config option.
+func setFlagToKeyValue(setFlag string) (string, interface{}, error) {
+	setParts := strings.SplitN(setFlag, "=", 2)
+	key := setParts[0]
+
+	field, err := getField(key)
+	if err != nil {
+		return key, nil, fmt.Errorf("%w: %w", errInvalidConfigOption, err)
+	}
+
+	kind := field.Kind()
+	if len(setParts) != 2 {
+		if kind == reflect.Bool {
+			return key, true, nil
+		}
+		return key, nil, fmt.Errorf("%w: expected key=value; got %v", errInvalidFormat, setFlag)
+	}
+
+	value := setParts[1]
+	switch kind {
+	case reflect.Bool:
+		b, err := strconv.ParseBool(value)
+		if err != nil {
+			return key, value, fmt.Errorf("%w: %w", errInvalidFormat, err)
+		}
+		return key, b, err
+	case reflect.String:
+		return key, value, nil
+	case reflect.Slice:
+		valueParts := strings.Split(value, ",")
+		switch field.Elem().Kind() {
+		case reflect.String:
+			return key, valueParts, nil
+		case reflect.Int:
+			var output []int64
+			for _, v := range valueParts {
+				vi, err := strconv.ParseInt(v, 10, 0)
+				if err != nil {
+					return key, nil, fmt.Errorf("%w: %w", errInvalidFormat, err)
+				}
+				output = append(output, vi)
+			}
+			return key, output, nil
+		}
+	}
+	return key, nil, fmt.Errorf("unsupported type for %v (%v)", setParts, kind)
+}
+
+func getField(key string) (reflect.Type, error) {
+	s, err := getStruct(reflect.TypeOf(config.Config{}), strings.Split(key, ".")...)
+	if err != nil {
+		return nil, err
+	}
+	return s.Type, err
+}
+
+func getStruct(current reflect.Type, paths ...string) (reflect.StructField, error) {
+	if len(paths) < 1 {
+		return reflect.StructField{}, fmt.Errorf("%w: no fields selected", errUndefinedField)
+	}
+	tomlField := paths[0]
+	for i := 0; i < current.NumField(); i++ {
+		f := current.Field(i)
+		v, ok := f.Tag.Lookup("toml")
+		if !ok {
+			continue
+		}
+		if v != tomlField {
+			continue
+		}
+		if len(paths) == 1 {
+			return f, nil
+		}
+		return getStruct(f.Type, paths[1:]...)
+	}
+	return reflect.StructField{}, fmt.Errorf("%w: %q", errUndefinedField, tomlField)
+}

cmd/nvidia-ctk/config/config_test.go

@@ -0,0 +1,130 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetFlagToKeyValue(t *testing.T) {
+	// TODO: We need to enable this test again since switching to reflect.
+	testCases := []struct {
+		description   string
+		setFlag       string
+		expectedKey   string
+		expectedValue interface{}
+		expectedError error
+	}{
+		{
+			description:   "option not present returns an error",
+			setFlag:       "undefined=new-value",
+			expectedKey:   "undefined",
+			expectedError: errInvalidConfigOption,
+		},
+		{
+			description:   "undefined nexted option returns error",
+			setFlag:       "nvidia-container-cli.undefined",
+			expectedKey:   "nvidia-container-cli.undefined",
+			expectedError: errInvalidConfigOption,
+		},
+		{
+			description:   "boolean option assumes true",
+			setFlag:       "disable-require",
+			expectedKey:   "disable-require",
+			expectedValue: true,
+		},
+		{
+			description:   "boolean option returns true",
+			setFlag:       "disable-require=true",
+			expectedKey:   "disable-require",
+			expectedValue: true,
+		},
+		{
+			description:   "boolean option returns false",
+			setFlag:       "disable-require=false",
+			expectedKey:   "disable-require",
+			expectedValue: false,
+		},
+		{
+			description:   "invalid boolean option returns error",
+			setFlag:       "disable-require=something",
+			expectedKey:   "disable-require",
+			expectedValue: "something",
+			expectedError: errInvalidFormat,
+		},
+		{
+			description:   "string option requires value",
+			setFlag:       "swarm-resource",
+			expectedKey:   "swarm-resource",
+			expectedValue: nil,
+			expectedError: errInvalidFormat,
+		},
+		{
+			description:   "string option returns value",
+			setFlag:       "swarm-resource=string-value",
+			expectedKey:   "swarm-resource",
+			expectedValue: "string-value",
+		},
+		{
+			description:   "string option returns value with equals",
+			setFlag:       "swarm-resource=string-value=more",
+			expectedKey:   "swarm-resource",
+			expectedValue: "string-value=more",
+		},
+		{
+			description:   "string option treats bool value as string",
+			setFlag:       "swarm-resource=true",
+			expectedKey:   "swarm-resource",
+			expectedValue: "true",
+		},
+		{
+			description:   "string option treats int value as string",
+			setFlag:       "swarm-resource=5",
+			expectedKey:   "swarm-resource",
+			expectedValue: "5",
+		},
+		{
+			description:   "[]string option returns single value",
+			setFlag:       "nvidia-container-cli.environment=string-value",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"string-value"},
+		},
+		{
+			description:   "[]string option returns multiple values",
+			setFlag:       "nvidia-container-cli.environment=first,second",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"first", "second"},
+		},
+		{
+			description:   "[]string option returns values with equals",
+			setFlag:       "nvidia-container-cli.environment=first=1,second=2",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"first=1", "second=2"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			k, v, err := setFlagToKeyValue(tc.setFlag)
+			require.ErrorIs(t, err, tc.expectedError)
+			require.EqualValues(t, tc.expectedKey, k)
+			require.EqualValues(t, tc.expectedValue, v)
+		})
+	}
+}

cmd/nvidia-ctk/config/create-default/create-default.go

@@ -0,0 +1,94 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package defaultsubcommand
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a default command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := flags.Options{}
+
+	// Create the 'default' command
+	c := cli.Command{
+		Name:    "default",
+		Aliases: []string{"create-default", "generate-default"},
+		Usage:   "Generate the default NVIDIA Container Toolkit configuration file",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *flags.Options) error {
+	return opts.Validate()
+}
+
+func (m command) run(c *cli.Context, opts *flags.Options) error {
+	cfgToml, err := config.New()
+	if err != nil {
+		return fmt.Errorf("unable to load or create config: %v", err)
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	_, err = cfgToml.Save(output)
+	if err != nil {
+		return fmt.Errorf("failed to write output: %v", err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/config/flags/options.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package flags
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Options stores options for the config commands
+type Options struct {
+	Config  string
+	Output  string
+	InPlace bool
+}
+
+// Validate checks whether the options are valid.
+func (o Options) Validate() error {
+	if o.InPlace && o.Output != "" {
+		return fmt.Errorf("cannot specify both --in-place and --output")
+	}
+
+	return nil
+}
+
+// GetOutput returns the effective output
+func (o Options) GetOutput() string {
+	if o.InPlace {
+		return o.Config
+	}
+
+	return o.Output
+}
+
+// EnsureOutputFolder creates the output folder if it does not exist.
+// If the output folder is not specified (i.e. output to STDOUT), it is ignored.
+func (o Options) EnsureOutputFolder() error {
+	output := o.GetOutput()
+	if output == "" {
+		return nil
+	}
+	if dir := filepath.Dir(output); dir != "" {
+		return os.MkdirAll(dir, 0755)
+	}
+	return nil
+}
+
+// CreateOutput creates the writer for the output.
+func (o Options) CreateOutput() (io.WriteCloser, error) {
+	output := o.GetOutput()
+	if output == "" {
+		return nullCloser{os.Stdout}, nil
+	}
+
+	return os.Create(output)
+}
+
+// nullCloser is a writer that does nothing on Close.
+type nullCloser struct {
+	io.Writer
+}
+
+// Close is a no-op for a nullCloser.
+func (d nullCloser) Close() error {
+	return nil
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -0,0 +1,146 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	paths         cli.StringSlice
+	mode          string
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specifiy a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.mode) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value())
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	locator := lookup.NewExecutableLocator(m.logger, "")
+	targets, err := locator.Locate("chmod")
+	if err != nil {
+		return fmt.Errorf("failed to locate chmod: %v", err)
+	}
+	chmodPath := targets[0]
+
+	args := append([]string{filepath.Base(chmodPath), cfg.mode}, paths...)
+
+	return syscall.Exec(chmodPath, args, nil)
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		path := filepath.Join(root, f)
+		if _, err := os.Stat(path); err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -22,15 +22,16 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -41,7 +42,7 @@ type config struct {
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -74,7 +75,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringSliceFlag{
 			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as source:target",
+			Usage:       "Specify a specific link to create. The link is specified as target::link",
 			Destination: &cfg.links,
 		},
 		&cli.StringFlag{
@@ -100,7 +101,10 @@ func (m command) run(c *cli.Context, cfg *config) error {
 
 	csvFiles := cfg.filenames.Value()
 
-	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
+	chainLocator := lookup.NewSymlinkChainLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(cfg.hostRoot),
+	)
 
 	var candidates []string
 	for _, file := range csvFiles {
@@ -116,7 +120,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 			}
 			targets, err := chainLocator.Locate(ms.Path)
 			if err != nil {
-				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
+				m.logger.Warningf("Failed to locate symlink %v", ms.Path)
 			}
 			candidates = append(candidates, targets...)
 		}
@@ -125,35 +129,32 @@ func (m command) run(c *cli.Context, cfg *config) error {
 	created := make(map[string]bool)
 	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
 	for _, candidate := range candidates {
-		targets, err := m.Locate(candidate)
+		target, err := symlinks.Resolve(candidate)
 		if err != nil {
 			m.logger.Debugf("Skipping invalid link: %v", err)
 			continue
-		} else if len(targets) != 1 {
-			m.logger.Debugf("Unexepected number of targets: %v", targets)
-			continue
-		} else if targets[0] == candidate {
+		} else if target == candidate {
 			m.logger.Debugf("%v is not a symlink", candidate)
 			continue
 		}
 
-		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
+		err = m.createLink(created, cfg.hostRoot, containerRoot, target, candidate)
 		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
+			m.logger.Warningf("Failed to create link %v: %v", []string{target, candidate}, err)
 		}
 	}
 
 	links := cfg.links.Value()
 	for _, l := range links {
-		parts := strings.Split(l, ":")
+		parts := strings.Split(l, "::")
 		if len(parts) != 2 {
-			m.logger.Warnf("Invalid link specification %v", l)
+			m.logger.Warningf("Invalid link specification %v", l)
 			continue
 		}
 
 		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
 		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", parts, err)
+			m.logger.Warningf("Failed to create link %v: %v", parts, err)
 		}
 	}
 
@@ -164,7 +165,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
 	linkPath, err := changeRoot(hostRoot, containerRoot, link)
 	if err != nil {
-		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
+		m.logger.Warningf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
 	}
 	if created[linkPath] {
 		m.logger.Debugf("Link %v already created", linkPath)
@@ -173,7 +174,7 @@ func (m command) createLink(created map[string]bool, hostRoot string, containerR
 
 	targetPath, err := changeRoot(hostRoot, "/", target)
 	if err != nil {
-		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
+		m.logger.Warningf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
 	}
 
 	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)

cmd/nvidia-ctk/hook/hook.go

@@ -17,18 +17,20 @@
 package hook
 
 import (
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type hookCommand struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := hookCommand{
 		logger: logger,
 	}
@@ -46,6 +48,7 @@ func (m hookCommand) build() *cli.Command {
 	hook.Subcommands = []*cli.Command{
 		ldcache.NewCommand(m.logger),
 		symlinks.NewCommand(m.logger),
+		chmod.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -22,13 +22,13 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -37,7 +37,7 @@ type config struct {
 }
 
 // NewCommand constructs an update-ldcache command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -84,6 +84,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
+	_, err = os.Stat(filepath.Join(containerRoot, "/etc/ld.so.cache"))
+	if err != nil && os.IsNotExist(err) {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		return nil
+	}
+
 	err = m.createConfig(containerRoot, cfg.folders.Value())
 	if err != nil {
 		return fmt.Errorf("failed to update ld.so.conf: %v", err)
@@ -105,6 +111,10 @@ func (m command) createConfig(root string, folders []string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
 	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
 	if err != nil {
 		return fmt.Errorf("failed to create config file: %v", err)
@@ -125,5 +135,10 @@ func (m command) createConfig(root string, folders []string) error {
 		configured[folder] = true
 	}
 
+	// The created file needs to be world readable for the cases where the container is run as a non-root user.
+	if err := os.Chmod(configFile.Name(), 0644); err != nil {
+		return fmt.Errorf("failed to chmod config file: %v", err)
+	}
+
 	return nil
 }

cmd/nvidia-ctk/info/info.go

@@ -0,0 +1,47 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'info' command
+	info := cli.Command{
+		Name:  "info",
+		Usage: "Provide information about the system",
+	}
+
+	info.Subcommands = []*cli.Command{}
+
+	return &info
+}

cmd/nvidia-ctk/main.go

@@ -19,24 +19,32 @@ package main
 import (
 	"os"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+
 	cli "github.com/urfave/cli/v2"
 )
 
-var logger = log.New()
-
-// config defines the options that can be set for the CLI through config files,
+// options defines the options that can be set for the CLI through config files,
 // environment variables, or command line flags
-type config struct {
+type options struct {
 	// Debug indicates whether the CLI is started in "debug" mode
 	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
 }
 
 func main() {
-	// Create a config struct to hold the parsed environment variables or command line flags
-	config := config{}
+	logger := logrus.New()
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
 
 	// Create the top-level CLI
 	c := cli.NewApp()
@@ -52,16 +60,25 @@ func main() {
 			Name:        "debug",
 			Aliases:     []string{"d"},
 			Usage:       "Enable debug-level logging",
-			Destination: &config.Debug,
+			Destination: &opts.Debug,
 			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CTK_QUIET"},
+		},
 	}
 
 	// Set log-level for all subcommands
 	c.Before = func(c *cli.Context) error {
-		logLevel := log.InfoLevel
-		if config.Debug {
-			logLevel = log.DebugLevel
+		logLevel := logrus.InfoLevel
+		if opts.Debug {
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
 		}
 		logger.SetLevel(logLevel)
 		return nil
@@ -70,12 +87,17 @@ func main() {
 	// Define the subcommands
 	c.Commands = []*cli.Command{
 		hook.NewCommand(logger),
+		runtime.NewCommand(logger),
+		infoCLI.NewCommand(logger),
+		cdi.NewCommand(logger),
+		system.NewCommand(logger),
+		config.NewCommand(logger),
 	}
 
 	// Run the CLI
 	err := c.Run(os.Args)
 	if err != nil {
-		log.Errorf("%v", err)
-		log.Exit(1)
+		logger.Errorf("%v", err)
+		os.Exit(1)
 	}
 }

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -0,0 +1,317 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package configure
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
+)
+
+const (
+	defaultRuntime = "docker"
+
+	// defaultNVIDIARuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	defaultNVIDIARuntimeName = "nvidia"
+	// defaultNVIDIARuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	defaultNVIDIARuntimeExecutable          = "nvidia-container-runtime"
+	defaultNVIDIARuntimeExpecutablePath     = "/usr/bin/nvidia-container-runtime"
+	defaultNVIDIARuntimeHookExpecutablePath = "/usr/bin/nvidia-container-runtime-hook"
+
+	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
+	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
+	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an configure command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line config
+type config struct {
+	dryRun         bool
+	runtime        string
+	configFilePath string
+	mode           string
+	hookFilePath   string
+
+	nvidiaRuntime struct {
+		name         string
+		path         string
+		hookPath     string
+		setAsDefault bool
+	}
+
+	// cdi-specific options
+	cdi struct {
+		enabled bool
+	}
+}
+
+func (m command) build() *cli.Command {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the 'configure' command
+	configure := cli.Command{
+		Name:  "configure",
+		Usage: "Add a runtime to the specified container engine",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &config)
+		},
+		Action: func(c *cli.Context) error {
+			return m.configureWrapper(c, &config)
+		},
+	}
+
+	configure.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "update the runtime configuration as required but don't write changes to disk",
+			Destination: &config.dryRun,
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Usage:       "the target runtime engine; one of [containerd, crio, docker]",
+			Value:       defaultRuntime,
+			Destination: &config.runtime,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to the config file for the target runtime",
+			Destination: &config.configFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the config mode for runtimes that support multiple configuration mechanisms",
+			Destination: &config.mode,
+		},
+		&cli.StringFlag{
+			Name:        "oci-hook-path",
+			Usage:       "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
+			Destination: &config.hookFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-name",
+			Usage:       "specify the name of the NVIDIA runtime that will be added",
+			Value:       defaultNVIDIARuntimeName,
+			Destination: &config.nvidiaRuntime.name,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-path",
+			Aliases:     []string{"runtime-path"},
+			Usage:       "specify the path to the NVIDIA runtime executable",
+			Value:       defaultNVIDIARuntimeExecutable,
+			Destination: &config.nvidiaRuntime.path,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-hook-path",
+			Usage:       "specify the path to the NVIDIA Container Runtime hook executable",
+			Value:       defaultNVIDIARuntimeHookExpecutablePath,
+			Destination: &config.nvidiaRuntime.hookPath,
+		},
+		&cli.BoolFlag{
+			Name:        "nvidia-set-as-default",
+			Aliases:     []string{"set-as-default"},
+			Usage:       "set the NVIDIA runtime as the default runtime",
+			Destination: &config.nvidiaRuntime.setAsDefault,
+		},
+		&cli.BoolFlag{
+			Name:        "cdi.enabled",
+			Aliases:     []string{"cdi.enable"},
+			Usage:       "Enable CDI in the configured runtime",
+			Destination: &config.cdi.enabled,
+		},
+	}
+
+	return &configure
+}
+
+func (m command) validateFlags(c *cli.Context, config *config) error {
+	if config.mode == "oci-hook" {
+		if !filepath.IsAbs(config.nvidiaRuntime.hookPath) {
+			return fmt.Errorf("the NVIDIA runtime hook path %q is not an absolute path", config.nvidiaRuntime.hookPath)
+		}
+		return nil
+	}
+	if config.mode != "" && config.mode != "config-file" {
+		m.logger.Warningf("Ignoring unsupported config mode for %v: %q", config.runtime, config.mode)
+	}
+	config.mode = "config-file"
+
+	switch config.runtime {
+	case "containerd", "crio", "docker":
+		break
+	default:
+		return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+
+	switch config.runtime {
+	case "containerd", "crio":
+		if config.nvidiaRuntime.path == defaultNVIDIARuntimeExecutable {
+			config.nvidiaRuntime.path = defaultNVIDIARuntimeExpecutablePath
+		}
+		if !filepath.IsAbs(config.nvidiaRuntime.path) {
+			return fmt.Errorf("the NVIDIA runtime path %q is not an absolute path", config.nvidiaRuntime.path)
+		}
+	}
+
+	if config.runtime != "containerd" && config.runtime != "docker" {
+		if config.cdi.enabled {
+			m.logger.Warningf("Ignoring cdi.enabled flag for %v", config.runtime)
+		}
+		config.cdi.enabled = false
+	}
+
+	return nil
+}
+
+// configureWrapper updates the specified container engine config to enable the NVIDIA runtime
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.mode {
+	case "oci-hook":
+		return m.configureOCIHook(c, config)
+	case "config-file":
+		return m.configureConfigFile(c, config)
+	}
+	return fmt.Errorf("unsupported config-mode: %v", config.mode)
+}
+
+// configureConfigFile updates the specified container engine config file to enable the NVIDIA runtime.
+func (m command) configureConfigFile(c *cli.Context, config *config) error {
+	configFilePath := config.resolveConfigFilePath()
+
+	var cfg engine.Interface
+	var err error
+	switch config.runtime {
+	case "containerd":
+		cfg, err = containerd.New(
+			containerd.WithLogger(m.logger),
+			containerd.WithPath(configFilePath),
+		)
+	case "crio":
+		cfg, err = crio.New(
+			crio.WithLogger(m.logger),
+			crio.WithPath(configFilePath),
+		)
+	case "docker":
+		cfg, err = docker.New(
+			docker.WithLogger(m.logger),
+			docker.WithPath(configFilePath),
+		)
+	default:
+		err = fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+	if err != nil || cfg == nil {
+		return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
+	}
+
+	err = cfg.AddRuntime(
+		config.nvidiaRuntime.name,
+		config.nvidiaRuntime.path,
+		config.nvidiaRuntime.setAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = enableCDI(config, cfg)
+	if err != nil {
+		return fmt.Errorf("failed to enable CDI in %s: %w", config.runtime, err)
+	}
+
+	outputPath := config.getOuputConfigPath()
+	n, err := cfg.Save(outputPath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	if outputPath != "" {
+		if n == 0 {
+			m.logger.Infof("Removed empty config from %v", outputPath)
+		} else {
+			m.logger.Infof("Wrote updated config to %v", outputPath)
+		}
+		m.logger.Infof("It is recommended that %v daemon be restarted.", config.runtime)
+	}
+
+	return nil
+}
+
+// resolveConfigFilePath returns the default config file path for the configured container engine
+func (c *config) resolveConfigFilePath() string {
+	if c.configFilePath != "" {
+		return c.configFilePath
+	}
+	switch c.runtime {
+	case "containerd":
+		return defaultContainerdConfigFilePath
+	case "crio":
+		return defaultCrioConfigFilePath
+	case "docker":
+		return defaultDockerConfigFilePath
+	}
+	return ""
+}
+
+// getOuputConfigPath returns the configured config path or "" if dry-run is enabled
+func (c *config) getOuputConfigPath() string {
+	if c.dryRun {
+		return ""
+	}
+	return c.resolveConfigFilePath()
+}
+
+// configureOCIHook creates and configures the OCI hook for the NVIDIA runtime
+func (m *command) configureOCIHook(c *cli.Context, config *config) error {
+	err := ocihook.CreateHook(config.hookFilePath, config.nvidiaRuntime.hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating OCI hook: %v", err)
+	}
+	return nil
+}
+
+// enableCDI enables the use of CDI in the corresponding container engine
+func enableCDI(config *config, cfg engine.Interface) error {
+	if !config.cdi.enabled {
+		return nil
+	}
+	switch config.runtime {
+	case "containerd":
+		return cfg.Set("enable_cdi", true)
+	case "docker":
+		return cfg.Set("features", map[string]bool{"cdi": true})
+	}
+	return fmt.Errorf("enabling CDI in %s is not supported", config.runtime)
+}

cmd/nvidia-ctk/runtime/runtime.go

@@ -0,0 +1,49 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type runtimeCommand struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := runtimeCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m runtimeCommand) build() *cli.Command {
+	// Create the 'runtime' command
+	runtime := cli.Command{
+		Name:  "runtime",
+		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	runtime.Subcommands = []*cli.Command{
+		configure.NewCommand(m.logger),
+	}
+
+	return &runtime
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -0,0 +1,188 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/go-nvlib/pkg/nvpci"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+)
+
+type allPossible struct {
+	logger       logger.Interface
+	devRoot      string
+	deviceMajors devices.Devices
+	migCaps      nvcaps.MigCaps
+}
+
+// newAllPossible returns a new allPossible device node lister.
+// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func newAllPossible(logger logger.Interface, devRoot string) (nodeLister, error) {
+	deviceMajors, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed reading device majors: %v", err)
+	}
+
+	var requiredMajors []devices.Name
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
+	}
+	if migCaps == nil {
+		migCaps = make(nvcaps.MigCaps)
+	} else {
+		requiredMajors = append(requiredMajors, devices.NVIDIACaps)
+	}
+
+	requiredMajors = append(requiredMajors, devices.NVIDIAGPU, devices.NVIDIAUVM)
+	for _, name := range requiredMajors {
+		if !deviceMajors.Exists(name) {
+			return nil, fmt.Errorf("missing required device major %s", name)
+		}
+	}
+
+	l := allPossible{
+		logger:       logger,
+		devRoot:      devRoot,
+		deviceMajors: deviceMajors,
+		migCaps:      migCaps,
+	}
+
+	return l, nil
+}
+
+// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func (m allPossible) DeviceNodes() ([]deviceNode, error) {
+	gpus, err := nvpci.New(
+		nvpci.WithPCIDevicesRoot(filepath.Join(m.devRoot, nvpci.PCIDevicesRoot)),
+		nvpci.WithLogger(m.logger),
+	).GetGPUs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GPU information: %v", err)
+	}
+
+	count := len(gpus)
+	if count == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
+		return nil, nil
+	}
+
+	deviceNodes, err := m.getControlDeviceNodes()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
+	}
+
+	for gpu := 0; gpu < count; gpu++ {
+		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
+		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
+	}
+
+	return deviceNodes, nil
+}
+
+// getControlDeviceNodes generates a list of control devices
+func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
+	var deviceNodes []deviceNode
+
+	// Define the control devices for standard GPUs.
+	controlDevices := []deviceNode{
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
+	}
+
+	deviceNodes = append(deviceNodes, controlDevices...)
+
+	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
+		migControlMinor, exist := m.migCaps[migControlDevice]
+		if !exist {
+			continue
+		}
+
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			migControlMinor.DevicePath(),
+			int(migControlMinor),
+		)
+
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes, nil
+}
+
+// getGPUDeviceNodes generates a list of device nodes for a given GPU.
+func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
+	d := m.newDeviceNode(
+		devices.NVIDIAGPU,
+		fmt.Sprintf("/dev/nvidia%d", gpu),
+		gpu,
+	)
+
+	return []deviceNode{d}
+}
+
+// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
+func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
+	var selectedCapMinors []nvcaps.MigMinor
+	for gi := 0; ; gi++ {
+		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+		giMinor, exist := m.migCaps[giCap]
+		if !exist {
+			break
+		}
+		selectedCapMinors = append(selectedCapMinors, giMinor)
+		for ci := 0; ; ci++ {
+			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+			ciMinor, exist := m.migCaps[ciCap]
+			if !exist {
+				break
+			}
+			selectedCapMinors = append(selectedCapMinors, ciMinor)
+		}
+	}
+
+	var deviceNodes []deviceNode
+	for _, capMinor := range selectedCapMinors {
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			capMinor.DevicePath(),
+			int(capMinor),
+		)
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes
+}
+
+// newDeviceNode creates a new device node with the specified path and major/minor numbers.
+// The path is adjusted for the specified driver root.
+func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
+	major, _ := m.deviceMajors.Get(deviceName)
+
+	return deviceNode{
+		path:  filepath.Join(m.devRoot, path),
+		major: uint32(major),
+		minor: uint32(minor),
+	}
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -0,0 +1,424 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+	"github.com/fsnotify/fsnotify"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultDevCharPath = "/dev/char"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	devCharPath       string
+	driverRoot        string
+	dryRun            bool
+	watch             bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'create-dev-char-symlinks' command
+	c := cli.Command{
+		Name:  "create-dev-char-symlinks",
+		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "dev-char-path",
+			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
+			Value:       defaultDevCharPath,
+			Destination: &cfg.devCharPath,
+			EnvVars:     []string{"DEV_CHAR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
+			Value:       "/",
+			Destination: &cfg.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "watch",
+			Usage:       "If set, the command will watch for changes to the driver root and recreate the symlinks when changes are detected.",
+			Value:       false,
+			Destination: &cfg.watch,
+			EnvVars:     []string{"WATCH"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-all",
+			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
+			Destination: &cfg.createAll,
+			EnvVars:     []string{"CREATE_ALL"},
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "Load the NVIDIA kernel modules before creating symlinks. This is only applicable when --create-all is set.",
+			Destination: &cfg.loadKernelModules,
+			EnvVars:     []string{"LOAD_KERNEL_MODULES"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-device-nodes",
+			Usage:       "Create the NVIDIA control device nodes in the driver root if they do not exist. This is only applicable when --create-all is set",
+			Destination: &cfg.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "If set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &cfg.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	if cfg.createAll && cfg.watch {
+		return fmt.Errorf("create-all and watch are mutually exclusive")
+	}
+
+	if cfg.loadKernelModules && !cfg.createAll {
+		m.logger.Warning("load-kernel-modules is only applicable when create-all is set; ignoring")
+		cfg.loadKernelModules = false
+	}
+
+	if cfg.createDeviceNodes && !cfg.createAll {
+		m.logger.Warning("create-device-nodes is only applicable when create-all is set; ignoring")
+		cfg.createDeviceNodes = false
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	var watcher *fsnotify.Watcher
+	var sigs chan os.Signal
+
+	if cfg.watch {
+		watcher, err := newFSWatcher(filepath.Join(cfg.driverRoot, "dev"))
+		if err != nil {
+			return fmt.Errorf("failed to create FS watcher: %v", err)
+		}
+		defer watcher.Close()
+
+		sigs = newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	}
+
+	l, err := NewSymlinkCreator(
+		WithLogger(m.logger),
+		WithDevCharPath(cfg.devCharPath),
+		WithDriverRoot(cfg.driverRoot),
+		WithDryRun(cfg.dryRun),
+		WithCreateAll(cfg.createAll),
+		WithLoadKernelModules(cfg.loadKernelModules),
+		WithCreateDeviceNodes(cfg.createDeviceNodes),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink creator: %v", err)
+	}
+
+create:
+	err = l.CreateLinks()
+	if err != nil {
+		return fmt.Errorf("failed to create links: %v", err)
+	}
+	if !cfg.watch {
+		return nil
+	}
+	for {
+		select {
+
+		case event := <-watcher.Events:
+			deviceNode := filepath.Base(event.Name)
+			if !strings.HasPrefix(deviceNode, "nvidia") {
+				continue
+			}
+			if event.Op&fsnotify.Create == fsnotify.Create {
+				m.logger.Infof("%s created, restarting.", event.Name)
+				goto create
+			}
+			if event.Op&fsnotify.Create == fsnotify.Remove {
+				m.logger.Infof("%s removed. Ignoring", event.Name)
+
+			}
+
+		// Watch for any other fs errors and log them.
+		case err := <-watcher.Errors:
+			m.logger.Errorf("inotify: %s", err)
+
+		// React to signals
+		case s := <-sigs:
+			switch s {
+			case syscall.SIGHUP:
+				m.logger.Infof("Received SIGHUP, recreating symlinks.")
+				goto create
+			default:
+				m.logger.Infof("Received signal %q, shutting down.", s)
+				return nil
+			}
+		}
+	}
+}
+
+type linkCreator struct {
+	logger            logger.Interface
+	lister            nodeLister
+	driverRoot        string
+	devRoot           string
+	devCharPath       string
+	dryRun            bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
+}
+
+// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
+type Creator interface {
+	CreateLinks() error
+}
+
+// Option is a functional option for configuring the linkCreator.
+type Option func(*linkCreator)
+
+// NewSymlinkCreator creates a new linkCreator.
+func NewSymlinkCreator(opts ...Option) (Creator, error) {
+	c := linkCreator{}
+	for _, opt := range opts {
+		opt(&c)
+	}
+	if c.logger == nil {
+		c.logger = logger.New()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+	if c.devRoot == "" {
+		c.devRoot = "/"
+	}
+	if c.devCharPath == "" {
+		c.devCharPath = defaultDevCharPath
+	}
+
+	if err := c.setup(); err != nil {
+		return nil, err
+	}
+
+	if c.createAll {
+		lister, err := newAllPossible(c.logger, c.devRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
+		}
+		c.lister = lister
+	} else {
+		c.lister = existing{c.logger, c.devRoot}
+	}
+	return c, nil
+}
+
+func (m linkCreator) setup() error {
+	if !m.loadKernelModules && !m.createDeviceNodes {
+		return nil
+	}
+
+	if m.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(m.dryRun),
+			nvmodules.WithRoot(m.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if m.createDeviceNodes {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(m.dryRun),
+			nvdevices.WithDevRoot(m.devRoot),
+		)
+		if err != nil {
+			return err
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA device nodes: %v", err)
+		}
+	}
+	return nil
+}
+
+// WithDriverRoot sets the driver root path.
+// This is the path in which kernel modules must be loaded.
+func WithDriverRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.driverRoot = root
+	}
+}
+
+// WithDevRoot sets the root path for the /dev directory.
+func WithDevRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.devRoot = root
+	}
+}
+
+// WithDevCharPath sets the path at which the symlinks will be created.
+func WithDevCharPath(path string) Option {
+	return func(c *linkCreator) {
+		c.devCharPath = path
+	}
+}
+
+// WithDryRun sets the dry run flag.
+func WithDryRun(dryRun bool) Option {
+	return func(c *linkCreator) {
+		c.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger.
+func WithLogger(logger logger.Interface) Option {
+	return func(c *linkCreator) {
+		c.logger = logger
+	}
+}
+
+// WithCreateAll sets the createAll flag for the linkCreator.
+func WithCreateAll(createAll bool) Option {
+	return func(lc *linkCreator) {
+		lc.createAll = createAll
+	}
+}
+
+// WithLoadKernelModules sets the loadKernelModules flag for the linkCreator.
+func WithLoadKernelModules(loadKernelModules bool) Option {
+	return func(lc *linkCreator) {
+		lc.loadKernelModules = loadKernelModules
+	}
+}
+
+// WithCreateDeviceNodes sets the createDeviceNodes flag for the linkCreator.
+func WithCreateDeviceNodes(createDeviceNodes bool) Option {
+	return func(lc *linkCreator) {
+		lc.createDeviceNodes = createDeviceNodes
+	}
+}
+
+// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
+func (m linkCreator) CreateLinks() error {
+	deviceNodes, err := m.lister.DeviceNodes()
+	if err != nil {
+		return fmt.Errorf("failed to get device nodes: %v", err)
+	}
+
+	if len(deviceNodes) != 0 && !m.dryRun {
+		err := os.MkdirAll(m.devCharPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
+		}
+	}
+
+	for _, deviceNode := range deviceNodes {
+		target := deviceNode.path
+		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
+
+		m.logger.Infof("Creating link %s => %s", linkPath, target)
+		if m.dryRun {
+			continue
+		}
+
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warningf("Could not create symlink: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type deviceNode struct {
+	path  string
+	major uint32
+	minor uint32
+}
+
+func (d deviceNode) devCharName() string {
+	return fmt.Sprintf("%d:%d", d.major, d.minor)
+}
+
+func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, f := range files {
+		err = watcher.Add(f)
+		if err != nil {
+			watcher.Close()
+			return nil, err
+		}
+	}
+
+	return watcher, nil
+}
+
+func newOSWatcher(sigs ...os.Signal) chan os.Signal {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, sigs...)
+
+	return sigChan
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -0,0 +1,95 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"golang.org/x/sys/unix"
+)
+
+type nodeLister interface {
+	DeviceNodes() ([]deviceNode, error)
+}
+
+type existing struct {
+	logger  logger.Interface
+	devRoot string
+}
+
+// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
+// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
+func (m existing) DeviceNodes() ([]deviceNode, error) {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(m.devRoot),
+		lookup.WithOptional(true),
+	)
+
+	devices, err := locator.Locate("/dev/nvidia*")
+	if err != nil {
+		m.logger.Warningf("Error while locating device: %v", err)
+	}
+
+	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
+	if err != nil {
+		m.logger.Warningf("Error while locating caps device: %v", err)
+	}
+
+	if len(devices) == 0 && len(capDevices) == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
+		return nil, nil
+	}
+
+	var deviceNodes []deviceNode
+	for _, d := range append(devices, capDevices...) {
+		if m.nodeIsBlocked(d) {
+			continue
+		}
+
+		var stat unix.Stat_t
+		err := unix.Stat(d, &stat)
+		if err != nil {
+			m.logger.Warningf("Could not stat device: %v", err)
+			continue
+		}
+		deviceNode := deviceNode{
+			path:  d,
+			major: unix.Major(uint64(stat.Rdev)),
+			minor: unix.Minor(uint64(stat.Rdev)),
+		}
+
+		deviceNodes = append(deviceNodes, deviceNode)
+	}
+
+	return deviceNodes, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m existing) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,126 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverRoot string
+
+	dryRun bool
+
+	control bool
+
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device nodes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "load the NVIDIA Kernel Modules before creating devices nodes",
+			Destination: &opts.loadKernelModules,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	if opts.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(opts.dryRun),
+			nvmodules.WithRoot(opts.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if opts.control {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(opts.dryRun),
+			nvdevices.WithDevRoot(opts.driverRoot),
+		)
+		if err != nil {
+			return err
+		}
+		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/print-ldcache/print-ldcache.go

@@ -0,0 +1,101 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverRoot string
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "print-ldcache",
+		Usage: "A utility to print the contents of the ldcache",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	cache, err := ldcache.New(m.logger, opts.driverRoot)
+	if err != nil {
+		return fmt.Errorf("failed to create ldcache: %v", err)
+	}
+
+	lib32, lib64 := cache.List()
+
+	if len(lib32) == 0 {
+		m.logger.Info("No 32-bit libraries found")
+	} else {
+		m.logger.Infof("%d 32-bit libraries found", len(lib32))
+		for _, lib := range lib32 {
+			m.logger.Infof("%v", lib)
+		}
+	}
+	if len(lib64) == 0 {
+		m.logger.Info("No 64-bit libraries found")
+	} else {
+		m.logger.Infof("%d 64-bit libraries found", len(lib64))
+		for _, lib := range lib64 {
+			m.logger.Infof("%v", lib)
+		}
+	}
+
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/print-ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m command) build() *cli.Command {
+	// Create the 'system' command
+	system := cli.Command{
+		Name:  "system",
+		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	system.Subcommands = []*cli.Command{
+		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
+		ldcache.NewCommand(m.logger),
+	}
+
+	return &system
+}

config/config.toml.amazonlinux

@@ -1,19 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false

config/config.toml.centos

@@ -1,19 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false

config/config.toml.debian

@@ -1,19 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false

config/config.toml.opensuse-leap

@@ -1,19 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false

config/config.toml.ubuntu

@@ -1,26 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig.real"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]

docker/Dockerfile.amazonlinux

@@ -1,70 +0,0 @@
-ARG BASEIMAGE
-FROM ${BASEIMAGE}
-
-RUN yum install -y \
-        ca-certificates \
-        gcc \
-        wget \
-        git \
-        rpm-build \
-        make && \
-    rm -rf /var/cache/yum/*
-
-ARG GOLANG_VERSION=0.0.0
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture"; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
-
-# packaging
-ARG PKG_VERS
-ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
-
-# output directory
-ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
-RUN mkdir -p $DIST_DIR /dist
-
-# nvidia-container-toolkit
-WORKDIR $GOPATH/src/nvidia-container-toolkit
-COPY . .
-
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
-RUN make PREFIX=${DIST_DIR} cmds
-
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-# This might not be useful on Amazon Linux, but it's simpler to keep the RHEL
-# and Amazon Linux packages identical.
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
-WORKDIR $DIST_DIR/..
-COPY packaging/rpm .
-
-CMD arch=$(uname -m) && \
-    rpmbuild --clean --target=$arch -bb \
-             -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
-             SPECS/nvidia-container-toolkit.spec && \
-    mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -32,6 +32,7 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -52,24 +53,20 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Debian Jessie still had ldconfig.real
-RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
-       sed -i 's;"@/sbin/ldconfig";"@/sbin/ldconfig.real";' $DIST_DIR/config.toml; \
-    fi
-
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
-    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
-    dch --changelog debian/changelog -r "" && \
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
+RUN dch --create --package="${PKG_NAME}" \
+        --newversion "${REVISION}" \
+            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER1_VERSION}" && \
+    dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
-    mv /tmp/nvidia-container-toolkit_*.deb /dist
+    mv /tmp/*.deb /dist

docker/Dockerfile.devel

@@ -14,7 +14,8 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go install golang.org/x/lint/golint@latest
+RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
 RUN go install github.com/matryer/moq@latest
-RUN go install github.com/gordonklaus/ineffassign@latest
+RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
 RUN go install github.com/client9/misspell/cmd/misspell@latest
+RUN go install github.com/google/go-licenses@latest

docker/Dockerfile.opensuse-leap

@@ -25,11 +25,12 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -43,24 +44,19 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
+             -D "release_date $(date +'%a %b %d %Y')" \
+             -D "git_commit ${GIT_COMMIT}" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.rpm-yum

@@ -1,3 +1,19 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the dockerfile for building packages on yum-based RPM systems.
+
 ARG BASEIMAGE
 FROM ${BASEIMAGE}
 
@@ -27,11 +43,12 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -45,24 +62,19 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
+             -D "release_date $(date +'%a %b %d %Y')" \
+             -D "git_commit ${GIT_COMMIT}" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -30,6 +30,7 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -50,19 +51,20 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
-    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
-    dch --changelog debian/changelog -r "" && \
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
+RUN dch --create --package="${PKG_NAME}" \
+        --newversion "${REVISION}" \
+            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" && \
+    dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -17,7 +17,7 @@ AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
 X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := centos8 rhel8 amazonlinux2
+AARCH64_TARGETS := centos7 centos8 rhel8 amazonlinux2
 
 # Define top-level build targets
 docker%: SHELL:=/bin/bash
@@ -85,39 +85,37 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 --%: docker-build-%
 	@
 
+LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
+LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
+
+LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
---ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
---debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
+--rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
 
-# We allow the CONFIG_TOML_SUFFIX to be overridden.
-CONFIG_TOML_SUFFIX ?= $(OS)
 
 docker-build-%:
 	@echo "Building for $(TARGET_PLATFORM)"
@@ -128,10 +126,11 @@ docker-build-%:
 	    --progress=plain \
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
-	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
-		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+	    --build-arg PKG_NAME="$(LIB_NAME)" \
+	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
+	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
+	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
+	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \

go.mod

@@ -1,18 +1,37 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.14
+go 1.20
 
 require (
-	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.11.6-0
-	github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
-	github.com/containers/podman/v4 v4.0.3
-	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
-	github.com/pelletier/go-toml v1.9.4
-	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
-	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
-	github.com/urfave/cli/v2 v2.3.0
-	golang.org/x/mod v0.5.0
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
+	github.com/NVIDIA/go-nvlib v0.2.0
+	github.com/NVIDIA/go-nvml v0.12.0-3
+	github.com/fsnotify/fsnotify v1.7.0
+	github.com/opencontainers/runtime-spec v1.2.0
+	github.com/pelletier/go-toml v1.9.5
+	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.9.0
+	github.com/urfave/cli/v2 v2.27.1
+	golang.org/x/mod v0.17.0
+	golang.org/x/sys v0.19.0
+	tags.cncf.io/container-device-interface v0.6.2
+	tags.cncf.io/container-device-interface/specs-go v0.6.0
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

internal/config/cli.go

@@ -16,33 +16,18 @@
 
 package config
 
-import (
-	"github.com/pelletier/go-toml"
-)
-
 // ContainerCLIConfig stores the options for the nvidia-container-cli
 type ContainerCLIConfig struct {
-	Root string
-}
-
-// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
-	cfg := getDefaultContainerCLIConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
-
-	return cfg
-}
-
-// getDefaultContainerCLIConfig defines the default values for the config
-func getDefaultContainerCLIConfig() *ContainerCLIConfig {
-	c := ContainerCLIConfig{
-		Root: "",
-	}
-
-	return &c
+	Root        string   `toml:"root"`
+	Path        string   `toml:"path"`
+	Environment []string `toml:"environment"`
+	Debug       string   `toml:"debug"`
+	Ldcache     string   `toml:"ldcache"`
+	LoadKmods   bool     `toml:"load-kmods"`
+	// NoPivot disables the pivot root operation in the NVIDIA Container CLI.
+	// This is not exposed in the config if not set.
+	NoPivot   bool   `toml:"no-pivot,omitempty"`
+	NoCgroups bool   `toml:"no-cgroups"`
+	User      string `toml:"user"`
+	Ldconfig  string `toml:"ldconfig"`
 }

internal/config/config.go

@@ -17,17 +17,27 @@
 package config
 
 import (
-	"fmt"
-	"io"
+	"bufio"
 	"os"
-	"path"
+	"path/filepath"
+	"strings"
 
-	"github.com/pelletier/go-toml"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 const (
 	configOverride = "XDG_CONFIG_HOME"
 	configFilePath = "nvidia-container-runtime/config.toml"
+
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+
+	nvidiaContainerRuntimeHookExecutable  = "nvidia-container-runtime-hook"
+	nvidiaContainerRuntimeHookDefaultPath = "/usr/bin/nvidia-container-runtime-hook"
 )
 
 var (
@@ -38,77 +48,178 @@ var (
 	NVIDIAContainerRuntimeHookExecutable = "nvidia-container-runtime-hook"
 	// NVIDIAContainerToolkitExecutable is the executable name for the NVIDIA Container Toolkit (an alias for the NVIDIA Container Runtime Hook)
 	NVIDIAContainerToolkitExecutable = "nvidia-container-toolkit"
-
-	configDir = "/etc/"
 )
 
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
-	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+	DisableRequire                 bool   `toml:"disable-require"`
+	SwarmResource                  string `toml:"swarm-resource"`
+	AcceptEnvvarUnprivileged       bool   `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+	AcceptDeviceListAsVolumeMounts bool   `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	SupportedDriverCapabilities    string `toml:"supported-driver-capabilities"`
+
+	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
+}
+
+// GetConfigFilePath returns the path to the config file for the configured system
+func GetConfigFilePath() string {
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		return filepath.Join(XDGConfigDir, configFilePath)
+	}
+
+	return filepath.Join("/etc", configFilePath)
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file
 // or set via the environment.
 func GetConfig() (*Config, error) {
-	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
-		configDir = XDGConfigDir
-	}
-
-	configFilePath := path.Join(configDir, configFilePath)
-
-	tomlFile, err := os.Open(configFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open config file %v: %v", configFilePath, err)
-	}
-	defer tomlFile.Close()
-
-	cfg, err := loadConfigFrom(tomlFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read config values: %v", err)
-	}
-
-	return cfg, nil
-}
-
-// loadRuntimeConfigFrom reads the config from the specified Reader
-func loadConfigFrom(reader io.Reader) (*Config, error) {
-	toml, err := toml.LoadReader(reader)
+	cfg, err := New(
+		WithConfigFile(GetConfigFilePath()),
+	)
 	if err != nil {
 		return nil, err
 	}
 
-	return getConfigFrom(toml)
+	return cfg.Config()
 }
 
-// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getConfigFrom(toml *toml.Tree) (*Config, error) {
-	cfg := getDefaultConfig()
+// GetDefault defines the default values for the config
+func GetDefault() (*Config, error) {
+	d := Config{
+		AcceptEnvvarUnprivileged:    true,
+		SupportedDriverCapabilities: image.SupportedDriverCapabilities.String(),
+		NVIDIAContainerCLIConfig: ContainerCLIConfig{
+			LoadKmods: true,
+			Ldconfig:  getLdConfigPath(),
+			User:      getUserGroup(),
+		},
+		NVIDIACTKConfig: CTKConfig{
+			Path: nvidiaCTKExecutable,
+		},
+		NVIDIAContainerRuntimeConfig: RuntimeConfig{
+			DebugFilePath: "/dev/null",
+			LogLevel:      "info",
+			Runtimes:      []string{"docker-runc", "runc", "crun"},
+			Mode:          "auto",
+			Modes: modesConfig{
+				CSV: csvModeConfig{
+					MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+				},
+				CDI: cdiModeConfig{
+					DefaultKind:        "nvidia.com/gpu",
+					AnnotationPrefixes: []string{cdi.AnnotationPrefix},
+					SpecDirs:           cdi.DefaultSpecDirs,
+				},
+			},
+		},
+		NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+			Path: NVIDIAContainerRuntimeHookExecutable,
+		},
+	}
+	return &d, nil
+}
 
-	if toml == nil {
-		return cfg, nil
+func getLdConfigPath() string {
+	if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
+		return "@/sbin/ldconfig.real"
+	}
+	return "@/sbin/ldconfig"
+}
+
+func getUserGroup() string {
+	if isSuse() {
+		return "root:video"
+	}
+	return ""
+}
+
+// isSuse returns whether a SUSE-based distribution was detected.
+func isSuse() bool {
+	suseDists := map[string]bool{
+		"suse":     true,
+		"opensuse": true,
 	}
 
-	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
-	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
-	runtimeConfig, err := getRuntimeConfigFrom(toml)
+	idsLike := getDistIDLike()
+	for _, id := range idsLike {
+		if suseDists[id] {
+			return true
+		}
+	}
+	return false
+}
+
+// getDistIDLike returns the ID_LIKE field from /etc/os-release.
+// We can override this for testing.
+var getDistIDLike = func() []string {
+	releaseFile, err := os.Open("/etc/os-release")
 	if err != nil {
-		return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
+		return nil
 	}
-	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
+	defer releaseFile.Close()
 
-	return cfg, nil
+	scanner := bufio.NewScanner(releaseFile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "ID_LIKE=") {
+			value := strings.Trim(strings.TrimPrefix(line, "ID_LIKE="), "\"")
+			return strings.Split(value, " ")
+		}
+	}
+	return nil
 }
 
-// getDefaultConfig defines the default values for the config
-func getDefaultConfig() *Config {
-	c := Config{
-		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
-		NVIDIACTKConfig:              *getDefaultCTKConfig(),
-		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
+// ResolveNVIDIACTKPath resolves the path to the nvidia-ctk binary.
+// This executable is used in hooks and needs to be an absolute path.
+// If the path is specified as an absolute path, it is used directly
+// without checking for existence of an executable at that path.
+func ResolveNVIDIACTKPath(logger logger.Interface, nvidiaCTKPath string) string {
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Toolkit CLI",
+		nvidiaCTKPath,
+		nvidiaCTKDefaultFilePath,
+	)
+}
+
+// ResolveNVIDIAContainerRuntimeHookPath resolves the path the nvidia-container-runtime-hook binary.
+func ResolveNVIDIAContainerRuntimeHookPath(logger logger.Interface, nvidiaContainerRuntimeHookPath string) string {
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Runtime Hook",
+		nvidiaContainerRuntimeHookPath,
+		nvidiaContainerRuntimeHookDefaultPath,
+	)
+}
+
+// resolveWithDefault resolves the path to the specified binary.
+// If an absolute path is specified, it is used directly without searching for the binary.
+// If the binary cannot be found in the path, the specified default is used instead.
+func resolveWithDefault(logger logger.Interface, label string, path string, defaultPath string) string {
+	if filepath.IsAbs(path) {
+		logger.Debugf("Using specified %v path %v", label, path)
+		return path
 	}
 
-	return &c
+	if path == "" {
+		path = filepath.Base(defaultPath)
+	}
+	logger.Debugf("Locating %v as %v", label, path)
+	lookup := lookup.NewExecutableLocator(logger, "")
+
+	resolvedPath := defaultPath
+	targets, err := lookup.Locate(path)
+	if err != nil {
+		logger.Warningf("Failed to locate %v: %v", path, err)
+	} else {
+		logger.Debugf("Found %v candidates: %v", path, targets)
+		resolvedPath = targets[0]
+	}
+	logger.Debugf("Using %v path %v", label, path)
+
+	return resolvedPath
 }

internal/config/config_test.go

@@ -17,7 +17,6 @@
 package config
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -27,50 +26,61 @@ import (
 )
 
 func TestGetConfigWithCustomConfig(t *testing.T) {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
+	testDir := t.TempDir()
+	t.Setenv(configOverride, testDir)
+
+	filename := filepath.Join(testDir, configFilePath)
 
 	// By default debug is disabled
 	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
-	testDir := filepath.Join(wd, "test")
-	filename := filepath.Join(testDir, configFilePath)
-
-	os.Setenv(configOverride, testDir)
 
 	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
-	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
-
-	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+	require.NoError(t, os.WriteFile(filename, contents, 0766))
 
 	cfg, err := GetConfig()
 	require.NoError(t, err)
-	require.Equal(t, cfg.NVIDIAContainerRuntimeConfig.DebugFilePath, "/nvidia-container-toolkit.log")
+	require.Equal(t, "/nvidia-container-toolkit.log", cfg.NVIDIAContainerRuntimeConfig.DebugFilePath)
 }
 
 func TestGetConfig(t *testing.T) {
 	testCases := []struct {
-		description    string
-		contents       []string
-		expectedError  error
-		expectedConfig *Config
+		description     string
+		contents        []string
+		expectedError   error
+		inspectLdconfig bool
+		distIdsLike     []string
+		expectedConfig  *Config
 	}{
 		{
-			description: "empty config is default",
+			description:     "empty config is default",
+			inspectLdconfig: true,
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "",
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/dev/null",
 					LogLevel:      "info",
-					Runtimes:      []string{"docker-runc", "runc"},
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
 					Mode:          "auto",
 					Modes: modesConfig{
 						CSV: csvModeConfig{
 							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "nvidia-ctk",
 				},
@@ -79,19 +89,32 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set inline",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
+				"supported-driver-capabilities = \"compute,utility\"",
 				"nvidia-container-cli.root = \"/bar/baz\"",
+				"nvidia-container-cli.load-kmods = false",
+				"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
+				"nvidia-container-cli.user = \"foo:bar\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
-				"nvidia-container-runtime.experimental = true",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
+				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"nvidia-container-runtime.modes.cdi.spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"nvidia-container-runtime-hook.path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    false,
+				SupportedDriverCapabilities: "compute,utility",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
+					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -102,8 +125,22 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "/foo/bar/nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
 				},
@@ -112,23 +149,38 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set in section",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
+				"supported-driver-capabilities = \"compute,utility\"",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
+				"load-kmods = false",
+				"ldconfig = \"/foo/bar/ldconfig\"",
+				"user = \"foo:bar\"",
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
-				"experimental = true",
 				"discover-mode = \"not-legacy\"",
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
 				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.cdi]",
+				"default-kind = \"example.vendor.com/device\"",
+				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"[nvidia-container-runtime-hook]",
+				"path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    false,
+				SupportedDriverCapabilities: "compute,utility",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
+					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -139,27 +191,147 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "/foo/bar/nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
 				},
 			},
 		},
+		{
+			description:     "suse config",
+			distIdsLike:     []string{"suse", "opensuse"},
+			inspectLdconfig: true,
+			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
+					User:      "root:video",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
+					},
+				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+			},
+		},
+		{
+			description:     "suse config overrides user",
+			distIdsLike:     []string{"suse", "opensuse"},
+			inspectLdconfig: true,
+			contents: []string{
+				"nvidia-container-cli.user = \"foo:bar\"",
+			},
+			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
+					User:      "foo:bar",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
+					},
+				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
+			defer setGetDistIDLikeForTest(tc.distIdsLike)()
 			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
 
-			cfg, err := loadConfigFrom(reader)
+			tomlCfg, err := loadConfigTomlFrom(reader)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
 			}
+			cfg, err := tomlCfg.Config()
+			require.NoError(t, err)
+
+			// We first handle the ldconfig path since this is currently system-dependent.
+			if tc.inspectLdconfig {
+				ldconfig := cfg.NVIDIAContainerCLIConfig.Ldconfig
+				require.True(t, strings.HasPrefix(ldconfig, "@/sbin/ldconfig"))
+				remaining := strings.TrimPrefix(ldconfig, "@/sbin/ldconfig")
+				require.True(t, remaining == ".real" || remaining == "")
+
+				cfg.NVIDIAContainerCLIConfig.Ldconfig = "WAS_CHECKED"
+			}
 
 			require.EqualValues(t, tc.expectedConfig, cfg)
 		})
 	}
 }
+
+// setGetDistIDsLikeForTest overrides the distribution IDs that would normally be read from the /etc/os-release file.
+func setGetDistIDLikeForTest(ids []string) func() {
+	if ids == nil {
+		return func() {}
+	}
+	original := getDistIDLike
+
+	getDistIDLike = func() []string {
+		return ids
+	}
+
+	return func() {
+		getDistIDLike = original
+	}
+}

internal/config/hook.go

@@ -0,0 +1,36 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeHookConfig struct {
+	// Path specifies the path to the NVIDIA Container Runtime hook binary.
+	// If an executable name is specified, this will be resolved in the path.
+	Path string `toml:"path"`
+	// SkipModeDetection disables the mode check for the runtime hook.
+	SkipModeDetection bool `toml:"skip-mode-detection"`
+}
+
+// GetDefaultRuntimeHookConfig defines the default values for the config
+func GetDefaultRuntimeHookConfig() (*RuntimeHookConfig, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg.NVIDIAContainerRuntimeHookConfig, nil
+}

internal/config/image/builder.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"strings"
+)
+
+type builder struct {
+	env            []string
+	disableRequire bool
+}
+
+// New creates a new CUDA image from the input options.
+func New(opt ...Option) (CUDA, error) {
+	b := &builder{}
+	for _, o := range opt {
+		o(b)
+	}
+
+	return b.build()
+}
+
+// build creates a CUDA image from the builder.
+func (b builder) build() (CUDA, error) {
+	c := make(CUDA)
+
+	for _, e := range b.env {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid environment variable: %v", e)
+		}
+		c[parts[0]] = parts[1]
+	}
+
+	if b.disableRequire {
+		c[envNVDisableRequire] = "true"
+	}
+
+	return c, nil
+}
+
+// Option is a functional option for creating a CUDA image.
+type Option func(*builder)
+
+// WithDisableRequire sets the disable require option.
+func WithDisableRequire(disableRequire bool) Option {
+	return func(b *builder) {
+		b.disableRequire = disableRequire
+	}
+}
+
+// WithEnv sets the environment variables to use when creating the CUDA image.
+func WithEnv(env []string) Option {
+	return func(b *builder) {
+		b.env = env
+	}
+}

internal/config/image/capabilities.go

@@ -0,0 +1,146 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"sort"
+	"strings"
+)
+
+// DriverCapability represents the possible values of NVIDIA_DRIVER_CAPABILITIES
+type DriverCapability string
+
+// Constants for the supported driver capabilities
+const (
+	DriverCapabilityAll      DriverCapability = "all"
+	DriverCapabilityNone     DriverCapability = "none"
+	DriverCapabilityCompat32 DriverCapability = "compat32"
+	DriverCapabilityCompute  DriverCapability = "compute"
+	DriverCapabilityDisplay  DriverCapability = "display"
+	DriverCapabilityGraphics DriverCapability = "graphics"
+	DriverCapabilityNgx      DriverCapability = "ngx"
+	DriverCapabilityUtility  DriverCapability = "utility"
+	DriverCapabilityVideo    DriverCapability = "video"
+)
+
+var (
+	driverCapabilitiesNone = NewDriverCapabilities()
+	driverCapabilitiesAll  = NewDriverCapabilities("all")
+
+	// DefaultDriverCapabilities sets the value for driver capabilities if no value is set.
+	DefaultDriverCapabilities = NewDriverCapabilities("utility,compute")
+	// SupportedDriverCapabilities defines the set of all supported driver capabilities.
+	SupportedDriverCapabilities = NewDriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
+)
+
+// NewDriverCapabilities creates a set of driver capabilities from the specified capabilities
+func NewDriverCapabilities(capabilities ...string) DriverCapabilities {
+	dc := make(DriverCapabilities)
+	for _, capability := range capabilities {
+		for _, c := range strings.Split(capability, ",") {
+			trimmed := strings.TrimSpace(c)
+			if trimmed == "" {
+				continue
+			}
+			dc[DriverCapability(trimmed)] = true
+		}
+	}
+	return dc
+}
+
+// DriverCapabilities represents the NVIDIA_DRIVER_CAPABILITIES set for the specified image.
+type DriverCapabilities map[DriverCapability]bool
+
+// Has check whether the specified capability is selected.
+func (c DriverCapabilities) Has(capability DriverCapability) bool {
+	if c.IsAll() {
+		return true
+	}
+	return c[capability]
+}
+
+// Any checks whether any of the specified capabilites are set
+func (c DriverCapabilities) Any(capabilities ...DriverCapability) bool {
+	if c.IsAll() {
+		return true
+	}
+	for _, cap := range capabilities {
+		if c.Has(cap) {
+			return true
+		}
+	}
+	return false
+}
+
+// List returns the list of driver capabilities.
+// The list is sorted.
+func (c DriverCapabilities) List() []string {
+	var capabilities []string
+	for capability := range c {
+		capabilities = append(capabilities, string(capability))
+	}
+	sort.Strings(capabilities)
+	return capabilities
+}
+
+// String returns the string repesentation of the driver capabilities.
+func (c DriverCapabilities) String() string {
+	if c.IsAll() {
+		return "all"
+	}
+	return strings.Join(c.List(), ",")
+}
+
+// IsAll indicates whether the set of capabilities is `all`
+func (c DriverCapabilities) IsAll() bool {
+	return c[DriverCapabilityAll]
+}
+
+// Intersection returns a new set which includes the item in BOTH d and s2.
+// For example: d = {a1, a2} s2 = {a2, a3} s1.Intersection(s2) = {a2}
+func (c DriverCapabilities) Intersection(s2 DriverCapabilities) DriverCapabilities {
+	if s2.IsAll() {
+		return c
+	}
+	if c.IsAll() {
+		return s2
+	}
+
+	intersection := make(DriverCapabilities)
+	for capability := range s2 {
+		if c[capability] {
+			intersection[capability] = true
+		}
+	}
+
+	return intersection
+}
+
+// IsSuperset returns true if and only if d is a superset of s2.
+func (c DriverCapabilities) IsSuperset(s2 DriverCapabilities) bool {
+	if c.IsAll() {
+		return true
+	}
+
+	for capability := range s2 {
+		if !c[capability] {
+			return false
+		}
+	}
+
+	return true
+}

internal/config/image/capabilities_test.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDriverCapabilitiesIntersection(t *testing.T) {
+	testCases := []struct {
+		capabilities          DriverCapabilities
+		supportedCapabilities DriverCapabilities
+		expectedIntersection  DriverCapabilities
+	}{
+		{
+			capabilities:          driverCapabilitiesNone,
+			supportedCapabilities: driverCapabilitiesNone,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          driverCapabilitiesAll,
+			supportedCapabilities: driverCapabilitiesNone,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          driverCapabilitiesAll,
+			supportedCapabilities: SupportedDriverCapabilities,
+			expectedIntersection:  SupportedDriverCapabilities,
+		},
+		{
+			capabilities:          SupportedDriverCapabilities,
+			supportedCapabilities: driverCapabilitiesAll,
+			expectedIntersection:  SupportedDriverCapabilities,
+		},
+		{
+			capabilities:          driverCapabilitiesNone,
+			supportedCapabilities: driverCapabilitiesAll,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          driverCapabilitiesNone,
+			supportedCapabilities: NewDriverCapabilities("cap1"),
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          NewDriverCapabilities("cap0,cap1"),
+			supportedCapabilities: NewDriverCapabilities("cap1,cap0"),
+			expectedIntersection:  NewDriverCapabilities("cap0,cap1"),
+		},
+		{
+			capabilities:          DefaultDriverCapabilities,
+			supportedCapabilities: SupportedDriverCapabilities,
+			expectedIntersection:  DefaultDriverCapabilities,
+		},
+		{
+			capabilities:          NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			supportedCapabilities: NewDriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			expectedIntersection:  NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+		{
+			capabilities:          NewDriverCapabilities("cap1"),
+			supportedCapabilities: driverCapabilitiesNone,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          NewDriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			supportedCapabilities: NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			expectedIntersection:  NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
+			require.EqualValues(t, tc.expectedIntersection, intersection)
+		})
+	}
+}
+
+func TestDriverCapabilitiesList(t *testing.T) {
+	testCases := []struct {
+		capabilities DriverCapabilities
+		expected     []string
+	}{
+		{
+			capabilities: NewDriverCapabilities(""),
+		},
+		{
+			capabilities: NewDriverCapabilities("  "),
+		},
+		{
+			capabilities: NewDriverCapabilities(","),
+		},
+		{
+			capabilities: NewDriverCapabilities(",cap"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: NewDriverCapabilities("cap,"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: NewDriverCapabilities("cap0,,cap1"),
+			expected:     []string{"cap0", "cap1"},
+		},
+		{
+			capabilities: NewDriverCapabilities("cap1,cap0,cap3"),
+			expected:     []string{"cap0", "cap1", "cap3"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			require.EqualValues(t, tc.expected, tc.capabilities.List())
+		})
+	}
+}

internal/config/image/cuda_image.go

@@ -26,10 +26,12 @@ import (
 )
 
 const (
-	envCUDAVersion      = "CUDA_VERSION"
-	envNVRequirePrefix  = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA    = envNVRequirePrefix + "CUDA"
-	envNVDisableRequire = "NVIDIA_DISABLE_REQUIRE"
+	envCUDAVersion          = "CUDA_VERSION"
+	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
+	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
+	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
 // CUDA represents a CUDA image that can be used for GPU computing. This wraps
@@ -40,27 +42,18 @@ type CUDA map[string]string
 // NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
 // The process environment is read (if present) to construc the CUDA Image.
 func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
-	if spec == nil || spec.Process == nil {
-		return NewCUDAImageFromEnv(nil)
+	var env []string
+	if spec != nil && spec.Process != nil {
+		env = spec.Process.Env
 	}
 
-	return NewCUDAImageFromEnv(spec.Process.Env)
+	return New(WithEnv(env))
 }
 
 // NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
 // is a list of strings of the form ENVAR=VALUE.
 func NewCUDAImageFromEnv(env []string) (CUDA, error) {
-	c := make(CUDA)
-
-	for _, e := range env {
-		parts := strings.SplitN(e, "=", 2)
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("invalid environment variable: %v", e)
-		}
-		c[parts[0]] = parts[1]
-	}
-
-	return c, nil
+	return New(WithEnv(env))
 }
 
 // IsLegacy returns whether the associated CUDA image is a "legacy" image. An
@@ -75,16 +68,14 @@ func (i CUDA) IsLegacy() bool {
 // GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
 // variables.
 func (i CUDA) GetRequirements() ([]string, error) {
-	// TODO: We need not process this if disable require is set, but this will be done
-	// in a single follow-up to ensure that the behavioural change is accurately captured.
-	// if i.HasDisableRequire() {
-	// 	return nil, nil
-	// }
+	if i.HasDisableRequire() {
+		return nil, nil
+	}
 
 	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
 	var requirements []string
 	for name, value := range i {
-		if strings.HasPrefix(name, envNVRequirePrefix) {
+		if strings.HasPrefix(name, envNVRequirePrefix) && !strings.HasPrefix(name, envNVRequireJetpack) {
 			requirements = append(requirements, value)
 		}
 	}
@@ -111,10 +102,56 @@ func (i CUDA) HasDisableRequire() bool {
 	return false
 }
 
+// DevicesFromEnvvars returns the devices requested by the image through environment variables
+func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+	// We concantenate all the devices from the specified envvars.
+	var isSet bool
+	var devices []string
+	requested := make(map[string]bool)
+	for _, envVar := range envVars {
+		if devs, ok := i[envVar]; ok {
+			isSet = true
+			for _, d := range strings.Split(devs, ",") {
+				trimmed := strings.TrimSpace(d)
+				if len(trimmed) == 0 {
+					continue
+				}
+				devices = append(devices, trimmed)
+				requested[trimmed] = true
+			}
+		}
+	}
+
+	// Environment variable unset with legacy image: default to "all".
+	if !isSet && len(devices) == 0 && i.IsLegacy() {
+		return NewVisibleDevices("all")
+	}
+
+	// Environment variable unset or empty or "void": return nil
+	if len(devices) == 0 || requested["void"] {
+		return NewVisibleDevices("void")
+	}
+
+	return NewVisibleDevices(devices...)
+}
+
+// GetDriverCapabilities returns the requested driver capabilities.
+func (i CUDA) GetDriverCapabilities() DriverCapabilities {
+	env := i[envNVDriverCapabilities]
+
+	capabilites := make(DriverCapabilities)
+	for _, c := range strings.Split(env, ",") {
+		capabilites[DriverCapability(c)] = true
+	}
+
+	return capabilites
+}
+
 func (i CUDA) legacyVersion() (string, error) {
-	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
+	cudaVersion := i[envCUDAVersion]
+	majorMinor, err := parseMajorMinorVersion(cudaVersion)
 	if err != nil {
-		return "", fmt.Errorf("invalid CUDA version: %v", err)
+		return "", fmt.Errorf("invalid CUDA version %v: %v", cudaVersion, err)
 	}
 
 	return majorMinor, nil

internal/config/image/cuda_image_test.go

@@ -69,3 +69,65 @@ func TestParseMajorMinorVersionInvalid(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRequirements(t *testing.T) {
+	testCases := []struct {
+		description  string
+		env          []string
+		requirements []string
+	}{
+		{
+			description:  "NVIDIA_REQUIRE_JETPACK is ignored",
+			env:          []string{"NVIDIA_REQUIRE_JETPACK=csv-mounts=all"},
+			requirements: nil,
+		},
+		{
+			description:  "NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS is ignored",
+			env:          []string{"NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS=base-only"},
+			requirements: nil,
+		},
+		{
+			description:  "single requirement set",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6"},
+			requirements: []string{"cuda>=11.6"},
+		},
+		{
+			description:  "requirements are concatenated requirement set",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
+			requirements: []string{"cuda>=11.6", "brand=tesla"},
+		},
+		{
+			description:  "legacy image",
+			env:          []string{"CUDA_VERSION=11.6"},
+			requirements: []string{"cuda>=11.6"},
+		},
+		{
+			description:  "legacy image with additional requirement",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
+			requirements: []string{"cuda>=11.6", "brand=tesla"},
+		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores requirements",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores legacy image requirements",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			image, err := NewCUDAImageFromEnv(tc.env)
+			require.NoError(t, err)
+
+			requirements, err := image.GetRequirements()
+			require.NoError(t, err)
+			require.ElementsMatch(t, tc.requirements, requirements)
+
+		})
+
+	}
+}

internal/config/image/devices.go

@@ -0,0 +1,127 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"strings"
+)
+
+// VisibleDevices represents the devices selected in a container image
+// through the NVIDIA_VISIBLE_DEVICES or other environment variables
+type VisibleDevices interface {
+	List() []string
+	Has(string) bool
+}
+
+var _ VisibleDevices = (*all)(nil)
+var _ VisibleDevices = (*none)(nil)
+var _ VisibleDevices = (*void)(nil)
+var _ VisibleDevices = (*devices)(nil)
+
+// NewVisibleDevices creates a VisibleDevices based on the value of the specified envvar.
+func NewVisibleDevices(envvars ...string) VisibleDevices {
+	for _, envvar := range envvars {
+		if envvar == "all" {
+			return all{}
+		}
+		if envvar == "none" {
+			return none{}
+		}
+		if envvar == "" || envvar == "void" {
+			return void{}
+		}
+	}
+
+	return newDevices(envvars...)
+}
+
+type all struct{}
+
+// List returns ["all"] for all devices
+func (a all) List() []string {
+	return []string{"all"}
+}
+
+// Has for all devices is true for any id except the empty ID
+func (a all) Has(id string) bool {
+	return id != ""
+}
+
+type none struct{}
+
+// List returns [""] for the none devices
+func (n none) List() []string {
+	return []string{""}
+}
+
+// Has for none devices is false for any id
+func (n none) Has(id string) bool {
+	return false
+}
+
+type void struct {
+	none
+}
+
+// List returns nil for the void devices
+func (v void) List() []string {
+	return nil
+}
+
+type devices struct {
+	len    int
+	lookup map[string]int
+}
+
+func newDevices(idOrCommaSeparated ...string) devices {
+	lookup := make(map[string]int)
+
+	i := 0
+	for _, commaSeparated := range idOrCommaSeparated {
+		for _, id := range strings.Split(commaSeparated, ",") {
+			lookup[id] = i
+			i++
+		}
+	}
+
+	d := devices{
+		len:    i,
+		lookup: lookup,
+	}
+	return d
+}
+
+// List returns the list of requested devices
+func (d devices) List() []string {
+	list := make([]string, d.len)
+
+	for id, i := range d.lookup {
+		list[i] = id
+	}
+
+	return list
+}
+
+// Has checks whether the specified ID is in the set of requested devices
+func (d devices) Has(id string) bool {
+	if id == "" {
+		return false
+	}
+
+	_, exist := d.lookup[id]
+	return exist
+}

internal/config/image/privileged.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged NVIDIA capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

@@ -16,20 +16,6 @@
 
 package config
 
-import (
-	"fmt"
-
-	"github.com/pelletier/go-toml"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	dockerRuncExecutableName = "docker-runc"
-	runcExecutableName       = "runc"
-
-	auto = "auto"
-)
-
 // RuntimeConfig stores the config options for the NVIDIA Container Runtime
 type RuntimeConfig struct {
 	DebugFilePath string `toml:"debug"`
@@ -44,52 +30,28 @@ type RuntimeConfig struct {
 // modesConfig defines (optional) per-mode configs
 type modesConfig struct {
 	CSV csvModeConfig `toml:"csv"`
+	CDI cdiModeConfig `toml:"cdi"`
+}
+
+type cdiModeConfig struct {
+	// SpecDirs allows for the default spec dirs for CDI to be overridden
+	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
 }
 
 type csvModeConfig struct {
 	MountSpecPath string `toml:"mount-spec-path"`
 }
 
-// dummy allows us to unmarshal only a RuntimeConfig from a *toml.Tree
-type dummy struct {
-	Runtime RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-// getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getRuntimeConfigFrom(toml *toml.Tree) (*RuntimeConfig, error) {
-	cfg := GetDefaultRuntimeConfig()
-
-	if toml == nil {
-		return cfg, nil
-	}
-
-	d := dummy{
-		Runtime: *cfg,
-	}
-
-	if err := toml.Unmarshal(&d); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
-	}
-
-	return &d.Runtime, nil
-}
-
 // GetDefaultRuntimeConfig defines the default values for the config
-func GetDefaultRuntimeConfig() *RuntimeConfig {
-	c := RuntimeConfig{
-		DebugFilePath: "/dev/null",
-		LogLevel:      logrus.InfoLevel.String(),
-		Runtimes: []string{
-			dockerRuncExecutableName,
-			runcExecutableName,
-		},
-		Mode: auto,
-		Modes: modesConfig{
-			CSV: csvModeConfig{
-				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
-			},
-		},
+func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
 	}
 
-	return &c
+	return &cfg.NVIDIAContainerRuntimeConfig, nil
 }

internal/config/toml.go

@@ -0,0 +1,218 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+
+	"github.com/pelletier/go-toml"
+)
+
+// Toml is a type for the TOML representation of a config.
+type Toml toml.Tree
+
+type options struct {
+	configFile string
+	required   bool
+}
+
+// Option is a functional option for loading TOML config files.
+type Option func(*options)
+
+// WithConfigFile sets the config file option.
+func WithConfigFile(configFile string) Option {
+	return func(o *options) {
+		o.configFile = configFile
+	}
+}
+
+// WithRequired sets the required option.
+// If this is set to true, a failure to open the specified file is treated as an error
+func WithRequired(required bool) Option {
+	return func(o *options) {
+		o.required = required
+	}
+}
+
+// New creates a new toml tree based on the provided options
+func New(opts ...Option) (*Toml, error) {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return o.loadConfigToml()
+}
+
+func (o options) loadConfigToml() (*Toml, error) {
+	filename := o.configFile
+	if filename == "" {
+		return defaultToml()
+	}
+
+	_, err := os.Stat(filename)
+	if os.IsNotExist(err) && o.required {
+		return nil, os.ErrNotExist
+	}
+
+	tomlFile, err := os.Open(filename)
+	if os.IsNotExist(err) {
+		return defaultToml()
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to load specified config file: %w", err)
+	}
+	defer tomlFile.Close()
+
+	return loadConfigTomlFrom(tomlFile)
+
+}
+
+func defaultToml() (*Toml, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	contents, err := toml.Marshal(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return loadConfigTomlFrom(bytes.NewReader(contents))
+}
+
+func loadConfigTomlFrom(reader io.Reader) (*Toml, error) {
+	tree, err := toml.LoadReader(reader)
+	if err != nil {
+		return nil, err
+	}
+	return (*Toml)(tree), nil
+}
+
+// Config returns the typed config associated with the toml tree.
+func (t *Toml) Config() (*Config, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	if t == nil {
+		return cfg, nil
+	}
+	if err := t.Unmarshal(cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal config: %v", err)
+	}
+	return cfg, nil
+}
+
+// Unmarshal wraps the toml.Tree Unmarshal function.
+func (t *Toml) Unmarshal(v interface{}) error {
+	return (*toml.Tree)(t).Unmarshal(v)
+}
+
+// Save saves the config to the specified Writer.
+func (t *Toml) Save(w io.Writer) (int64, error) {
+	contents, err := t.contents()
+	if err != nil {
+		return 0, err
+	}
+
+	n, err := w.Write(contents)
+	return int64(n), err
+}
+
+// contents returns the config TOML as a byte slice.
+// Any required formatting is applied.
+func (t Toml) contents() ([]byte, error) {
+	commented := t.commentDefaults()
+
+	buffer := bytes.NewBuffer(nil)
+
+	enc := toml.NewEncoder(buffer).Indentation("")
+	if err := enc.Encode((*toml.Tree)(commented)); err != nil {
+		return nil, fmt.Errorf("invalid config: %v", err)
+	}
+	return t.format(buffer.Bytes())
+}
+
+// format fixes the comments for the config to ensure that they start in column
+// 1 and are not followed by a space.
+func (t Toml) format(contents []byte) ([]byte, error) {
+	r, err := regexp.Compile(`(\n*)\s*?#\s*(\S.*)`)
+	if err != nil {
+		return nil, fmt.Errorf("unable to compile regexp: %v", err)
+	}
+	replaced := r.ReplaceAll(contents, []byte("$1#$2"))
+
+	return replaced, nil
+}
+
+// Delete deletes the specified key from the TOML config.
+func (t *Toml) Delete(key string) error {
+	return (*toml.Tree)(t).Delete(key)
+}
+
+// Get returns the value for the specified key.
+func (t *Toml) Get(key string) interface{} {
+	return (*toml.Tree)(t).Get(key)
+}
+
+// Set sets the specified key to the specified value in the TOML config.
+func (t *Toml) Set(key string, value interface{}) {
+	(*toml.Tree)(t).Set(key, value)
+}
+
+// commentDefaults applies the required comments for default values to the Toml.
+func (t *Toml) commentDefaults() *Toml {
+	asToml := (*toml.Tree)(t)
+	commentedDefaults := map[string]interface{}{
+		"swarm-resource": "DOCKER_RESOURCE_GPU",
+		"accept-nvidia-visible-devices-envvar-when-unprivileged": true,
+		"accept-nvidia-visible-devices-as-volume-mounts":         false,
+		"nvidia-container-cli.root":                              "/run/nvidia/driver",
+		"nvidia-container-cli.path":                              "/usr/bin/nvidia-container-cli",
+		"nvidia-container-cli.debug":                             "/var/log/nvidia-container-toolkit.log",
+		"nvidia-container-cli.ldcache":                           "/etc/ld.so.cache",
+		"nvidia-container-cli.no-cgroups":                        false,
+		"nvidia-container-cli.user":                              "root:video",
+		"nvidia-container-runtime.debug":                         "/var/log/nvidia-container-runtime.log",
+	}
+	for k, v := range commentedDefaults {
+		set := asToml.Get(k)
+		if !shouldComment(k, v, set) {
+			continue
+		}
+		asToml.SetWithComment(k, "", true, v)
+	}
+	return (*Toml)(asToml)
+}
+
+func shouldComment(key string, defaultValue interface{}, setTo interface{}) bool {
+	if key == "nvidia-container-cli.user" && defaultValue == setTo && isSuse() {
+		return false
+	}
+	if key == "nvidia-container-runtime.debug" && setTo == "/dev/null" {
+		return true
+	}
+	if setTo == nil || defaultValue == setTo || setTo == "" {
+		return true
+	}
+
+	return false
+}

internal/config/toml_test.go

@@ -0,0 +1,248 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTomlSave(t *testing.T) {
+	testCases := []struct {
+		description string
+		config      *Toml
+		expected    string
+	}{
+		{
+			description: "defaultConfig",
+			config: func() *Toml {
+				t, _ := defaultToml()
+				// TODO: We handle the ldconfig path specifically, since this is platform
+				// dependent.
+				(*toml.Tree)(t).Set("nvidia-container-cli.ldconfig", "OVERRIDDEN")
+				return t
+			}(),
+			expected: `
+#accept-nvidia-visible-devices-as-volume-mounts = false
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+#debug = "/var/log/nvidia-container-toolkit.log"
+environment = []
+#ldcache = "/etc/ld.so.cache"
+ldconfig = "OVERRIDDEN"
+load-kmods = true
+#no-cgroups = false
+#path = "/usr/bin/nvidia-container-cli"
+#root = "/run/nvidia/driver"
+#user = "root:video"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"
+log-level = "info"
+mode = "auto"
+runtimes = ["docker-runc", "runc", "crun"]
+
+[nvidia-container-runtime.modes]
+
+[nvidia-container-runtime.modes.cdi]
+annotation-prefixes = ["cdi.k8s.io/"]
+default-kind = "nvidia.com/gpu"
+spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+[nvidia-container-runtime.modes.csv]
+mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+path = "nvidia-container-runtime-hook"
+skip-mode-detection = false
+
+[nvidia-ctk]
+path = "nvidia-ctk"
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			buffer := new(bytes.Buffer)
+			_, err := tc.config.Save(buffer)
+			require.NoError(t, err)
+
+			require.EqualValues(t,
+				strings.TrimSpace(tc.expected),
+				strings.TrimSpace(buffer.String()),
+			)
+		})
+	}
+}
+
+func TestFormat(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "# comment",
+			expected: "#comment",
+		},
+		{
+			input:    " #comment",
+			expected: "#comment",
+		},
+		{
+			input:    " # comment",
+			expected: "#comment",
+		},
+		{
+			input: strings.Join([]string{
+				"some",
+				"# comment",
+				" # comment",
+				" #comment",
+				"other"}, "\n"),
+			expected: strings.Join([]string{
+				"some",
+				"#comment",
+				"#comment",
+				"#comment",
+				"other"}, "\n"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			actual, _ := (Toml{}).format([]byte(tc.input))
+			require.Equal(t, tc.expected, string(actual))
+		})
+	}
+}
+
+func TestGetFormattedConfig(t *testing.T) {
+	expectedLines := []string{
+		"#no-cgroups = false",
+		"#debug = \"/var/log/nvidia-container-toolkit.log\"",
+		"#debug = \"/var/log/nvidia-container-runtime.log\"",
+	}
+
+	contents, err := createEmpty().contents()
+	require.NoError(t, err)
+	lines := strings.Split(string(contents), "\n")
+
+	for _, line := range expectedLines {
+		require.Contains(t, lines, line)
+	}
+}
+
+func TestTomlContents(t *testing.T) {
+	testCases := []struct {
+		description string
+		contents    map[string]interface{}
+		expected    string
+	}{
+		{
+			description: "empty config returns commented defaults",
+			expected: `
+#accept-nvidia-visible-devices-as-volume-mounts = false
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+#no-cgroups = false
+#path = "/usr/bin/nvidia-container-cli"
+#root = "/run/nvidia/driver"
+#user = "root:video"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tree, err := toml.TreeFromMap(tc.contents)
+			require.NoError(t, err)
+			cfg := (*Toml)(tree)
+			contents, err := cfg.contents()
+			require.NoError(t, err)
+
+			require.EqualValues(t,
+				strings.TrimSpace(tc.expected),
+				strings.TrimSpace(string(contents)),
+			)
+		})
+	}
+}
+
+func TestConfigFromToml(t *testing.T) {
+	testCases := []struct {
+		description    string
+		contents       map[string]interface{}
+		expectedConfig *Config
+	}{
+		{
+			description: "empty config returns default config",
+			contents:    nil,
+			expectedConfig: func() *Config {
+				c, _ := GetDefault()
+				return c
+			}(),
+		},
+		{
+			description: "contents overrides default",
+			contents: map[string]interface{}{
+				"nvidia-container-runtime": map[string]interface{}{
+					"debug": "/some/log/file.log",
+					"mode":  "csv",
+				},
+			},
+			expectedConfig: func() *Config {
+				c, _ := GetDefault()
+				c.NVIDIAContainerRuntimeConfig.DebugFilePath = "/some/log/file.log"
+				c.NVIDIAContainerRuntimeConfig.Mode = "csv"
+				return c
+			}(),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tomlCfg := fromMap(tc.contents)
+			config, err := tomlCfg.Config()
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expectedConfig, config)
+		})
+	}
+}
+
+func fromMap(c map[string]interface{}) *Toml {
+	tree, _ := toml.TreeFromMap(c)
+	return (*Toml)(tree)
+}
+
+func createEmpty() *Toml {
+	return fromMap(nil)
+}

internal/config/toolkit-cli.go

@@ -16,31 +16,7 @@
 
 package config
 
-import "github.com/pelletier/go-toml"
-
 // CTKConfig stores the config options for the NVIDIA Container Toolkit CLI (nvidia-ctk)
 type CTKConfig struct {
 	Path string `toml:"path"`
 }
-
-// getCTKConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getCTKConfigFrom(toml *toml.Tree) *CTKConfig {
-	cfg := getDefaultCTKConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Path = toml.GetDefault("nvidia-ctk.path", cfg.Path).(string)
-
-	return cfg
-}
-
-// getDefaultCTKConfig defines the default values for the config
-func getDefaultCTKConfig() *CTKConfig {
-	c := CTKConfig{
-		Path: "nvidia-ctk",
-	}
-
-	return &c
-}

internal/discover/char_devices.go

@@ -0,0 +1,69 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
+func NewCharDeviceDiscoverer(logger logger.Interface, devices []string, root string) Discover {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(root),
+	)
+
+	return NewDeviceDiscoverer(logger, locator, root, devices)
+}
+
+// NewDeviceDiscoverer creates a discoverer which locates the specified set of device nodes using the specified locator.
+func NewDeviceDiscoverer(logger logger.Interface, locator lookup.Locator, root string, devices []string) Discover {
+	m := NewMounts(logger, locator, root, devices).(*mounts)
+
+	return (*charDevices)(m)
+}
+
+// Mounts returns the discovered mounts for the charDevices.
+// Since this explicitly specifies a device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices.
+// Here the device nodes are first discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	devicesAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range devicesAsMounts {
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
+	}
+
+	return devices, nil
+}

internal/discover/char_devices_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
@@ -49,7 +48,7 @@ func TestCharDevices(t *testing.T) {
 					required: []string{"required"},
 				},
 			),
-			expectedDevices: []Device{{Path: "located"}},
+			expectedDevices: []Device{{Path: "located", HostPath: "located"}},
 		},
 		{
 			description:          "dev devices returns error for nil lookup",
@@ -82,79 +81,3 @@ func TestCharDevices(t *testing.T) {
 		})
 	}
 }
-
-func TestNewFromMountSpec(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	locators := map[csv.MountSpecType]lookup.Locator{
-		"dev": &lookup.LocatorMock{},
-		"lib": &lookup.LocatorMock{},
-	}
-
-	testCases := []struct {
-		description        string
-		targets            []*csv.MountSpec
-		expectedError      error
-		expectedDiscoverer Discover
-	}{
-		{
-			description:        "empty targets returns None discoverer list",
-			expectedDiscoverer: &None{},
-		},
-		{
-			description: "unexpected locator returns error",
-			targets: []*csv.MountSpec{
-				{
-					Type: "foo",
-					Path: "bar",
-				},
-			},
-			expectedError: fmt.Errorf("no locator defined for foo"),
-		},
-		{
-			description: "creates discoverers based on type",
-			targets: []*csv.MountSpec{
-				{
-					Type: "dev",
-					Path: "dev0",
-				},
-				{
-					Type: "lib",
-					Path: "lib0",
-				},
-				{
-					Type: "dev",
-					Path: "dev1",
-				},
-			},
-			expectedDiscoverer: &list{
-				discoverers: []Discover{
-					(*charDevices)(
-						&mounts{
-							logger:   logger,
-							lookup:   locators["dev"],
-							required: []string{"dev0", "dev1"},
-						},
-					),
-					&mounts{
-						logger:   logger,
-						lookup:   locators["lib"],
-						required: []string{"lib0"},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			discoverer, err := newFromMountSpecs(logger, locators, tc.targets)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-				return
-			}
-			require.NoError(t, err)
-			require.EqualValues(t, tc.expectedDiscoverer, discoverer)
-		})
-	}
-}

internal/discover/csv.go

@@ -1,139 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package discover
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
-)
-
-// charDevices is a discover for a list of character devices
-type charDevices mounts
-
-var _ Discover = (*charDevices)(nil)
-
-// NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
-// The constructed discoverer is comprised of a list, with each element in the list being associated with a
-// single CSV files.
-func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
-	if len(files) == 0 {
-		logger.Warnf("No CSV files specified")
-		return None{}, nil
-	}
-
-	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
-	locators := map[csv.MountSpecType]lookup.Locator{
-		csv.MountSpecDev: lookup.NewCharDeviceLocator(logger, root),
-		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
-		// Libraries and symlinks are handled in the same way
-		csv.MountSpecLib: symlinkLocator,
-		csv.MountSpecSym: symlinkLocator,
-	}
-
-	var discoverers []Discover
-	for _, filename := range files {
-		d, err := NewFromCSVFile(logger, locators, filename)
-		if err != nil {
-			logger.Warnf("Skipping CSV file %v: %v", filename, err)
-			continue
-		}
-		discoverers = append(discoverers, d)
-	}
-
-	return &list{discoverers: discoverers}, nil
-}
-
-// NewFromCSVFile creates a discoverer for the specified CSV file. A logger is also supplied.
-// The constructed discoverer is comprised of a list, with each element in the list being associated with a particular
-// MountSpecType.
-func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, filename string) (Discover, error) {
-	// Create a discoverer for each file-kind combination
-	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse CSV file: %v", err)
-	}
-	if len(targets) == 0 {
-		return nil, fmt.Errorf("CSV file is empty")
-	}
-
-	return newFromMountSpecs(logger, locators, targets)
-}
-
-// newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
-// A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
-func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, targets []*csv.MountSpec) (Discover, error) {
-	if len(targets) == 0 {
-		return &None{}, nil
-	}
-
-	var discoverers []Discover
-	var mountSpecTypes []csv.MountSpecType
-	candidatesByType := make(map[csv.MountSpecType][]string)
-	for _, t := range targets {
-		if _, exists := candidatesByType[t.Type]; !exists {
-			mountSpecTypes = append(mountSpecTypes, t.Type)
-		}
-		candidatesByType[t.Type] = append(candidatesByType[t.Type], t.Path)
-	}
-
-	for _, t := range mountSpecTypes {
-		locator, exists := locators[t]
-		if !exists {
-			return nil, fmt.Errorf("no locator defined for '%v'", t)
-		}
-
-		m := &mounts{
-			logger:   logger,
-			lookup:   locator,
-			required: candidatesByType[t],
-		}
-
-		switch t {
-		case csv.MountSpecDev:
-			// For device mount specs, we insert a charDevices into the list of discoverers.
-			discoverers = append(discoverers, (*charDevices)(m))
-		default:
-			discoverers = append(discoverers, m)
-		}
-	}
-
-	return &list{discoverers: discoverers}, nil
-}
-
-// Mounts returns the discovered mounts for the charDevices. Since this explicitly specifies a
-// device list, the mounts are nil.
-func (d *charDevices) Mounts() ([]Mount, error) {
-	return nil, nil
-}
-
-// Devices returns the discovered devices for the charDevices. Here the device nodes are first
-// discovered as mounts and these are converted to devices.
-func (d *charDevices) Devices() ([]Device, error) {
-	devicesAsMounts, err := (*mounts)(d).Mounts()
-	if err != nil {
-		return nil, err
-	}
-	var devices []Device
-	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
-	}
-
-	return devices, nil
-}

internal/discover/discover.go

@@ -16,20 +16,17 @@
 
 package discover
 
-// Config represents the configuration options for discovery
-type Config struct {
-	Root                                    string
-	NVIDIAContainerToolkitCLIExecutablePath string
-}
-
 // Device represents a discovered character device.
 type Device struct {
-	Path string
+	HostPath string
+	Path     string
 }
 
 // Mount represents a discovered mount.
 type Mount struct {
-	Path string
+	HostPath string
+	Path     string
+	Options  []string
 }
 
 // Hook represents a discovered hook.
@@ -39,8 +36,9 @@ type Hook struct {
 	Args      []string
 }
 
-//go:generate moq -stub -out discover_mock.go . Discover
 // Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+//
+//go:generate moq -stub -out discover_mock.go . Discover
 type Discover interface {
 	Devices() ([]Device, error)
 	Mounts() ([]Mount, error)

internal/discover/discover_mock.go

@@ -13,25 +13,25 @@ var _ Discover = &DiscoverMock{}
 
 // DiscoverMock is a mock implementation of Discover.
 //
-// 	func TestSomethingThatUsesDiscover(t *testing.T) {
+//	func TestSomethingThatUsesDiscover(t *testing.T) {
 //
-// 		// make and configure a mocked Discover
-// 		mockedDiscover := &DiscoverMock{
-// 			DevicesFunc: func() ([]Device, error) {
-// 				panic("mock out the Devices method")
-// 			},
-// 			HooksFunc: func() ([]Hook, error) {
-// 				panic("mock out the Hooks method")
-// 			},
-// 			MountsFunc: func() ([]Mount, error) {
-// 				panic("mock out the Mounts method")
-// 			},
-// 		}
+//		// make and configure a mocked Discover
+//		mockedDiscover := &DiscoverMock{
+//			DevicesFunc: func() ([]Device, error) {
+//				panic("mock out the Devices method")
+//			},
+//			HooksFunc: func() ([]Hook, error) {
+//				panic("mock out the Hooks method")
+//			},
+//			MountsFunc: func() ([]Mount, error) {
+//				panic("mock out the Mounts method")
+//			},
+//		}
 //
-// 		// use mockedDiscover in code that requires Discover
-// 		// and then make assertions.
+//		// use mockedDiscover in code that requires Discover
+//		// and then make assertions.
 //
-// 	}
+//	}
 type DiscoverMock struct {
 	// DevicesFunc mocks the Devices method.
 	DevicesFunc func() ([]Device, error)
@@ -78,7 +78,8 @@ func (mock *DiscoverMock) Devices() ([]Device, error) {
 
 // DevicesCalls gets all the calls that were made to Devices.
 // Check the length with:
-//     len(mockedDiscover.DevicesCalls())
+//
+//	len(mockedDiscover.DevicesCalls())
 func (mock *DiscoverMock) DevicesCalls() []struct {
 } {
 	var calls []struct {
@@ -108,7 +109,8 @@ func (mock *DiscoverMock) Hooks() ([]Hook, error) {
 
 // HooksCalls gets all the calls that were made to Hooks.
 // Check the length with:
-//     len(mockedDiscover.HooksCalls())
+//
+//	len(mockedDiscover.HooksCalls())
 func (mock *DiscoverMock) HooksCalls() []struct {
 } {
 	var calls []struct {
@@ -138,7 +140,8 @@ func (mock *DiscoverMock) Mounts() ([]Mount, error) {
 
 // MountsCalls gets all the calls that were made to Mounts.
 // Check the length with:
-//     len(mockedDiscover.MountsCalls())
+//
+//	len(mockedDiscover.MountsCalls())
 func (mock *DiscoverMock) MountsCalls() []struct {
 } {
 	var calls []struct {

internal/discover/filter.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// Filter defines an interface for filtering discovered entities
+type Filter interface {
+	DeviceIsSelected(device Device) bool
+}
+
+// filtered represents a filtered discoverer
+type filtered struct {
+	Discover
+	logger logger.Interface
+	filter Filter
+}
+
+// newFilteredDisoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
+func newFilteredDisoverer(logger logger.Interface, applyTo Discover, filter Filter) Discover {
+	return filtered{
+		Discover: applyTo,
+		logger:   logger,
+		filter:   filter,
+	}
+}
+
+// Devices returns a filtered list of devices based on the specified filter.
+func (d filtered) Devices() ([]Device, error) {
+	devices, err := d.Discover.Devices()
+	if err != nil {
+		return nil, err
+	}
+
+	if d.filter == nil {
+		return devices, nil
+	}
+
+	var selected []Device
+	for _, device := range devices {
+		if d.filter.DeviceIsSelected(device) {
+			selected = append(selected, device)
+		}
+		d.logger.Debugf("skipping device %v", device)
+	}
+
+	return selected, nil
+}

internal/discover/gds.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type gdsDeviceDiscoverer struct {
+	None
+	logger  logger.Interface
+	devices Discover
+	mounts  Discover
+}
+
+// NewGDSDiscoverer creates a discoverer for GPUDirect Storage devices and mounts.
+func NewGDSDiscoverer(logger logger.Interface, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{"/dev/nvidia-fs*"},
+		root,
+	)
+
+	udev := NewMounts(
+		logger,
+		lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(root)),
+		root,
+		[]string{"/run/udev"},
+	)
+
+	cufile := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{"/etc/cufile.json"},
+	)
+
+	d := gdsDeviceDiscoverer{
+		logger:  logger,
+		devices: devices,
+		mounts:  Merge(udev, cufile),
+	}
+
+	return &d, nil
+}
+
+// Devices discovers the nvidia-fs device nodes for use with GPUDirect Storage
+func (d *gdsDeviceDiscoverer) Devices() ([]Device, error) {
+	return d.devices.Devices()
+}
+
+// Mounts discovers the required mounts for GPUDirect Storage.
+// If no devices are discovered the discovered mounts are empty
+func (d *gdsDeviceDiscoverer) Mounts() ([]Mount, error) {
+	devices, err := d.Devices()
+	if err != nil || len(devices) == 0 {
+		d.logger.Debugf("No nvidia-fs devices detected; skipping detection of mounts")
+		return nil, nil
+	}
+
+	return d.mounts.Mounts()
+}

internal/discover/graphics.go

@@ -0,0 +1,386 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
+)
+
+// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
+func NewGraphicsDiscoverer(logger logger.Interface, devices image.VisibleDevices, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	mounts, err := NewGraphicsMountsDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
+	}
+
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, driverRoot, nvidiaCTKPath)
+
+	discover := Merge(
+		Merge(drmDeviceNodes, drmByPathSymlinks),
+		mounts,
+	)
+
+	return discover, nil
+}
+
+// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
+func NewGraphicsMountsDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct library locator: %v", err)
+	}
+	libraries := NewMounts(
+		logger,
+		locator,
+		driverRoot,
+		[]string{
+			"libnvidia-egl-gbm.so",
+		},
+	)
+
+	jsonMounts := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/etc", "/usr/share"),
+		),
+		driverRoot,
+		[]string{
+			"glvnd/egl_vendor.d/10_nvidia.json",
+			"vulkan/icd.d/nvidia_icd.json",
+			"vulkan/icd.d/nvidia_layers.json",
+			"vulkan/implicit_layer.d/nvidia_layers.json",
+			"egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"egl/egl_external_platform.d/10_nvidia_wayland.json",
+			"nvidia/nvoptix.bin",
+		},
+	)
+
+	xorg := optionalXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+
+	discover := Merge(
+		libraries,
+		jsonMounts,
+		xorg,
+	)
+
+	return discover, nil
+}
+
+type drmDevicesByPath struct {
+	None
+	logger        logger.Interface
+	nvidiaCTKPath string
+	driverRoot    string
+	devicesFrom   Discover
+}
+
+// newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
+func newCreateDRMByPathSymlinks(logger logger.Interface, devices Discover, driverRoot string, nvidiaCTKPath string) Discover {
+	d := drmDevicesByPath{
+		logger:        logger,
+		nvidiaCTKPath: nvidiaCTKPath,
+		driverRoot:    driverRoot,
+		devicesFrom:   devices,
+	}
+
+	return &d
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d drmDevicesByPath) Hooks() ([]Hook, error) {
+	devices, err := d.devicesFrom.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices for by-path symlinks: %v", err)
+	}
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	links, err := d.getSpecificLinkArgs(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []Hook{hook}, nil
+}
+
+// getSpecificLinkArgs returns the required specic links that need to be created
+func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error) {
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[filepath.Base(d.HostPath)] = true
+	}
+
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.driverRoot),
+	)
+	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
+	if err != nil {
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
+	}
+
+	var links []string
+	for _, c := range candidates {
+		device, err := os.Readlink(c)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", c)
+			continue
+		}
+
+		if selectedDevices[filepath.Base(device)] {
+			d.logger.Debugf("adding device symlink %v -> %v", c, device)
+			links = append(links, fmt.Sprintf("%v::%v", device, c))
+		}
+	}
+
+	return links, nil
+}
+
+// newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
+func newDRMDeviceDiscoverer(logger logger.Interface, devices image.VisibleDevices, driverRoot string) (Discover, error) {
+	allDevices := NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+		),
+		driverRoot,
+		[]string{
+			"/dev/dri/card*",
+			"/dev/dri/renderD*",
+		},
+	)
+
+	filter, err := newDRMDeviceFilter(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
+	}
+
+	// We return a discoverer that applies the DRM device filter created above to all discovered DRM device nodes.
+	d := newFilteredDisoverer(
+		logger,
+		allDevices,
+		filter,
+	)
+
+	return d, err
+}
+
+// newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
+func newDRMDeviceFilter(logger logger.Interface, devices image.VisibleDevices, driverRoot string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read GPU information: %v", err)
+	}
+
+	var selectedBusIds []string
+	for _, f := range gpuInformationPaths {
+		info, err := proc.ParseGPUInformationFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %v: %v", f, err)
+		}
+		uuid := info[proc.GPUInfoGPUUUID]
+		busID := info[proc.GPUInfoBusLocation]
+		minor := info[proc.GPUInfoDeviceMinor]
+
+		if devices.Has(minor) || devices.Has(uuid) || devices.Has(busID) {
+			selectedBusIds = append(selectedBusIds, busID)
+		}
+	}
+
+	filter := make(selectDeviceByPath)
+	for _, busID := range selectedBusIds {
+		drmDeviceNodes, err := drm.GetDeviceNodesByBusID(busID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", busID, err)
+		}
+		for _, drmDeviceNode := range drmDeviceNodes {
+			filter[filepath.Join(drmDeviceNode)] = true
+		}
+	}
+
+	return filter, nil
+}
+
+type xorgHooks struct {
+	libraries     Discover
+	driverVersion string
+	nvidiaCTKPath string
+}
+
+var _ Discover = (*xorgHooks)(nil)
+
+// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
+// If the creation of the discoverer fails, a None discoverer is returned.
+func optionalXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		logger.Warningf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
+		return None{}
+	}
+	return xorg
+}
+
+func newXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate(".*.*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+	libcudaPath := libCudaPaths[0]
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	libRoot := filepath.Dir(libcudaPath)
+	xorgLibs := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"nvidia/xorg/nvidia_drv.so",
+			fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
+		},
+	)
+	xorgHooks := xorgHooks{
+		libraries:     xorgLibs,
+		driverVersion: version,
+		nvidiaCTKPath: nvidiaCTKPath,
+	}
+
+	xorgConfg := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/usr/share"),
+		),
+		driverRoot,
+		[]string{"X11/xorg.conf.d/10-nvidia.conf"},
+	)
+
+	d := Merge(
+		xorgLibs,
+		xorgConfg,
+		xorgHooks,
+	)
+
+	return d, nil
+}
+
+// Devices returns no devices for Xorg
+func (m xorgHooks) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a hook to create symlinks for Xorg libraries
+func (m xorgHooks) Hooks() ([]Hook, error) {
+	mounts, err := m.libraries.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get mounts: %v", err)
+	}
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+
+	var target string
+	for _, mount := range mounts {
+		filename := filepath.Base(mount.HostPath)
+		if filename == "libglxserver_nvidia.so."+m.driverVersion {
+			target = mount.Path
+		}
+	}
+
+	if target == "" {
+		return nil, nil
+	}
+
+	link := strings.TrimSuffix(target, "."+m.driverVersion)
+	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
+	symlinkHook := CreateCreateSymlinkHook(
+		m.nvidiaCTKPath,
+		links,
+	)
+
+	return symlinkHook.Hooks()
+}
+
+// Mounts returns the libraries required for Xorg
+func (m xorgHooks) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// selectDeviceByPath is a filter that allows devices to be selected by the path
+type selectDeviceByPath map[string]bool
+
+var _ Filter = (*selectDeviceByPath)(nil)
+
+// DeviceIsSelected determines whether the device's path has been selected
+func (s selectDeviceByPath) DeviceIsSelected(device Device) bool {
+	return s[device.Path]
+}
+
+// MountIsSelected is always true
+func (s selectDeviceByPath) MountIsSelected(Mount) bool {
+	return true
+}
+
+// HookIsSelected is always true
+func (s selectDeviceByPath) HookIsSelected(Hook) bool {
+	return true
+}

internal/discover/hooks.go

@@ -0,0 +1,67 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+)
+
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+// CreateCreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateCreateSymlinkHook(nvidiaCTKPath string, links []string) Discover {
+	if len(links) == 0 {
+		return None{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateNvidiaCTKHook(
+		nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+}
+
+// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
+func CreateNvidiaCTKHook(nvidiaCTKPath string, hookName string, additionalArgs ...string) Hook {
+	return Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      nvidiaCTKPath,
+		Args:      append([]string{filepath.Base(nvidiaCTKPath), "hook", hookName}, additionalArgs...),
+	}
+}

internal/discover/icp_test.go

@@ -0,0 +1,61 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIPCMounts(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	l := ipcMounts(
+		mounts{
+			logger: logger,
+			lookup: &lookup.LocatorMock{
+				LocateFunc: func(path string) ([]string, error) {
+					return []string{"/host/path"}, nil
+				},
+			},
+			required: []string{"target"},
+		},
+	)
+
+	mounts, err := l.Mounts()
+	require.NoError(t, err)
+
+	require.EqualValues(
+		t,
+		[]Mount{
+			{
+				HostPath: "/host/path",
+				Path:     "/host/path",
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+					"noexec",
+				},
+			},
+		},
+		mounts,
+	)
+}

internal/discover/ipc.go

@@ -0,0 +1,78 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type ipcMounts mounts
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger logger.Interface, driverRoot string) (Discover, error) {
+	sockets := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/run", "/var/run"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/nvidia-persistenced/socket",
+			"/nvidia-fabricmanager/socket",
+		},
+	)
+
+	mps := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	d := Merge(
+		(*ipcMounts)(sockets),
+		(*ipcMounts)(mps),
+	)
+	return d, nil
+}
+
+// Mounts returns the discovered mounts with "noexec" added to the mount options.
+func (d *ipcMounts) Mounts() ([]Mount, error) {
+	mounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+
+	var modifiedMounts []Mount
+	for _, m := range mounts {
+		mount := m
+		mount.Options = append(m.Options, "noexec")
+		modifiedMounts = append(modifiedMounts, mount)
+	}
+
+	return modifiedMounts, nil
+}

internal/discover/ldconfig.go

@@ -19,36 +19,27 @@ package discover
 import (
 	"fmt"
 	"path/filepath"
-	"sort"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
 // NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
-func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
+func NewLDCacheUpdateHook(logger logger.Interface, mounts Discover, nvidiaCTKPath string) (Discover, error) {
 	d := ldconfig{
-		logger:                  logger,
-		mountsFrom:              mounts,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+		logger:        logger,
+		nvidiaCTKPath: nvidiaCTKPath,
+		mountsFrom:    mounts,
 	}
 
 	return &d, nil
 }
 
-const (
-	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
-)
-
 type ldconfig struct {
 	None
-	logger                  *logrus.Logger
-	mountsFrom              Discover
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
+	logger        logger.Interface
+	nvidiaCTKPath string
+	mountsFrom    Discover
 }
 
 // Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
@@ -57,70 +48,75 @@ func (d ldconfig) Hooks() ([]Hook, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
 	}
-
-	libDirs := getLibDirs(mounts)
-
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{hookPath, "hook", "update-ldcache"}
-	for _, f := range libDirs {
-		args = append(args, "--folder", f)
-	}
-	h := Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
-
+	h := CreateLDCacheUpdateHook(
+		d.nvidiaCTKPath,
+		getLibraryPaths(mounts),
+	)
 	return []Hook{h}, nil
 }
 
-// getLibDirs extracts the library dirs from the specified mounts
-func getLibDirs(mounts []Mount) []string {
-	var paths []string
-	checked := make(map[string]bool)
-
-	for _, m := range mounts {
-		dir := filepath.Dir(m.Path)
-		if dir == "" {
-			continue
-		}
-
-		_, exists := checked[dir]
-		if exists {
-			continue
-		}
-		checked[dir] = isLibName(filepath.Base(m.Path))
-
-		if checked[dir] {
-			paths = append(paths, dir)
-		}
+// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
+func CreateLDCacheUpdateHook(executable string, libraries []string) Hook {
+	var args []string
+	for _, f := range uniqueFolders(libraries) {
+		args = append(args, "--folder", f)
 	}
 
-	sort.Strings(paths)
+	hook := CreateNvidiaCTKHook(
+		executable,
+		"update-ldcache",
+		args...,
+	)
 
+	return hook
+
+}
+
+// getLibraryPaths extracts the library dirs from the specified mounts
+func getLibraryPaths(mounts []Mount) []string {
+	var paths []string
+	for _, m := range mounts {
+		if !isLibName(m.Path) {
+			continue
+		}
+		paths = append(paths, m.Path)
+	}
 	return paths
 }
 
 // isLibName checks if the specified filename is a library (i.e. ends in `.so*`)
 func isLibName(filename string) bool {
-	parts := strings.Split(filename, ".")
 
-	for _, p := range parts {
-		if p == "so" {
-			return true
-		}
+	base := filepath.Base(filename)
+
+	isLib, err := filepath.Match("lib?*.so*", base)
+	if !isLib || err != nil {
+		return false
 	}
 
-	return false
+	parts := strings.Split(base, ".so")
+	if len(parts) == 1 {
+		return true
+	}
+
+	return parts[len(parts)-1] == "" || strings.HasPrefix(parts[len(parts)-1], ".")
+}
+
+// uniqueFolders returns the unique set of folders for the specified files
+func uniqueFolders(libraries []string) []string {
+	var paths []string
+	checked := make(map[string]bool)
+
+	for _, l := range libraries {
+		dir := filepath.Dir(l)
+		if dir == "" {
+			continue
+		}
+		if checked[dir] {
+			continue
+		}
+		checked[dir] = true
+		paths = append(paths, dir)
+	}
+	return paths
 }

internal/discover/ldconfig_test.go

@@ -0,0 +1,163 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testNvidiaCTKPath = "/foo/bar/nvidia-ctk"
+)
+
+func TestLDCacheUpdateHook(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		mounts        []Mount
+		mountError    error
+		expectedError error
+		expectedArgs  []string
+	}{
+		{
+			description:  "empty mounts",
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache"},
+		},
+		{
+			description:   "mount error",
+			mountError:    fmt.Errorf("mountError"),
+			expectedError: fmt.Errorf("mountError"),
+		},
+		{
+			description: "library folders are added to args",
+			mounts: []Mount{
+				{
+					Path: "/usr/local/lib/libfoo.so",
+				},
+				{
+					Path: "/usr/bin/notlib",
+				},
+				{
+					Path: "/usr/local/libother/libfoo.so",
+				},
+				{
+					Path: "/usr/local/lib/libbar.so",
+				},
+			},
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
+		},
+		{
+			description: "host paths are ignored",
+			mounts: []Mount{
+				{
+					HostPath: "/usr/local/other/libfoo.so",
+					Path:     "/usr/local/lib/libfoo.so",
+				},
+			},
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache", "--folder", "/usr/local/lib"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mountMock := &DiscoverMock{
+				MountsFunc: func() ([]Mount, error) {
+					return tc.mounts, tc.mountError
+				},
+			}
+			expectedHook := Hook{
+				Path:      testNvidiaCTKPath,
+				Args:      tc.expectedArgs,
+				Lifecycle: "createContainer",
+			}
+
+			d, err := NewLDCacheUpdateHook(logger, mountMock, testNvidiaCTKPath)
+			require.NoError(t, err)
+
+			hooks, err := d.Hooks()
+			require.Len(t, mountMock.MountsCalls(), 1)
+			require.Len(t, mountMock.DevicesCalls(), 0)
+			require.Len(t, mountMock.HooksCalls(), 0)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Len(t, hooks, 1)
+
+			require.EqualValues(t, hooks[0], expectedHook)
+
+			devices, err := d.Devices()
+			require.NoError(t, err)
+			require.Empty(t, devices)
+
+			mounts, err := d.Mounts()
+			require.NoError(t, err)
+			require.Empty(t, mounts)
+
+		})
+	}
+
+}
+
+func TestIsLibName(t *testing.T) {
+	testCases := []struct {
+		name  string
+		isLib bool
+	}{
+		{
+			name:  "",
+			isLib: false,
+		},
+		{
+			name:  "lib/not/.so",
+			isLib: false,
+		},
+		{
+			name:  "lib.so",
+			isLib: false,
+		},
+		{
+			name:  "notlibcuda.so",
+			isLib: false,
+		},
+		{
+			name:  "libcuda.so",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.so.1",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.soNOT",
+			isLib: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.isLib, isLibName(tc.name))
+		})
+	}
+}

internal/discover/list.go

@@ -27,8 +27,8 @@ type list struct {
 
 var _ Discover = (*list)(nil)
 
-// NewList creates a discoverer that is the composite of a list of discoveres.
-func NewList(d ...Discover) Discover {
+// Merge creates a discoverer that is the composite of a list of discoveres.
+func Merge(d ...Discover) Discover {
 	l := list{
 		discoverers: d,
 	}

internal/discover/mofed.go

@@ -0,0 +1,33 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// NewMOFEDDiscoverer creates a discoverer for MOFED devices.
+func NewMOFEDDiscoverer(logger logger.Interface, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{
+			"/dev/infiniband/uverbs*",
+			"/dev/infiniband/rdma_cm",
+		},
+		root,
+	)
+
+	return devices, nil
+}