Merge pull request #380 from tariq1890/cpick-exists-fallback

[R550 driver support] add fallback logic to device.Exists(name)

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker:stable
+  image: docker
   services:
-    - name: docker:stable-dind
+    - name: docker:dind
       command: ["--experimental"]
 
 variables:
@@ -23,6 +23,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -34,51 +35,68 @@ stages:
   - scan
   - release
 
-# Define the distribution targets
-.dist-amazonlinux2:
-  variables:
-    DIST: amazonlinux2
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
 
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
+.main-or-manual:
+  rules:
+    - !reference [.pipeline-trigger-rules, rules]
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: manual
+
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
+  rules:
+    - !reference [.main-or-manual, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
+
+# Define the distribution targets
 .dist-centos7:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: centos7
-    CVE_UPDATES: "cyrus-sasl-lib"
 
 .dist-centos8:
   variables:
     DIST: centos8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-debian10:
-  variables:
-    DIST: debian10
-
-.dist-debian9:
-  variables:
-    DIST: debian9
-
-.dist-opensuse-leap15.1:
-  variables:
-    DIST: opensuse-leap15.1
 
 .dist-ubi8:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubi8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-ubuntu16.04:
-  variables:
-    DIST: ubuntu16.04
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
 
 .dist-ubuntu20.04:
   variables:
     DIST: ubuntu20.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
 
 .dist-packaging:
   variables:
@@ -98,6 +116,8 @@ stages:
     ARCH: arm64
 
 .arch-ppc64le:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     ARCH: ppc64le
 
@@ -138,7 +158,7 @@ test-packaging:
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.3.10
+    - export REGCTL_VERSION=v0.4.5
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
@@ -194,6 +214,8 @@ test-packaging:
 .release:external:
   extends:
     - .release
+  variables:
+    FORCE_PUBLISH_IMAGES: "yes"
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -217,16 +239,6 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
 release:staging-ubuntu20.04:
   extends:
     - .release:staging

.github/workflows/pre-sanity.yml

@@ -0,0 +1,22 @@
+name: Run pre sanity
+
+# run this workflow for each commit
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+             
+      - name: Build dev image
+        run: make .build-image
+      
+      - name: Build
+        run: make docker-build
+
+      - name: Tests
+        run: make docker-coverage
+
+      - name: Checks
+        run: make docker-check

.gitignore

@@ -1,9 +1,13 @@
 dist
+artifacts
 *.swp
 *.swo
 /coverage.out*
 /test/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
+/nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk
 /shared-*
+/release-*

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,7 +94,7 @@ unit-tests:
     - .multi-arch-build
     - .package-artifacts
   stage: package-build
-  timeout: 2h 30m
+  timeout: 3h
   script:
     - ./scripts/build-packages.sh ${DIST}-${ARCH}
 
@@ -102,25 +102,35 @@ unit-tests:
     name: ${ARTIFACTS_NAME}
     paths:
       - ${ARTIFACTS_ROOT}
+  needs:
+    - job: package-meta-packages
+      artifacts: true
 
 # Define the package build targets
-package-amazonlinux2-aarch64:
+package-meta-packages:
   extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-aarch64
+    - .package-artifacts
+  stage: package-build
+  variables:
+    SKIP_LIBNVIDIA_CONTAINER: "yes"
+    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
+  parallel:
+    matrix:
+      - PACKAGING: [deb, rpm]
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+  script:
+    - ./scripts/build-packages.sh ${PACKAGING}
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
 
-package-amazonlinux2-x86_64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-package-centos7-ppc64le:
+package-centos7-aarch64:
   extends:
     - .package-build
     - .dist-centos7
-    - .arch-ppc64le
+    - .arch-aarch64
 
 package-centos7-x86_64:
   extends:
@@ -128,54 +138,12 @@ package-centos7-x86_64:
     - .dist-centos7
     - .arch-x86_64
 
-package-centos8-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-aarch64
-
 package-centos8-ppc64le:
   extends:
     - .package-build
     - .dist-centos8
     - .arch-ppc64le
 
-package-centos8-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-x86_64
-
-package-debian10-amd64:
-  extends:
-    - .package-build
-    - .dist-debian10
-    - .arch-amd64
-
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-opensuse-leap15.1-x86_64:
-  extends:
-    - .package-build
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -216,7 +184,7 @@ package-ubuntu18.04-ppc64le:
   before_script:
     - !reference [.buildx-setup, before_script]
 
-    - apk add --no-cache bash make
+    - apk add --no-cache bash make git
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
@@ -228,7 +196,6 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
-    - package-centos7-ppc64le
     - package-centos7-x86_64
 
 image-ubi8:
@@ -237,20 +204,9 @@ image-ubi8:
     - .package-artifacts
     - .dist-ubi8
   needs:
-    # Note: The ubi8 image uses the centos8 packages
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    # Note: The ubi8 image uses the centos7 packages
+    - package-centos7-aarch64
+    - package-centos7-x86_64
 
 image-ubuntu20.04:
   extends:
@@ -260,7 +216,8 @@ image-ubuntu20.04:
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -269,21 +226,24 @@ image-packaging:
     - .package-artifacts
     - .dist-packaging
   needs:
-    - package-amazonlinux2-aarch64
-    - package-amazonlinux2-x86_64
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-    - package-centos8-aarch64
-    - package-centos8-ppc64le
-    - package-centos8-x86_64
-    - package-debian10-amd64
-    - package-debian9-amd64
-    - package-opensuse-leap15.1-x86_64
-    - package-ubuntu16.04-amd64
-    - package-ubuntu16.04-ppc64le
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-amd64
+    - job: package-ubuntu18.04-arm64
+    - job: package-amazonlinux2-aarch64
+      optional: true
+    - job: package-amazonlinux2-x86_64
+      optional: true
+    - job: package-centos7-aarch64
+      optional: true
+    - job: package-centos7-x86_64
+      optional: true
+    - job: package-centos8-ppc64le
+      optional: true
+    - job: package-debian10-amd64
+      optional: true
+    - job: package-opensuse-leap15.1-x86_64
+      optional: true
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # Define publish test helpers
 .test:toolkit:
@@ -315,34 +275,6 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu18.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-containerd-ubuntu18.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-crio-ubuntu18.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-docker-ubuntu18.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
 test-toolkit-ubuntu20.04:
   extends:
     - .test:toolkit

.gitmodules

@@ -1,10 +1,4 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
 	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
-	branch = main
-[submodule "third_party/nvidia-container-runtime"]
-	path = third_party/nvidia-container-runtime
-	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
-[submodule "third_party/nvidia-docker"]
-	path = third_party/nvidia-docker
-	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
+	branch = release-1.14

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,6 +35,9 @@ variables:
   # Define the public staging registry
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
+  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
 .image-pull:
   stage: image-build
@@ -48,8 +51,9 @@ variables:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PUSH_MULTIPLE_TAGS: "false"
   # We delay the job start to allow the public pipeline to generate the required images.
-  when: delayed
-  start_in: 30 minutes
+  rules:
+    - when: delayed
+      start_in: 30 minutes
   timeout: 30 minutes
   retry:
     max: 2
@@ -67,29 +71,24 @@ variables:
 
 image-centos7:
   extends:
-    - .image-pull
     - .dist-centos7
+    - .image-pull
 
 image-ubi8:
   extends:
-    - .image-pull
     - .dist-ubi8
-
-image-ubuntu18.04:
-  extends:
     - .image-pull
-    - .dist-ubuntu18.04
 
 image-ubuntu20.04:
   extends:
-    - .image-pull
     - .dist-ubuntu20.04
+    - .image-pull
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
   extends:
-    - .image-pull
     - .dist-packaging
+    - .image-pull
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -106,10 +105,10 @@ image-packaging:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
-  except:
-    variables:
-      - $SKIP_SCANS && $SKIP_SCANS == "yes"
+    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
+  rules:
+    - if: $SKIP_SCANS != "yes"
+    - when: manual
   before_script:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     # TODO: We should specify the architecture here and scan all architectures
@@ -121,6 +120,7 @@ image-packaging:
     - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
   script:
     - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+    - rm -f "${IMAGE_ARCHIVE}"
   artifacts:
     when: always
     expire_in: 1 week
@@ -134,63 +134,53 @@ image-packaging:
 # Define the scan targets
 scan-centos7-amd64:
   extends:
-    - .scan
     - .dist-centos7
     - .platform-amd64
+    - .scan
   needs:
     - image-centos7
 
-scan-centos7-arm64:
-  extends:
-    - .scan
-    - .dist-centos7
-    - .platform-arm64
-  needs:
-    - image-centos7
-    - scan-centos7-amd64
-
-scan-ubuntu18.04-amd64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-amd64
-  needs:
-    - image-ubuntu18.04
-
 scan-ubuntu20.04-amd64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-amd64
+    - .scan
   needs:
     - image-ubuntu20.04
 
 scan-ubuntu20.04-arm64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-arm64
+    - .scan
   needs:
     - image-ubuntu20.04
     - scan-ubuntu20.04-amd64
 
 scan-ubi8-amd64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-amd64
+    - .scan
   needs:
     - image-ubi8
 
 scan-ubi8-arm64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-arm64
+    - .scan
   needs:
     - image-ubi8
     - scan-ubi8-amd64
 
+scan-packaging:
+  extends:
+    - .dist-packaging
+    - .scan
+  needs:
+    - image-packaging
+
 # Define external release helpers
 .release:ngc:
   extends:
@@ -201,12 +191,48 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
+.release:packages:
+  stage: release
   needs:
-    - image-ubuntu18.04
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+    ARTIFACTS_DIR: "${CI_PROJECT_DIR}/artifacts"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash git
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+    - rm -rf ${ARTIFACTS_DIR}
+
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
+
+release:archive:
+  extends:
+    - .release:external
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}"
+  script:
+    - apk add --no-cache bash git
+    - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}"
 
 release:staging-ubuntu20.04:
   extends:
@@ -219,20 +245,20 @@ release:staging-ubuntu20.04:
 # Release to NGC
 release:ngc-centos7:
   extends:
-    - .release:ngc
     - .dist-centos7
-
-release:ngc-ubuntu18.04:
-  extends:
     - .release:ngc
-    - .dist-ubuntu18.04
 
 release:ngc-ubuntu20.04:
   extends:
-    - .release:ngc
     - .dist-ubuntu20.04
+    - .release:ngc
 
 release:ngc-ubi8:
   extends:
-    - .release:ngc
     - .dist-ubi8
+    - .release:ngc
+
+release:ngc-packaging:
+  extends:
+    - .dist-packaging
+    - .release:ngc

CHANGELOG.md

@@ -1,5 +1,227 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.14.6
+* Add support for extracting device major number from `/proc/devices` if `nvidia` is used as a device name over `nvidia-frontend`.
+* Add support for selecting IMEX channels using the NVIDIA_IMEX_CHANNELS environement variable.
+
+## v1.14.5
+* Fix `nvidia-ctk runtime configure --cdi.enabled` for Docker. This was incorrectly setting `experimental = true` instead
+  of setting `features.cdi = true`.
+
+## v1.14.4
+* Include `nvidia/nvoptix.bin` in list of graphics mounts.
+* Include `vulkan/icd.d/nvidia_layers.json` in list of graphics mounts.
+* Fixed bug in `nvidia-ctk config` command when using `--set`. The types of applied config options are now applied correctly.
+* Log explicitly requested runtime mode.
+* Remove package dependency on libseccomp.
+* Added detection of libnvdxgdmal.so.1 on WSL2.
+* Fix bug in determining default nvidia-container-runtime.user config value on SUSE-based systems.
+* Add `crun` to the list of configured low-level runtimes.
+* Add `--cdi.enabled` option to `nvidia-ctk runtime configure` command to enable CDI in containerd.
+* Added support for `nvidia-ctk runtime configure --enable-cdi` for the `docker` runtime. Note that this requires Docker >= 25.
+
+* [toolkit-container] Bump CUDA base image version to 12.3.1.
+* [libnvidia-container] Added detection of libnvdxgdmal.so.1 on WSL2.
+
+## v1.14.3
+* [toolkit-container] Bump CUDA base image version to 12.2.2.
+
+## v1.14.2
+* Fix bug on Tegra-based systems where symlinks were not created in containers.
+* Add --csv.ignore-pattern command line option to nvidia-ctk cdi generate command.
+
+## v1.14.1
+* Fixed bug where contents of `/etc/nvidia-container-runtime/config.toml` is ignored by the NVIDIA Container Runtime Hook.
+
+* [libnvidia-container] Use libelf.so on RPM-based systems due to removed mageia repositories hosting pmake and bmake.
+
+## v1.14.0
+* Promote v1.14.0-rc.3 to v1.14.0
+
+## v1.14.0-rc.3
+* Added support for generating OCI hook JSON file to `nvidia-ctk runtime configure` command.
+* Remove installation of OCI hook JSON from RPM package.
+* Refactored config for `nvidia-container-runtime-hook`.
+* Added a `nvidia-ctk config` command which supports setting config options using a `--set` flag.
+* Added `--library-search-path` option to `nvidia-ctk cdi generate` command in `csv` mode. This allows folders where
+  libraries are located to be specified explicitly.
+* Updated go-nvlib to support devices which are not present in the PCI device database. This allows the creation of dev/char symlinks on systems with such devices installed.
+* Added `UsesNVGPUModule` info function for more robust platform detection. This is required on Tegra-based systems where libnvidia-ml.so is also supported.
+
+* [toolkit-container] Set `NVIDIA_VISIBLE_DEVICES=void` to prevent injection of NVIDIA devices and drivers into the NVIDIA Container Toolkit container.
+
+## v1.14.0-rc.2
+* Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
+* Remove dependency on coreutils when installing package on RPM-based systems.
+* Create ouput folders if required when running `nvidia-ctk runtime configure`
+* Generate default config as post-install step.
+* Added support for detecting GSP firmware at custom paths when generating CDI specifications.
+* Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.
+
+* [libnvidia-container] Include Shared Compiler Library (libnvidia-gpucomp.so) in the list of compute libaries.
+
+* [toolkit-container] Ensure that common envvars have higher priority when configuring the container engines.
+* [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
+
+## v1.14.0-rc.1
+
+* Add support for updating containerd configs to the `nvidia-ctk runtime configure` command.
+* Create file in `etc/ld.so.conf.d` with permissions `644` to support non-root containers.
+* Generate CDI specification files with `644` permissions to allow rootless applications (e.g. podman)
+* Add `nvidia-ctk cdi list` command to show the known CDI devices.
+* Add support for generating merged devices (e.g. `all` device) to the nvcdi API.
+* Use *.* pattern to locate libcuda.so when generating a CDI specification to support platforms where a patch version is not specified.
+* Update go-nvlib to skip devices that are not MIG capable when generating CDI specifications.
+* Add `nvidia-container-runtime-hook.path` config option to specify NVIDIA Container Runtime Hook path explicitly.
+* Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
+* Add option to load kernel modules when creating device nodes
+* Add option to create device nodes when creating `/dev/char` symlinks
+
+* [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
+
+* [toolkit-container] Allow same envars for all runtime configs
+
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
+## v1.12.0-rc.1
+
+* Add support for multiple Docker Swarm resources
+* Improve injection of Vulkan configurations and libraries
+* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
+* [libnvidia-container] Include NVVM compiler library in compute libs
+
+## v1.11.0
+
+* Promote v1.11.0-rc.3 to v1.11.0
+
+## v1.11.0-rc.3
+
+* Build fedora35 packages
+* Introduce an `nvidia-container-toolkit-base` package for better dependency management
+* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
+* Inject platform files into container on Tegra-based systems
+* [toolkit container] Update CUDA base images to 11.7.1
+* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
+
+## v1.11.0-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.0-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
 ## v1.10.0
 
 * Promote v1.10.0-rc.3 to v1.10.0

DEVELOPMENT.md

@@ -19,7 +19,7 @@ where `TARGET` is a make target that is valid for each of the sub-components.
 
 These include:
 * `ubuntu18.04-amd64`
-* `centos8-x86_64`
+* `centos7-x86_64`
 
 If no `TARGET` is specified, all valid release targets are built.
 

Makefile

@@ -39,7 +39,7 @@ CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
 CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate licenses $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
@@ -51,6 +51,7 @@ CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 else
 CLI_VERSION = $(VERSION)
 endif
+CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
 
 GOOS ?= linux
 
@@ -60,7 +61,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.gitCommit=$(GIT_COMMIT) -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-extldflags=-Wl,-z,lazy -s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...
@@ -102,6 +103,9 @@ misspell:
 vet:
 	go vet $(MODULE)/...
 
+licenses:
+	go-licenses csv $(MODULE)/...
+
 COVERAGE_FILE := coverage.out
 test: build cmds
 	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...

build/container/Dockerfile.centos

@@ -17,19 +17,29 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN yum install -y \
+    wget make git gcc \
+     && \
+    rm -rf /var/cache/yum/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -53,7 +63,7 @@ RUN [[ "${BASE_DIST}" != "centos8" ]] || \
     )
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_VISIBLE_DEVICES=void
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
 ARG ARTIFACTS_ROOT
@@ -67,9 +77,9 @@ ARG TARGETARCH
 ENV PACKAGE_ARCH ${TARGETARCH}
 RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
     yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
 
@@ -87,11 +97,4 @@ LABEL description="See summary"
 
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        yum update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
-
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.packaging

@@ -15,15 +15,27 @@
 ARG BASE_DIST
 ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
-ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
-
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
+# build-args are added to the manifest.txt file below.
+ARG BASE_DIST
+ARG PACKAGE_DIST
+ARG PACKAGE_VERSION
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
+ARG SOURCE_DATE_EPOCH
+ARG VERSION
+
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
+
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

build/container/Dockerfile.ubuntu

@@ -17,19 +17,28 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN apt-get update && \
+    apt-get install -y wget make git gcc \
+    && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -53,7 +62,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_VISIBLE_DEVICES=void
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
 ARG ARTIFACTS_ROOT
@@ -66,7 +75,7 @@ ARG PACKAGE_VERSION
 ARG TARGETARCH
 ENV PACKAGE_ARCH ${TARGETARCH}
 
-ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
+ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container/stable"
 ARG LIBNVIDIA_CONTAINER0_VERSION
 RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
         curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
@@ -75,9 +84,9 @@ RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
     fi
 
 RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
 
@@ -95,11 +104,4 @@ LABEL description="See summary"
 
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
-
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -14,6 +14,7 @@
 
 BUILD_MULTI_ARCH_IMAGES ?= false
 DOCKER ?= docker
+REGCTL ?= regctl
 
 BUILDX =
 ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
@@ -44,13 +45,13 @@ OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
 DEFAULT_PUSH_TARGET := ubuntu20.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7
+DISTRIBUTIONS := ubuntu20.04 ubi8 centos7
 
 META_TARGETS := packaging
 
 BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
 PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
 
 .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -74,6 +75,14 @@ endif
 push-%: DIST = $(*)
 push-short: DIST = $(DEFAULT_PUSH_TARGET)
 
+# Define the push targets
+$(PUSH_TARGETS): push-%:
+	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+
+push-short:
+	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+
+
 build-%: DIST = $(*)
 build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
 
@@ -94,7 +103,10 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
-		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
+		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
+		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
+		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
 
@@ -102,24 +114,20 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
 build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
 
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
-build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-ubi8: PACKAGE_DIST = centos7
 
 build-centos7: BASE_DIST = $(*)
 build-centos7: DOCKERFILE_SUFFIX := centos
 build-centos7: PACKAGE_DIST = $(BASE_DIST)
-build-centos7: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
@@ -135,18 +143,9 @@ $(TEST_TARGETS): test-%:
 test-packaging: DIST = packaging
 test-packaging:
 	@echo "Testing package image contents"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/aarch64" || echo "Missing centos7/aarch64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/multi-arch.mk

@@ -16,17 +16,6 @@ PUSH_ON_BUILD ?= false
 DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD)
 DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64
 
-REGCTL ?= regctl
-$(PUSH_TARGETS): push-%:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE)
-
-push-short:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)
-
 # We only have x86_64 packages for centos7
 build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
 

build/container/native-only.mk

@@ -13,11 +13,3 @@
 # limitations under the License.
 
 DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
-
-$(PUSH_TARGETS): push-%:
-	$(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)"
-	$(DOCKER) push "$(OUT_IMAGE)"
-
-push-short:
-	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"
-	$(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"

cmd/nvidia-container-runtime-hook/capabilities.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"log"
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -9,11 +9,11 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
-)
 
-var envSwarmGPU *string
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
 
 const (
 	envCUDAVersion          = "CUDA_VERSION"
@@ -23,6 +23,7 @@ const (
 	envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
 	envNVMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
 	envNVMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
+	envNVImexChannels       = "NVIDIA_IMEX_CHANNELS"
 	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
@@ -38,15 +39,18 @@ type nvidiaConfig struct {
 	Devices            string
 	MigConfigDevices   string
 	MigMonitorDevices  string
+	ImexChannels       string
 	DriverCapabilities string
-	Requirements       []string
-	DisableRequire     bool
+	// Requirements defines the requirements DSL for the container to run.
+	// This is empty if no specific requirements are needed, or if requirements are
+	// explicitly disabled.
+	Requirements []string
 }
 
 type containerConfig struct {
 	Pid    int
 	Rootfs string
-	Env    map[string]string
+	Image  image.CUDA
 	Nvidia *nvidiaConfig
 }
 
@@ -132,7 +136,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -141,67 +145,58 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		for _, c := range caps {
+			if c == capSysAdmin {
+				return true
+			}
 		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
+		return false
 	}
 
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	process := specs.Process{
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
-func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
-	// Build a list of envvars to consider.
-	envVars := []string{envNVVisibleDevices}
-	if envSwarmGPU != nil {
-		// The Swarm envvar has higher precedence.
-		envVars = append([]string{*envSwarmGPU}, envVars...)
-	}
-
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
-	for _, envVar := range envVars {
-		if devs, ok := env[envVar]; ok {
-			devices = &devs
+func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
+	// We check if the image has at least one of the Swarm resource envvars defined and use this
+	// if specified.
+	var hasSwarmEnvvar bool
+	for _, envvar := range swarmResourceEnvvars {
+		if _, exists := image[envvar]; exists {
+			hasSwarmEnvvar = true
 			break
 		}
 	}
 
-	// Environment variable unset with legacy image: default to "all".
-	if devices == nil && legacyImage {
-		all := "all"
-		return &all
+	var devices []string
+	if hasSwarmEnvvar {
+		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
+	} else {
+		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
 	}
 
-	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
+	if len(devices) == 0 {
 		return nil
 	}
 
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		empty := ""
-		return &empty
-	}
+	devicesString := strings.Join(devices, ",")
 
-	// Any other value.
-	return devices
+	return &devicesString
 }
 
 func getDevicesFromMounts(mounts []Mount) *string {
@@ -241,7 +236,7 @@ func getDevicesFromMounts(mounts []Mount) *string {
 	return &ret
 }
 
-func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
+func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
 		devices := getDevicesFromMounts(mounts)
@@ -251,7 +246,7 @@ func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, p
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(env, legacyImage)
+	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
 	if devices == nil {
 		return nil
 	}
@@ -279,10 +274,19 @@ func getMigMonitorDevices(env map[string]string) *string {
 	return nil
 }
 
-func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
+func getImexChannels(env map[string]string) *string {
+	if chans, ok := env[envNVImexChannels]; ok {
+		return &chans
+	}
+	return nil
+}
+
+func (c *HookConfig) getDriverCapabilities(env map[string]string, legacyImage bool) image.DriverCapabilities {
 	// We use the default driver capabilities by default. This is filtered to only include the
 	// supported capabilities
-	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
+	supportedDriverCapabilities := image.NewDriverCapabilities(c.SupportedDriverCapabilities)
+
+	capabilities := supportedDriverCapabilities.Intersection(image.DefaultDriverCapabilities)
 
 	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
 
@@ -293,9 +297,9 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr
 
 	if capsEnvSpecified && len(capsEnv) > 0 {
 		// If the envvironment variable is specified and is non-empty, use the capabilities value
-		envCapabilities := DriverCapabilities(capsEnv)
+		envCapabilities := image.NewDriverCapabilities(capsEnv)
 		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
-		if envCapabilities != all && capabilities != envCapabilities {
+		if !envCapabilities.IsAll() && len(capabilities) != len(envCapabilities) {
 			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
 		}
 	}
@@ -307,7 +311,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	legacyImage := image.IsLegacy()
 
 	var devices string
-	if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil {
+	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
 		devices = *d
 	} else {
 		// 'nil' devices means this is not a GPU container.
@@ -330,22 +334,25 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+	var imexChannels string
+	if c := getImexChannels(image); c != nil {
+		imexChannels = *c
+	}
+
+	driverCapabilities := hookConfig.getDriverCapabilities(image, legacyImage).String()
 
 	requirements, err := image.GetRequirements()
 	if err != nil {
 		log.Panicln("failed to get requirements", err)
 	}
 
-	disableRequire := image.HasDisableRequire()
-
 	return &nvidiaConfig{
 		Devices:            devices,
 		MigConfigDevices:   migConfigDevices,
 		MigMonitorDevices:  migMonitorDevices,
+		ImexChannels:       imexChannels,
 		DriverCapabilities: driverCapabilities,
 		Requirements:       requirements,
-		DisableRequire:     disableRequire,
 	}
 }
 
@@ -363,17 +370,19 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	image, err := image.New(
+		image.WithEnv(s.Process.Env),
+		image.WithDisableRequire(hook.DisableRequire),
+	)
 	if err != nil {
 		log.Panicln(err)
 	}
 
 	privileged := isPrivileged(s)
-	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    image,
+		Image:  image,
 		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"fmt"
 	"path/filepath"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -36,9 +38,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -50,9 +51,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -68,7 +68,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Legacy image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
+				envNVVisibleDevices: "void",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -82,9 +82,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -96,9 +95,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -111,9 +109,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -126,9 +123,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -141,9 +137,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -158,9 +153,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -176,9 +170,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -207,9 +200,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -225,7 +217,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Modern image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
+				envNVVisibleDevices: "void",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -239,9 +231,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -253,9 +244,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -268,9 +258,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -283,9 +272,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -298,9 +286,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -315,9 +302,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -333,9 +319,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -347,9 +332,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -363,9 +347,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigConfigDevices:   "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -389,9 +372,8 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigMonitorDevices:  "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -416,7 +398,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			},
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 			},
 		},
 		{
@@ -431,7 +413,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			},
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: "video,display",
+				DriverCapabilities: "display,video",
 			},
 		},
 		{
@@ -445,7 +427,39 @@ func TestGetNvidiaConfig(t *testing.T) {
 			},
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, swarmResource overrides device selection",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SwarmResource:               "DOCKER_SWARM_RESOURCE",
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "GPU1,GPU2",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, comma separated swarmResource is split and overrides device selection",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SwarmResource:               "NOT_DOCKER_SWARM_RESOURCE,DOCKER_SWARM_RESOURCE",
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "GPU1,GPU2",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
 			},
 		},
 	}
@@ -456,7 +470,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			getConfig := func() {
 				hookConfig := tc.hookConfig
 				if hookConfig == nil {
-					defaultConfig := getDefaultHookConfig()
+					defaultConfig, _ := getDefaultHookConfig()
 					hookConfig = &defaultConfig
 				}
 				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
@@ -485,7 +499,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
 
 			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
-			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
 		})
 	}
 }
@@ -668,10 +681,10 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				env := map[string]string{
 					envNVVisibleDevices: tc.envvarDevices,
 				}
-				hookConfig := getDefaultHookConfig()
+				hookConfig, _ := getDefaultHookConfig()
 				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
+				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)
 			}
 
 			// For all other tests, just grab the devices and check the results
@@ -688,13 +701,13 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
 	gpuID := "GPU-12345"
 	anotherGPUID := "GPU-67890"
+	thirdGPUID := "MIG-12345"
 
 	var tests = []struct {
-		description     string
-		envSwarmGPU     *string
-		env             map[string]string
-		legacyImage     bool
-		expectedDevices *string
+		description          string
+		swarmResourceEnvvars []string
+		env                  map[string]string
+		expectedDevices      *string
 	}{
 		{
 			description: "empty env returns nil for non-legacy image",
@@ -729,13 +742,15 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
 			env: map[string]string{
 				envNVVisibleDevices: gpuID,
+				envCUDAVersion:      "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description:     "empty env returns all for legacy image",
-			legacyImage:     true,
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envCUDAVersion: "legacy",
+			},
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
@@ -781,86 +796,116 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
 			description: "empty env returns all for legacy image",
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
 		// enabled
 		{
-			description: "empty env returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "empty env returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 		},
 		{
-			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "",
 			},
 		},
 		{
-			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "void",
 			},
 		},
 		{
-			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "none",
 			},
 			expectedDevices: &empty,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
 			},
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS is selected if present",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS is selected if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
+		{
+			description:          "All available swarm resource envvars are selected and override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS":            thirdGPUID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: func() *string {
+				result := fmt.Sprintf("%s,%s", thirdGPUID, anotherGPUID)
+				return &result
+			}(),
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
 	}
 
 	for i, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			envSwarmGPU = tc.envSwarmGPU
-			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
+			devices := getDevicesFromEnvvar(image.CUDA(tc.env), tc.swarmResourceEnvvars)
 			if tc.expectedDevices == nil {
 				require.Nil(t, devices, "%d: %v", i, tc)
 				return
@@ -874,7 +919,7 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 
 func TestGetDriverCapabilities(t *testing.T) {
 
-	supportedCapabilities := "compute,utility,display,video"
+	supportedCapabilities := "compute,display,utility,video"
 
 	testCases := []struct {
 		description           string
@@ -909,7 +954,7 @@ func TestGetDriverCapabilities(t *testing.T) {
 			},
 			legacyImage:           true,
 			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
 		},
 		{
 			description:           "Env unset for legacy image is 'all'",
@@ -932,7 +977,7 @@ func TestGetDriverCapabilities(t *testing.T) {
 			env:                   map[string]string{},
 			legacyImage:           false,
 			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
 		},
 		{
 			description: "Env is all for modern image",
@@ -950,7 +995,7 @@ func TestGetDriverCapabilities(t *testing.T) {
 			},
 			legacyImage:           false,
 			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
 		},
 		{
 			description: "Invalid capabilities panic",
@@ -970,11 +1015,14 @@ func TestGetDriverCapabilities(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			var capabilites DriverCapabilities
+			var capabilites string
+
+			c := HookConfig{
+				SupportedDriverCapabilities: tc.supportedCapabilities,
+			}
 
 			getDriverCapabilities := func() {
-				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
-				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
+				capabilites = c.getDriverCapabilities(tc.env, tc.legacyImage).String()
 			}
 
 			if tc.expectedPanic {

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
+
+const (
+	configPath = "/etc/nvidia-container-runtime/config.toml"
+	driverPath = "/run/nvidia/driver"
+)
+
+var defaultPaths = [...]string{}
+
+// HookConfig : options for the nvidia-container-runtime-hook.
+type HookConfig config.Config
+
+func getDefaultHookConfig() (HookConfig, error) {
+	defaultCfg, err := config.GetDefault()
+	if err != nil {
+		return HookConfig{}, err
+	}
+
+	return *(*HookConfig)(defaultCfg), nil
+}
+
+// loadConfig loads the required paths for the hook config.
+func loadConfig() (*config.Config, error) {
+	var configPaths []string
+	var required bool
+	if len(*configflag) != 0 {
+		configPaths = append(configPaths, *configflag)
+		required = true
+	} else {
+		configPaths = append(configPaths, path.Join(driverPath, configPath), configPath)
+	}
+
+	for _, p := range configPaths {
+		cfg, err := config.New(
+			config.WithConfigFile(p),
+			config.WithRequired(true),
+		)
+		if err == nil {
+			return cfg.Config()
+		} else if os.IsNotExist(err) && !required {
+			continue
+		}
+		return nil, fmt.Errorf("couldn't open required configuration file: %v", err)
+	}
+
+	return config.GetDefault()
+}
+
+func getHookConfig() (*HookConfig, error) {
+	cfg, err := loadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config := (*HookConfig)(cfg)
+
+	allSupportedDriverCapabilities := image.SupportedDriverCapabilities
+	if config.SupportedDriverCapabilities == "all" {
+		config.SupportedDriverCapabilities = allSupportedDriverCapabilities.String()
+	}
+	configuredCapabilities := image.NewDriverCapabilities(config.SupportedDriverCapabilities)
+	// We ensure that the configured value is a subset of all supported capabilities
+	if !allSupportedDriverCapabilities.IsSuperset(configuredCapabilities) {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allSupportedDriverCapabilities.String())
+	}
+
+	return config, nil
+}
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c HookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}
+
+// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
+func (c *HookConfig) getSwarmResourceEnvvars() []string {
+	if c.SwarmResource == "" {
+		return nil
+	}
+
+	candidates := strings.Split(c.SwarmResource, ",")
+
+	var envvars []string
+	for _, c := range candidates {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) > 0 {
+			envvars = append(envvars, trimmed)
+		}
+	}
+
+	return envvars
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -28,16 +29,16 @@ func TestGetHookConfig(t *testing.T) {
 	testCases := []struct {
 		lines                      []string
 		expectedPanic              bool
-		expectedDriverCapabilities DriverCapabilities
+		expectedDriverCapabilities string
 	}{
 		{
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"all\"",
 			},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
@@ -47,19 +48,19 @@ func TestGetHookConfig(t *testing.T) {
 		},
 		{
 			lines:                      []string{},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"\"",
 			},
-			expectedDriverCapabilities: none,
+			expectedDriverCapabilities: "",
 		},
 		{
 			lines: []string{
-				"supported-driver-capabilities = \"utility,compute\"",
+				"supported-driver-capabilities = \"compute,utility\"",
 			},
-			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+			expectedDriverCapabilities: "compute,utility",
 		},
 	}
 
@@ -89,7 +90,8 @@ func TestGetHookConfig(t *testing.T) {
 
 			var config HookConfig
 			getHookConfig := func() {
-				config = getHookConfig()
+				c, _ := getHookConfig()
+				config = *c
 			}
 
 			if tc.expectedPanic {
@@ -103,3 +105,50 @@ func TestGetHookConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSwarmResourceEnvvars(t *testing.T) {
+	testCases := []struct {
+		value    string
+		expected []string
+	}{
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value:    " ",
+			expected: nil,
+		},
+		{
+			value:    "single",
+			expected: []string{"single"},
+		},
+		{
+			value:    "single ",
+			expected: []string{"single"},
+		},
+		{
+			value:    "one,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one ,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one, two",
+			expected: []string{"one", "two"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			c := &HookConfig{
+				SwarmResource: tc.value,
+			}
+
+			envvars := c.getSwarmResourceEnvvars()
+			require.EqualValues(t, tc.expected, envvars)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/main.go

@@ -13,7 +13,9 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
@@ -36,16 +38,12 @@ func exit() {
 	os.Exit(0)
 }
 
-func getCLIPath(config CLIConfig) string {
-	if config.Path != nil {
-		return *config.Path
+func getCLIPath(config config.ContainerCLIConfig) string {
+	if config.Path != "" {
+		return config.Path
 	}
 
-	var root string
-	if config.Root != nil {
-		root = *config.Root
-	}
-	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
+	if err := os.Setenv("PATH", lookup.GetPath(config.Root)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -71,25 +69,28 @@ func doPrestart() {
 	defer exit()
 	log.SetFlags(0)
 
-	hook := getHookConfig()
-	cli := hook.NvidiaContainerCLI
-
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+	hook, err := getHookConfig()
+	if err != nil || hook == nil {
+		log.Panicln("error getting hook config:", err)
 	}
+	cli := hook.NVIDIAContainerCLIConfig
 
-	container := getContainerConfig(hook)
+	container := getContainerConfig(*hook)
 	nvidia := container.Nvidia
 	if nvidia == nil {
 		// Not a GPU container, nothing to do.
 		return
 	}
 
+	if !hook.NVIDIAContainerRuntimeHookConfig.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntimeConfig.Mode, container.Image) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
+	}
+
 	rootfs := getRootfsPath(container)
 
 	args := []string{getCLIPath(cli)}
-	if cli.Root != nil {
-		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
+	if cli.Root != "" {
+		args = append(args, fmt.Sprintf("--root=%s", cli.Root))
 	}
 	if cli.LoadKmods {
 		args = append(args, "--load-kmods")
@@ -99,19 +100,19 @@ func doPrestart() {
 	}
 	if *debugflag {
 		args = append(args, "--debug=/dev/stderr")
-	} else if cli.Debug != nil {
-		args = append(args, fmt.Sprintf("--debug=%s", *cli.Debug))
+	} else if cli.Debug != "" {
+		args = append(args, fmt.Sprintf("--debug=%s", cli.Debug))
 	}
-	if cli.Ldcache != nil {
-		args = append(args, fmt.Sprintf("--ldcache=%s", *cli.Ldcache))
+	if cli.Ldcache != "" {
+		args = append(args, fmt.Sprintf("--ldcache=%s", cli.Ldcache))
 	}
-	if cli.User != nil {
-		args = append(args, fmt.Sprintf("--user=%s", *cli.User))
+	if cli.User != "" {
+		args = append(args, fmt.Sprintf("--user=%s", cli.User))
 	}
 	args = append(args, "configure")
 
-	if cli.Ldconfig != nil {
-		args = append(args, fmt.Sprintf("--ldconfig=%s", *cli.Ldconfig))
+	if cli.Ldconfig != "" {
+		args = append(args, fmt.Sprintf("--ldconfig=%s", cli.Ldconfig))
 	}
 	if cli.NoCgroups {
 		args = append(args, "--no-cgroups")
@@ -125,6 +126,9 @@ func doPrestart() {
 	if len(nvidia.MigMonitorDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
 	}
+	if len(nvidia.ImexChannels) > 0 {
+		args = append(args, fmt.Sprintf("--imex-channel=%s", nvidia.ImexChannels))
+	}
 
 	for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
 		if len(cap) == 0 {
@@ -133,10 +137,8 @@ func doPrestart() {
 		args = append(args, capabilityToCLI(cap))
 	}
 
-	if !hook.DisableRequire && !nvidia.DisableRequire {
-		for _, req := range nvidia.Requirements {
-			args = append(args, fmt.Sprintf("--require=%s", req))
-		}
+	for _, req := range nvidia.Requirements {
+		args = append(args, fmt.Sprintf("--require=%s", req))
 	}
 
 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
@@ -185,11 +187,11 @@ func main() {
 	}
 }
 
-// logInterceptor implements the info.Logger interface to allow for logging from this function.
-type logInterceptor struct{}
+// logInterceptor implements the logger.Interface to allow for logging from executable.
+type logInterceptor struct {
+	logger.NullLogger
+}
 
 func (l *logInterceptor) Infof(format string, args ...interface{}) {
 	log.Printf(format, args...)
 }
-
-func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/main.go

@@ -1,84 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer logger.Reset()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -4,15 +4,17 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,7 +43,7 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		log.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
@@ -53,11 +55,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,7 +79,7 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -91,9 +93,10 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -170,7 +173,8 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	logger, _ := testlog.NewNullLogger()
+	m := modifier.NewStableRuntimeModifier(logger, nvidiaHook)
 	return m.Modify(spec)
 }
 

cmd/nvidia-container-runtime/modifier/csv.go

@@ -1,166 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/cuda"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
-)
-
-// csvMode represents the modifications as performed by the csv runtime mode
-type csvMode struct {
-	logger     *logrus.Logger
-	discoverer discover.Discover
-}
-
-const (
-	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
-	visibleDevicesVoid   = "void"
-
-	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
-)
-
-// NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
-// The modifications are defined by CSV MountSpecs.
-func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	rawSpec, err := ociSpec.Load()
-	if err != nil {
-		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
-	// if no modification is required.
-	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
-	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
-		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
-		return nil, nil
-	}
-	logger.Infof("Constructing modifier from config: %+v", *cfg)
-
-	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
-	}
-
-	// TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the
-	// visible devices are checked.
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := checkRequirements(logger, &image); err != nil {
-		return nil, fmt.Errorf("requirements not met: %v", err)
-	}
-
-	csvFiles, err := csv.GetFileList(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.MountSpecPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
-	}
-
-	nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
-	if nvidiaRequireJetpack != "csv-mounts=all" {
-		csvFiles = csv.BaseFilesOnly(csvFiles)
-	}
-
-	csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
-	}
-
-	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
-	}
-
-	createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
-	}
-
-	d := discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
-
-	return newModifierFromDiscoverer(logger, d)
-}
-
-// newModifierFromDiscoverer created a modifier that aplies the discovered
-// modifications to an OCI spec if require by the runtime wrapper.
-func newModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
-	m := csvMode{
-		logger:     logger,
-		discoverer: d,
-	}
-	return &m, nil
-}
-
-// Modify applies the required modifications to the incomming OCI spec. These modifications
-// are applied in-place.
-func (m csvMode) Modify(spec *specs.Spec) error {
-	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
-	if err != nil {
-		return fmt.Errorf("failed to remove existing hooks: %v", err)
-	}
-
-	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
-	if err != nil {
-		return fmt.Errorf("failed to get required container edits: %v", err)
-	}
-
-	return specEdits.Modify(spec)
-}
-
-func checkRequirements(logger *logrus.Logger, image *image.CUDA) error {
-	if image.HasDisableRequire() {
-		// TODO: We could print the real value here instead
-		logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true)
-		return nil
-	}
-
-	imageRequirements, err := image.GetRequirements()
-	if err != nil {
-		//  TODO: Should we treat this as a failure, or just issue a warning?
-		return fmt.Errorf("failed to get image requirements: %v", err)
-	}
-
-	r := requirements.New(logger, imageRequirements)
-
-	cudaVersion, err := cuda.Version()
-	if err != nil {
-		logger.Warnf("Failed to get CUDA version: %v", err)
-	} else {
-		r.AddVersionProperty(requirements.CUDA, cudaVersion)
-	}
-
-	compteCapability, err := cuda.ComputeCapability(0)
-	if err != nil {
-		logger.Warnf("Failed to get CUDA Compute Capability: %v", err)
-	} else {
-		r.AddVersionProperty(requirements.ARCH, compteCapability)
-	}
-
-	return r.Assert()
-}

cmd/nvidia-container-runtime/modifier/csv_test.go

@@ -1,284 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewCSVModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description    string
-		cfg            *config.Config
-		spec           oci.Spec
-		visibleDevices string
-		expectedError  error
-		expectedNil    bool
-	}{
-		{
-			description: "spec load error returns error",
-			spec: &oci.SpecMock{
-				LoadFunc: func() (*specs.Spec, error) {
-					return nil, fmt.Errorf("load failed")
-				},
-			},
-			expectedError: fmt.Errorf("load failed"),
-		},
-		{
-			description:    "visible devices not set returns nil",
-			visibleDevices: "NOT_SET",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices empty returns nil",
-			visibleDevices: "",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices 'void' returns nil",
-			visibleDevices: "void",
-			expectedNil:    true,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			spec := tc.spec
-			if spec == nil {
-				spec = &oci.SpecMock{
-					LookupEnvFunc: func(s string) (string, bool) {
-						if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
-							return tc.visibleDevices, true
-						}
-						return "", false
-					},
-				}
-			}
-
-			m, err := NewCSVModifier(logger, tc.cfg, spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			if tc.expectedNil || tc.expectedError != nil {
-				require.Nil(t, m)
-			} else {
-				require.NotNil(t, m)
-			}
-		})
-	}
-}
-
-func TestExperimentalModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description   string
-		discover      *discover.DiscoverMock
-		spec          *specs.Spec
-		expectedError error
-		expectedSpec  *specs.Spec
-	}{
-		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
-		},
-		{
-			description: "failed hooks discoverer returns error",
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					return nil, fmt.Errorf("discover.Hooks error")
-				},
-			},
-			expectedError: fmt.Errorf("discover.Hooks error"),
-		},
-		{
-			description: "discovered hooks are injected into spec",
-			spec:        &specs.Spec{},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/a",
-							Args:      []string{"/hook/a", "arga"},
-						},
-						{
-							Lifecycle: "createContainer",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-					CreateContainer: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "existing hooks are maintained",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-runtime-hook",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-runtime-hook",
-							Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-toolkit",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-toolkit",
-							Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			m, err := newModifierFromDiscoverer(logger, tc.discover)
-			require.NoError(t, err)
-
-			err = m.Modify(tc.spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			require.EqualValues(t, tc.expectedSpec, tc.spec)
-		})
-	}
-}

cmd/nvidia-container-toolkit/capabilities.go

@@ -1,83 +0,0 @@
-package main
-
-import (
-	"log"
-	"strings"
-)
-
-const (
-	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
-	defaultDriverCapabilities = DriverCapabilities("utility,compute")
-
-	none = DriverCapabilities("")
-	all  = DriverCapabilities("all")
-)
-
-func capabilityToCLI(cap string) string {
-	switch cap {
-	case "compute":
-		return "--compute"
-	case "compat32":
-		return "--compat32"
-	case "graphics":
-		return "--graphics"
-	case "utility":
-		return "--utility"
-	case "video":
-		return "--video"
-	case "display":
-		return "--display"
-	case "ngx":
-		return "--ngx"
-	default:
-		log.Panicln("unknown driver capability:", cap)
-	}
-	return ""
-}
-
-// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
-// variable. Operations include default values, filtering, and handling meta values such as "all"
-type DriverCapabilities string
-
-// Intersection returns intersection between two sets of capabilities.
-func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
-	if capabilities == all {
-		return d
-	}
-	if d == all {
-		return capabilities
-	}
-
-	lookup := make(map[string]bool)
-	for _, c := range d.list() {
-		lookup[c] = true
-	}
-	var found []string
-	for _, c := range capabilities.list() {
-		if lookup[c] {
-			found = append(found, c)
-		}
-	}
-
-	intersection := DriverCapabilities(strings.Join(found, ","))
-	return intersection
-}
-
-// String returns the string representation of the driver capabilities
-func (d DriverCapabilities) String() string {
-	return string(d)
-}
-
-// list returns the driver capabilities as a list
-func (d DriverCapabilities) list() []string {
-	var caps []string
-	for _, c := range strings.Split(string(d), ",") {
-		trimmed := strings.TrimSpace(c)
-		if len(trimmed) == 0 {
-			continue
-		}
-		caps = append(caps, trimmed)
-	}
-
-	return caps
-}

cmd/nvidia-container-toolkit/capabilities_test.go

@@ -1,134 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestDriverCapabilitiesIntersection(t *testing.T) {
-	testCases := []struct {
-		capabilities          DriverCapabilities
-		supportedCapabilities DriverCapabilities
-		expectedIntersection  DriverCapabilities
-	}{
-		{
-			capabilities:          none,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          allDriverCapabilities,
-			supportedCapabilities: all,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: all,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: DriverCapabilities("cap1"),
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("cap0,cap1"),
-			supportedCapabilities: DriverCapabilities("cap1,cap0"),
-			expectedIntersection:  DriverCapabilities("cap0,cap1"),
-		},
-		{
-			capabilities:          defaultDriverCapabilities,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  defaultDriverCapabilities,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-		{
-			capabilities:          DriverCapabilities("cap1"),
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
-			require.EqualValues(t, tc.expectedIntersection, intersection)
-		})
-	}
-}
-
-func TestDriverCapabilitiesList(t *testing.T) {
-	testCases := []struct {
-		capabilities DriverCapabilities
-		expected     []string
-	}{
-		{
-			capabilities: DriverCapabilities(""),
-		},
-		{
-			capabilities: DriverCapabilities("  "),
-		},
-		{
-			capabilities: DriverCapabilities(","),
-		},
-		{
-			capabilities: DriverCapabilities(",cap"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap,"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap0,,cap1"),
-			expected:     []string{"cap0", "cap1"},
-		},
-		{
-			capabilities: DriverCapabilities("cap1,cap0,cap3"),
-			expected:     []string{"cap1", "cap0", "cap3"},
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			require.EqualValues(t, tc.expected, tc.capabilities.list())
-		})
-	}
-}

cmd/nvidia-container-toolkit/hook_config.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"log"
-	"os"
-	"path"
-	"reflect"
-
-	"github.com/BurntSushi/toml"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-)
-
-const (
-	configPath = "/etc/nvidia-container-runtime/config.toml"
-	driverPath = "/run/nvidia/driver"
-)
-
-var defaultPaths = [...]string{
-	path.Join(driverPath, configPath),
-	configPath,
-}
-
-// CLIConfig : options for nvidia-container-cli.
-type CLIConfig struct {
-	Root        *string  `toml:"root"`
-	Path        *string  `toml:"path"`
-	Environment []string `toml:"environment"`
-	Debug       *string  `toml:"debug"`
-	Ldcache     *string  `toml:"ldcache"`
-	LoadKmods   bool     `toml:"load-kmods"`
-	NoPivot     bool     `toml:"no-pivot"`
-	NoCgroups   bool     `toml:"no-cgroups"`
-	User        *string  `toml:"user"`
-	Ldconfig    *string  `toml:"ldconfig"`
-}
-
-// HookConfig : options for the nvidia-container-toolkit.
-type HookConfig struct {
-	DisableRequire                 bool               `toml:"disable-require"`
-	SwarmResource                  *string            `toml:"swarm-resource"`
-	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
-	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
-
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-func getDefaultHookConfig() HookConfig {
-	return HookConfig{
-		DisableRequire:                 false,
-		SwarmResource:                  nil,
-		AcceptEnvvarUnprivileged:       true,
-		AcceptDeviceListAsVolumeMounts: false,
-		SupportedDriverCapabilities:    allDriverCapabilities,
-		NvidiaContainerCLI: CLIConfig{
-			Root:        nil,
-			Path:        nil,
-			Environment: []string{},
-			Debug:       nil,
-			Ldcache:     nil,
-			LoadKmods:   true,
-			NoPivot:     false,
-			NoCgroups:   false,
-			User:        nil,
-			Ldconfig:    nil,
-		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
-	}
-}
-
-func getHookConfig() (config HookConfig) {
-	var err error
-
-	if len(*configflag) > 0 {
-		config = getDefaultHookConfig()
-		_, err = toml.DecodeFile(*configflag, &config)
-		if err != nil {
-			log.Panicln("couldn't open configuration file:", err)
-		}
-	} else {
-		for _, p := range defaultPaths {
-			config = getDefaultHookConfig()
-			_, err = toml.DecodeFile(p, &config)
-			if err == nil {
-				break
-			} else if !os.IsNotExist(err) {
-				log.Panicln("couldn't open default configuration file:", err)
-			}
-		}
-	}
-
-	if config.SupportedDriverCapabilities == all {
-		config.SupportedDriverCapabilities = allDriverCapabilities
-	}
-	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
-	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
-		configName := config.getConfigOption("SupportedDriverCapabilities")
-		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
-	}
-
-	return config
-}
-
-// getConfigOption returns the toml config option associated with the
-// specified struct field.
-func (c HookConfig) getConfigOption(fieldName string) string {
-	t := reflect.TypeOf(c)
-	f, ok := t.FieldByName(fieldName)
-	if !ok {
-		return fieldName
-	}
-	v, ok := f.Tag.Lookup("toml")
-	if !ok {
-		return fieldName
-	}
-	return v
-}

cmd/nvidia-ctk/README.md

@@ -1,3 +1,73 @@
 # NVIDIA Container Toolkit CLI
 
 The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.
+
+## Configure the NVIDIA Container Toolkit
+
+The `config` command of the `nvidia-ctk` CLI allows a user to display and manipulate the NVIDIA Container Toolkit
+configuration.
+
+For example, running the following command:
+```bash
+nvidia-ctk config default
+```
+will display the default config for the detected platform.
+
+Whereas
+```bash
+nvidia-ctk config
+```
+will display the effective NVIDIA Container Toolkit config using the configured config file, and running:
+
+Individual config options can be set by specifying these are key-value pairs to the `--set` argument:
+
+```bash
+nvidia-ctk config --set nvidia-container-cli.no-cgroups=true
+```
+
+By default, all commands output to `STDOUT`, but specifying the `--output` flag writes the config to the specified file.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,54 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/list"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
+		list.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,273 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+)
+
+const (
+	allDeviceName = "all"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	output             string
+	format             string
+	deviceNameStrategy string
+	driverRoot         string
+	nvidiaCTKPath      string
+	mode               string
+	vendor             string
+	class              string
+
+	librarySearchPaths cli.StringSlice
+
+	csv struct {
+		files          cli.StringSlice
+		ignorePatterns cli.StringSlice
+	}
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       spec.FormatYAML,
+			Destination: &opts.format,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Aliases:     []string{"discovery-mode"},
+			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       nvcdi.ModeAuto,
+			Destination: &opts.mode,
+		},
+		&cli.StringFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
+			Value:       nvcdi.DeviceNameStrategyIndex,
+			Destination: &opts.deviceNameStrategy,
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.driverRoot,
+		},
+		&cli.StringSliceFlag{
+			Name:        "library-search-path",
+			Usage:       "Specify the path to search for libraries when discovering the entities that should be included in the CDI specification.\n\tNote: This option only applies to CSV mode.",
+			Destination: &opts.librarySearchPaths,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-ctk-path",
+			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
+			Destination: &opts.nvidiaCTKPath,
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &opts.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &opts.class,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.file",
+			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CSV mode.",
+			Value:       cli.NewStringSlice(csv.DefaultFileList()...),
+			Destination: &opts.csv.files,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.ignore-pattern",
+			Usage:       "Specify a pattern the CSV mount specifications.",
+			Destination: &opts.csv.ignorePatterns,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	opts.format = strings.ToLower(opts.format)
+	switch opts.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", opts.format)
+	}
+
+	opts.mode = strings.ToLower(opts.mode)
+	switch opts.mode {
+	case nvcdi.ModeAuto:
+	case nvcdi.ModeCSV:
+	case nvcdi.ModeNvml:
+	case nvcdi.ModeWsl:
+	case nvcdi.ModeManagement:
+	default:
+		return fmt.Errorf("invalid discovery mode: %v", opts.mode)
+	}
+
+	_, err := nvcdi.NewDeviceNamer(opts.deviceNameStrategy)
+	if err != nil {
+		return err
+	}
+
+	opts.nvidiaCTKPath = config.ResolveNVIDIACTKPath(m.logger, opts.nvidiaCTKPath)
+
+	if outputFileFormat := formatFromFilename(opts.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			opts.format = outputFileFormat
+		} else if outputFileFormat != opts.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", opts.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(opts.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(opts.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := m.generateSpec(opts)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
+
+	if opts.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return spec.Save(opts.output)
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
+	}
+
+	return ""
+}
+
+func (m command) generateSpec(opts *options) (spec.Interface, error) {
+	deviceNamer, err := nvcdi.NewDeviceNamer(opts.deviceNameStrategy)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device namer: %v", err)
+	}
+
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(opts.driverRoot),
+		nvcdi.WithNVIDIACTKPath(opts.nvidiaCTKPath),
+		nvcdi.WithDeviceNamer(deviceNamer),
+		nvcdi.WithMode(string(opts.mode)),
+		nvcdi.WithLibrarySearchPaths(opts.librarySearchPaths.Value()),
+		nvcdi.WithCSVFiles(opts.csv.files.Value()),
+		nvcdi.WithCSVIgnorePatterns(opts.csv.ignorePatterns.Value()),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
+	}
+
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+
+	commonEdits, err := cdilib.GetCommonEdits()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
+	}
+
+	return spec.New(
+		spec.WithVendor(opts.vendor),
+		spec.WithClass(opts.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(opts.format),
+		spec.WithMergedDeviceOptions(
+			transform.WithName(allDeviceName),
+			transform.WithSkipIfExists(true),
+		),
+		spec.WithPermissions(0644),
+	)
+}

cmd/nvidia-ctk/cdi/list/list.go

@@ -0,0 +1,87 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package list
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct{}
+
+// NewCommand constructs a cdi list command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the command
+	c := cli.Command{
+		Name:  "list",
+		Usage: "List the available CDI devices",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	registry, err := cdi.NewCache(
+		cdi.WithAutoRefresh(false),
+		cdi.WithSpecDirs(cdi.DefaultSpecDirs...),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI cache: %v", err)
+	}
+
+	refreshErr := registry.Refresh()
+	devices := registry.ListDevices()
+	m.logger.Infof("Found %d CDI devices", len(devices))
+	if refreshErr != nil {
+		m.logger.Warningf("Refreshing the CDI registry returned the following error(s): %v", refreshErr)
+	}
+	for _, device := range devices {
+		fmt.Printf("%s\n", device)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,160 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+)
+
+type loadSaver interface {
+	Load() (spec.Interface, error)
+	Save(spec.Interface) error
+}
+
+type command struct {
+	logger logger.Interface
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from string
+	to   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transform.NewRootTransformer(
+		opts.from,
+		opts.to,
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/config/config.go

@@ -0,0 +1,213 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	createdefault "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/create-default"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// options stores the subcommand options
+type options struct {
+	flags.Options
+	sets cli.StringSlice
+}
+
+// NewCommand constructs an config command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'config' command
+	c := cli.Command{
+		Name:  "config",
+		Usage: "Interact with the NVIDIA Container Toolkit configuration",
+		Action: func(ctx *cli.Context) error {
+			return run(ctx, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config-file",
+			Aliases:     []string{"config", "c"},
+			Usage:       "Specify the config file to modify.",
+			Value:       config.GetConfigFilePath(),
+			Destination: &opts.Config,
+		},
+		&cli.StringSliceFlag{
+			Name:        "set",
+			Usage:       "Set a config value using the pattern key=value. If value is empty, this is equivalent to specifying the same key in unset. This flag can be specified multiple times",
+			Destination: &opts.sets,
+		},
+		&cli.BoolFlag{
+			Name:        "in-place",
+			Aliases:     []string{"i"},
+			Usage:       "Modify the config file in-place",
+			Destination: &opts.InPlace,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	c.Subcommands = []*cli.Command{
+		createdefault.NewCommand(m.logger),
+	}
+
+	return &c
+}
+
+func run(c *cli.Context, opts *options) error {
+	cfgToml, err := config.New(
+		config.WithConfigFile(opts.Config),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create config: %v", err)
+	}
+
+	for _, set := range opts.sets.Value() {
+		key, value, err := setFlagToKeyValue(set)
+		if err != nil {
+			return fmt.Errorf("invalid --set option %v: %w", set, err)
+		}
+		cfgToml.Set(key, value)
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	if err != nil {
+		return err
+	}
+	cfgToml.Save(output)
+	return nil
+}
+
+var errInvalidConfigOption = errors.New("invalid config option")
+var errUndefinedField = errors.New("undefined field")
+var errInvalidFormat = errors.New("invalid format")
+
+// setFlagToKeyValue converts a --set flag to a key-value pair.
+// The set flag is of the form key[=value], with the value being optional if key refers to a
+// boolean config option.
+func setFlagToKeyValue(setFlag string) (string, interface{}, error) {
+	setParts := strings.SplitN(setFlag, "=", 2)
+	key := setParts[0]
+
+	field, err := getField(key)
+	if err != nil {
+		return key, nil, fmt.Errorf("%w: %w", errInvalidConfigOption, err)
+	}
+
+	kind := field.Kind()
+	if len(setParts) != 2 {
+		if kind == reflect.Bool {
+			return key, true, nil
+		}
+		return key, nil, fmt.Errorf("%w: expected key=value; got %v", errInvalidFormat, setFlag)
+	}
+
+	value := setParts[1]
+	switch kind {
+	case reflect.Bool:
+		b, err := strconv.ParseBool(value)
+		if err != nil {
+			return key, value, fmt.Errorf("%w: %w", errInvalidFormat, err)
+		}
+		return key, b, err
+	case reflect.String:
+		return key, value, nil
+	case reflect.Slice:
+		valueParts := strings.Split(value, ",")
+		switch field.Elem().Kind() {
+		case reflect.String:
+			return key, valueParts, nil
+		case reflect.Int:
+			var output []int64
+			for _, v := range valueParts {
+				vi, err := strconv.ParseInt(v, 10, 0)
+				if err != nil {
+					return key, nil, fmt.Errorf("%w: %w", errInvalidFormat, err)
+				}
+				output = append(output, vi)
+			}
+			return key, output, nil
+		}
+	}
+	return key, nil, fmt.Errorf("unsupported type for %v (%v)", setParts, kind)
+}
+
+func getField(key string) (reflect.Type, error) {
+	s, err := getStruct(reflect.TypeOf(config.Config{}), strings.Split(key, ".")...)
+	if err != nil {
+		return nil, err
+	}
+	return s.Type, err
+}
+
+func getStruct(current reflect.Type, paths ...string) (reflect.StructField, error) {
+	if len(paths) < 1 {
+		return reflect.StructField{}, fmt.Errorf("%w: no fields selected", errUndefinedField)
+	}
+	tomlField := paths[0]
+	for i := 0; i < current.NumField(); i++ {
+		f := current.Field(i)
+		v, ok := f.Tag.Lookup("toml")
+		if !ok {
+			continue
+		}
+		if v != tomlField {
+			continue
+		}
+		if len(paths) == 1 {
+			return f, nil
+		}
+		return getStruct(f.Type, paths[1:]...)
+	}
+	return reflect.StructField{}, fmt.Errorf("%w: %q", errUndefinedField, tomlField)
+}

cmd/nvidia-ctk/config/config_test.go

@@ -0,0 +1,130 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetFlagToKeyValue(t *testing.T) {
+	// TODO: We need to enable this test again since switching to reflect.
+	testCases := []struct {
+		description   string
+		setFlag       string
+		expectedKey   string
+		expectedValue interface{}
+		expectedError error
+	}{
+		{
+			description:   "option not present returns an error",
+			setFlag:       "undefined=new-value",
+			expectedKey:   "undefined",
+			expectedError: errInvalidConfigOption,
+		},
+		{
+			description:   "undefined nexted option returns error",
+			setFlag:       "nvidia-container-cli.undefined",
+			expectedKey:   "nvidia-container-cli.undefined",
+			expectedError: errInvalidConfigOption,
+		},
+		{
+			description:   "boolean option assumes true",
+			setFlag:       "disable-require",
+			expectedKey:   "disable-require",
+			expectedValue: true,
+		},
+		{
+			description:   "boolean option returns true",
+			setFlag:       "disable-require=true",
+			expectedKey:   "disable-require",
+			expectedValue: true,
+		},
+		{
+			description:   "boolean option returns false",
+			setFlag:       "disable-require=false",
+			expectedKey:   "disable-require",
+			expectedValue: false,
+		},
+		{
+			description:   "invalid boolean option returns error",
+			setFlag:       "disable-require=something",
+			expectedKey:   "disable-require",
+			expectedValue: "something",
+			expectedError: errInvalidFormat,
+		},
+		{
+			description:   "string option requires value",
+			setFlag:       "swarm-resource",
+			expectedKey:   "swarm-resource",
+			expectedValue: nil,
+			expectedError: errInvalidFormat,
+		},
+		{
+			description:   "string option returns value",
+			setFlag:       "swarm-resource=string-value",
+			expectedKey:   "swarm-resource",
+			expectedValue: "string-value",
+		},
+		{
+			description:   "string option returns value with equals",
+			setFlag:       "swarm-resource=string-value=more",
+			expectedKey:   "swarm-resource",
+			expectedValue: "string-value=more",
+		},
+		{
+			description:   "string option treats bool value as string",
+			setFlag:       "swarm-resource=true",
+			expectedKey:   "swarm-resource",
+			expectedValue: "true",
+		},
+		{
+			description:   "string option treats int value as string",
+			setFlag:       "swarm-resource=5",
+			expectedKey:   "swarm-resource",
+			expectedValue: "5",
+		},
+		{
+			description:   "[]string option returns single value",
+			setFlag:       "nvidia-container-cli.environment=string-value",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"string-value"},
+		},
+		{
+			description:   "[]string option returns multiple values",
+			setFlag:       "nvidia-container-cli.environment=first,second",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"first", "second"},
+		},
+		{
+			description:   "[]string option returns values with equals",
+			setFlag:       "nvidia-container-cli.environment=first=1,second=2",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"first=1", "second=2"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			k, v, err := setFlagToKeyValue(tc.setFlag)
+			require.ErrorIs(t, err, tc.expectedError)
+			require.EqualValues(t, tc.expectedKey, k)
+			require.EqualValues(t, tc.expectedValue, v)
+		})
+	}
+}

cmd/nvidia-ctk/config/create-default/create-default.go

@@ -0,0 +1,94 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package defaultsubcommand
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a default command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := flags.Options{}
+
+	// Create the 'default' command
+	c := cli.Command{
+		Name:    "default",
+		Aliases: []string{"create-default", "generate-default"},
+		Usage:   "Generate the default NVIDIA Container Toolkit configuration file",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *flags.Options) error {
+	return opts.Validate()
+}
+
+func (m command) run(c *cli.Context, opts *flags.Options) error {
+	cfgToml, err := config.New()
+	if err != nil {
+		return fmt.Errorf("unable to load or create config: %v", err)
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	_, err = cfgToml.Save(output)
+	if err != nil {
+		return fmt.Errorf("failed to write output: %v", err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/config/flags/options.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package flags
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Options stores options for the config commands
+type Options struct {
+	Config  string
+	Output  string
+	InPlace bool
+}
+
+// Validate checks whether the options are valid.
+func (o Options) Validate() error {
+	if o.InPlace && o.Output != "" {
+		return fmt.Errorf("cannot specify both --in-place and --output")
+	}
+
+	return nil
+}
+
+// GetOutput returns the effective output
+func (o Options) GetOutput() string {
+	if o.InPlace {
+		return o.Config
+	}
+
+	return o.Output
+}
+
+// EnsureOutputFolder creates the output folder if it does not exist.
+// If the output folder is not specified (i.e. output to STDOUT), it is ignored.
+func (o Options) EnsureOutputFolder() error {
+	output := o.GetOutput()
+	if output == "" {
+		return nil
+	}
+	if dir := filepath.Dir(output); dir != "" {
+		return os.MkdirAll(dir, 0755)
+	}
+	return nil
+}
+
+// CreateOutput creates the writer for the output.
+func (o Options) CreateOutput() (io.WriteCloser, error) {
+	output := o.GetOutput()
+	if output == "" {
+		return nullCloser{os.Stdout}, nil
+	}
+
+	return os.Create(output)
+}
+
+// nullCloser is a writer that does nothing on Close.
+type nullCloser struct {
+	io.Writer
+}
+
+// Close is a no-op for a nullCloser.
+func (d nullCloser) Close() error {
+	return nil
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -0,0 +1,146 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	paths         cli.StringSlice
+	mode          string
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specifiy a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.mode) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value())
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	locator := lookup.NewExecutableLocator(m.logger, "")
+	targets, err := locator.Locate("chmod")
+	if err != nil {
+		return fmt.Errorf("failed to locate chmod: %v", err)
+	}
+	chmodPath := targets[0]
+
+	args := append([]string{filepath.Base(chmodPath), cfg.mode}, paths...)
+
+	return syscall.Exec(chmodPath, args, nil)
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		path := filepath.Join(root, f)
+		if _, err := os.Stat(path); err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -22,15 +22,16 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -41,7 +42,7 @@ type config struct {
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -74,7 +75,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringSliceFlag{
 			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as source:target",
+			Usage:       "Specify a specific link to create. The link is specified as target::link",
 			Destination: &cfg.links,
 		},
 		&cli.StringFlag{
@@ -100,7 +101,10 @@ func (m command) run(c *cli.Context, cfg *config) error {
 
 	csvFiles := cfg.filenames.Value()
 
-	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
+	chainLocator := lookup.NewSymlinkChainLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(cfg.hostRoot),
+	)
 
 	var candidates []string
 	for _, file := range csvFiles {
@@ -116,7 +120,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 			}
 			targets, err := chainLocator.Locate(ms.Path)
 			if err != nil {
-				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
+				m.logger.Warningf("Failed to locate symlink %v", ms.Path)
 			}
 			candidates = append(candidates, targets...)
 		}
@@ -125,35 +129,32 @@ func (m command) run(c *cli.Context, cfg *config) error {
 	created := make(map[string]bool)
 	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
 	for _, candidate := range candidates {
-		targets, err := m.Locate(candidate)
+		target, err := symlinks.Resolve(candidate)
 		if err != nil {
 			m.logger.Debugf("Skipping invalid link: %v", err)
 			continue
-		} else if len(targets) != 1 {
-			m.logger.Debugf("Unexepected number of targets: %v", targets)
-			continue
-		} else if targets[0] == candidate {
+		} else if target == candidate {
 			m.logger.Debugf("%v is not a symlink", candidate)
 			continue
 		}
 
-		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
+		err = m.createLink(created, cfg.hostRoot, containerRoot, target, candidate)
 		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
+			m.logger.Warningf("Failed to create link %v: %v", []string{target, candidate}, err)
 		}
 	}
 
 	links := cfg.links.Value()
 	for _, l := range links {
-		parts := strings.Split(l, ":")
+		parts := strings.Split(l, "::")
 		if len(parts) != 2 {
-			m.logger.Warnf("Invalid link specification %v", l)
+			m.logger.Warningf("Invalid link specification %v", l)
 			continue
 		}
 
 		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
 		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", parts, err)
+			m.logger.Warningf("Failed to create link %v: %v", parts, err)
 		}
 	}
 
@@ -164,7 +165,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
 	linkPath, err := changeRoot(hostRoot, containerRoot, link)
 	if err != nil {
-		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
+		m.logger.Warningf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
 	}
 	if created[linkPath] {
 		m.logger.Debugf("Link %v already created", linkPath)
@@ -173,7 +174,7 @@ func (m command) createLink(created map[string]bool, hostRoot string, containerR
 
 	targetPath, err := changeRoot(hostRoot, "/", target)
 	if err != nil {
-		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
+		m.logger.Warningf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
 	}
 
 	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)

cmd/nvidia-ctk/hook/hook.go

@@ -17,18 +17,20 @@
 package hook
 
 import (
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type hookCommand struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := hookCommand{
 		logger: logger,
 	}
@@ -46,6 +48,7 @@ func (m hookCommand) build() *cli.Command {
 	hook.Subcommands = []*cli.Command{
 		ldcache.NewCommand(m.logger),
 		symlinks.NewCommand(m.logger),
+		chmod.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -22,13 +22,13 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -37,7 +37,7 @@ type config struct {
 }
 
 // NewCommand constructs an update-ldcache command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -84,6 +84,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
+	_, err = os.Stat(filepath.Join(containerRoot, "/etc/ld.so.cache"))
+	if err != nil && os.IsNotExist(err) {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		return nil
+	}
+
 	err = m.createConfig(containerRoot, cfg.folders.Value())
 	if err != nil {
 		return fmt.Errorf("failed to update ld.so.conf: %v", err)
@@ -105,6 +111,10 @@ func (m command) createConfig(root string, folders []string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
 	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
 	if err != nil {
 		return fmt.Errorf("failed to create config file: %v", err)
@@ -125,5 +135,10 @@ func (m command) createConfig(root string, folders []string) error {
 		configured[folder] = true
 	}
 
+	// The created file needs to be world readable for the cases where the container is run as a non-root user.
+	if err := os.Chmod(configFile.Name(), 0644); err != nil {
+		return fmt.Errorf("failed to chmod config file: %v", err)
+	}
+
 	return nil
 }

cmd/nvidia-ctk/info/info.go

@@ -0,0 +1,47 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'info' command
+	info := cli.Command{
+		Name:  "info",
+		Usage: "Provide information about the system",
+	}
+
+	info.Subcommands = []*cli.Command{}
+
+	return &info
+}

cmd/nvidia-ctk/main.go

@@ -19,24 +19,32 @@ package main
 import (
 	"os"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+
 	cli "github.com/urfave/cli/v2"
 )
 
-var logger = log.New()
-
-// config defines the options that can be set for the CLI through config files,
+// options defines the options that can be set for the CLI through config files,
 // environment variables, or command line flags
-type config struct {
+type options struct {
 	// Debug indicates whether the CLI is started in "debug" mode
 	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
 }
 
 func main() {
-	// Create a config struct to hold the parsed environment variables or command line flags
-	config := config{}
+	logger := logrus.New()
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
 
 	// Create the top-level CLI
 	c := cli.NewApp()
@@ -52,16 +60,25 @@ func main() {
 			Name:        "debug",
 			Aliases:     []string{"d"},
 			Usage:       "Enable debug-level logging",
-			Destination: &config.Debug,
+			Destination: &opts.Debug,
 			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CTK_QUIET"},
+		},
 	}
 
 	// Set log-level for all subcommands
 	c.Before = func(c *cli.Context) error {
-		logLevel := log.InfoLevel
-		if config.Debug {
-			logLevel = log.DebugLevel
+		logLevel := logrus.InfoLevel
+		if opts.Debug {
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
 		}
 		logger.SetLevel(logLevel)
 		return nil
@@ -70,12 +87,17 @@ func main() {
 	// Define the subcommands
 	c.Commands = []*cli.Command{
 		hook.NewCommand(logger),
+		runtime.NewCommand(logger),
+		infoCLI.NewCommand(logger),
+		cdi.NewCommand(logger),
+		system.NewCommand(logger),
+		config.NewCommand(logger),
 	}
 
 	// Run the CLI
 	err := c.Run(os.Args)
 	if err != nil {
-		log.Errorf("%v", err)
-		log.Exit(1)
+		logger.Errorf("%v", err)
+		os.Exit(1)
 	}
 }

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -0,0 +1,317 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package configure
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
+)
+
+const (
+	defaultRuntime = "docker"
+
+	// defaultNVIDIARuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	defaultNVIDIARuntimeName = "nvidia"
+	// defaultNVIDIARuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	defaultNVIDIARuntimeExecutable          = "nvidia-container-runtime"
+	defaultNVIDIARuntimeExpecutablePath     = "/usr/bin/nvidia-container-runtime"
+	defaultNVIDIARuntimeHookExpecutablePath = "/usr/bin/nvidia-container-runtime-hook"
+
+	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
+	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
+	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an configure command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line config
+type config struct {
+	dryRun         bool
+	runtime        string
+	configFilePath string
+	mode           string
+	hookFilePath   string
+
+	nvidiaRuntime struct {
+		name         string
+		path         string
+		hookPath     string
+		setAsDefault bool
+	}
+
+	// cdi-specific options
+	cdi struct {
+		enabled bool
+	}
+}
+
+func (m command) build() *cli.Command {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the 'configure' command
+	configure := cli.Command{
+		Name:  "configure",
+		Usage: "Add a runtime to the specified container engine",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &config)
+		},
+		Action: func(c *cli.Context) error {
+			return m.configureWrapper(c, &config)
+		},
+	}
+
+	configure.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "update the runtime configuration as required but don't write changes to disk",
+			Destination: &config.dryRun,
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Usage:       "the target runtime engine; one of [containerd, crio, docker]",
+			Value:       defaultRuntime,
+			Destination: &config.runtime,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to the config file for the target runtime",
+			Destination: &config.configFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the config mode for runtimes that support multiple configuration mechanisms",
+			Destination: &config.mode,
+		},
+		&cli.StringFlag{
+			Name:        "oci-hook-path",
+			Usage:       "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
+			Destination: &config.hookFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-name",
+			Usage:       "specify the name of the NVIDIA runtime that will be added",
+			Value:       defaultNVIDIARuntimeName,
+			Destination: &config.nvidiaRuntime.name,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-path",
+			Aliases:     []string{"runtime-path"},
+			Usage:       "specify the path to the NVIDIA runtime executable",
+			Value:       defaultNVIDIARuntimeExecutable,
+			Destination: &config.nvidiaRuntime.path,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-hook-path",
+			Usage:       "specify the path to the NVIDIA Container Runtime hook executable",
+			Value:       defaultNVIDIARuntimeHookExpecutablePath,
+			Destination: &config.nvidiaRuntime.hookPath,
+		},
+		&cli.BoolFlag{
+			Name:        "nvidia-set-as-default",
+			Aliases:     []string{"set-as-default"},
+			Usage:       "set the NVIDIA runtime as the default runtime",
+			Destination: &config.nvidiaRuntime.setAsDefault,
+		},
+		&cli.BoolFlag{
+			Name:        "cdi.enabled",
+			Aliases:     []string{"cdi.enable"},
+			Usage:       "Enable CDI in the configured runtime",
+			Destination: &config.cdi.enabled,
+		},
+	}
+
+	return &configure
+}
+
+func (m command) validateFlags(c *cli.Context, config *config) error {
+	if config.mode == "oci-hook" {
+		if !filepath.IsAbs(config.nvidiaRuntime.hookPath) {
+			return fmt.Errorf("the NVIDIA runtime hook path %q is not an absolute path", config.nvidiaRuntime.hookPath)
+		}
+		return nil
+	}
+	if config.mode != "" && config.mode != "config-file" {
+		m.logger.Warningf("Ignoring unsupported config mode for %v: %q", config.runtime, config.mode)
+	}
+	config.mode = "config-file"
+
+	switch config.runtime {
+	case "containerd", "crio", "docker":
+		break
+	default:
+		return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+
+	switch config.runtime {
+	case "containerd", "crio":
+		if config.nvidiaRuntime.path == defaultNVIDIARuntimeExecutable {
+			config.nvidiaRuntime.path = defaultNVIDIARuntimeExpecutablePath
+		}
+		if !filepath.IsAbs(config.nvidiaRuntime.path) {
+			return fmt.Errorf("the NVIDIA runtime path %q is not an absolute path", config.nvidiaRuntime.path)
+		}
+	}
+
+	if config.runtime != "containerd" && config.runtime != "docker" {
+		if config.cdi.enabled {
+			m.logger.Warningf("Ignoring cdi.enabled flag for %v", config.runtime)
+		}
+		config.cdi.enabled = false
+	}
+
+	return nil
+}
+
+// configureWrapper updates the specified container engine config to enable the NVIDIA runtime
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.mode {
+	case "oci-hook":
+		return m.configureOCIHook(c, config)
+	case "config-file":
+		return m.configureConfigFile(c, config)
+	}
+	return fmt.Errorf("unsupported config-mode: %v", config.mode)
+}
+
+// configureConfigFile updates the specified container engine config file to enable the NVIDIA runtime.
+func (m command) configureConfigFile(c *cli.Context, config *config) error {
+	configFilePath := config.resolveConfigFilePath()
+
+	var cfg engine.Interface
+	var err error
+	switch config.runtime {
+	case "containerd":
+		cfg, err = containerd.New(
+			containerd.WithLogger(m.logger),
+			containerd.WithPath(configFilePath),
+		)
+	case "crio":
+		cfg, err = crio.New(
+			crio.WithLogger(m.logger),
+			crio.WithPath(configFilePath),
+		)
+	case "docker":
+		cfg, err = docker.New(
+			docker.WithLogger(m.logger),
+			docker.WithPath(configFilePath),
+		)
+	default:
+		err = fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+	if err != nil || cfg == nil {
+		return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
+	}
+
+	err = cfg.AddRuntime(
+		config.nvidiaRuntime.name,
+		config.nvidiaRuntime.path,
+		config.nvidiaRuntime.setAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = enableCDI(config, cfg)
+	if err != nil {
+		return fmt.Errorf("failed to enable CDI in %s: %w", config.runtime, err)
+	}
+
+	outputPath := config.getOuputConfigPath()
+	n, err := cfg.Save(outputPath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	if outputPath != "" {
+		if n == 0 {
+			m.logger.Infof("Removed empty config from %v", outputPath)
+		} else {
+			m.logger.Infof("Wrote updated config to %v", outputPath)
+		}
+		m.logger.Infof("It is recommended that %v daemon be restarted.", config.runtime)
+	}
+
+	return nil
+}
+
+// resolveConfigFilePath returns the default config file path for the configured container engine
+func (c *config) resolveConfigFilePath() string {
+	if c.configFilePath != "" {
+		return c.configFilePath
+	}
+	switch c.runtime {
+	case "containerd":
+		return defaultContainerdConfigFilePath
+	case "crio":
+		return defaultCrioConfigFilePath
+	case "docker":
+		return defaultDockerConfigFilePath
+	}
+	return ""
+}
+
+// getOuputConfigPath returns the configured config path or "" if dry-run is enabled
+func (c *config) getOuputConfigPath() string {
+	if c.dryRun {
+		return ""
+	}
+	return c.resolveConfigFilePath()
+}
+
+// configureOCIHook creates and configures the OCI hook for the NVIDIA runtime
+func (m *command) configureOCIHook(c *cli.Context, config *config) error {
+	err := ocihook.CreateHook(config.hookFilePath, config.nvidiaRuntime.hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating OCI hook: %v", err)
+	}
+	return nil
+}
+
+// enableCDI enables the use of CDI in the corresponding container engine
+func enableCDI(config *config, cfg engine.Interface) error {
+	if !config.cdi.enabled {
+		return nil
+	}
+	switch config.runtime {
+	case "containerd":
+		return cfg.Set("enable_cdi", true)
+	case "docker":
+		return cfg.Set("features", map[string]bool{"cdi": true})
+	}
+	return fmt.Errorf("enabling CDI in %s is not supported", config.runtime)
+}

cmd/nvidia-ctk/runtime/runtime.go

@@ -0,0 +1,49 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type runtimeCommand struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := runtimeCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m runtimeCommand) build() *cli.Command {
+	// Create the 'runtime' command
+	runtime := cli.Command{
+		Name:  "runtime",
+		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	runtime.Subcommands = []*cli.Command{
+		configure.NewCommand(m.logger),
+	}
+
+	return &runtime
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -0,0 +1,187 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
+)
+
+type allPossible struct {
+	logger       logger.Interface
+	devRoot      string
+	deviceMajors devices.Devices
+	migCaps      nvcaps.MigCaps
+}
+
+// newAllPossible returns a new allPossible device node lister.
+// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func newAllPossible(logger logger.Interface, devRoot string) (nodeLister, error) {
+	deviceMajors, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed reading device majors: %v", err)
+	}
+
+	var requiredMajors []devices.Name
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
+	}
+	if migCaps == nil {
+		migCaps = make(nvcaps.MigCaps)
+	} else {
+		requiredMajors = append(requiredMajors, devices.NVIDIACaps)
+	}
+
+	requiredMajors = append(requiredMajors, devices.NVIDIAGPU, devices.NVIDIAUVM)
+	for _, name := range requiredMajors {
+		if !deviceMajors.Exists(name) {
+			return nil, fmt.Errorf("missing required device major %s", name)
+		}
+	}
+
+	l := allPossible{
+		logger:       logger,
+		devRoot:      devRoot,
+		deviceMajors: deviceMajors,
+		migCaps:      migCaps,
+	}
+
+	return l, nil
+}
+
+// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func (m allPossible) DeviceNodes() ([]deviceNode, error) {
+	gpus, err := nvpci.New(
+		nvpci.WithPCIDevicesRoot(filepath.Join(m.devRoot, nvpci.PCIDevicesRoot)),
+		nvpci.WithLogger(m.logger),
+	).GetGPUs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GPU information: %v", err)
+	}
+
+	count := len(gpus)
+	if count == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
+		return nil, nil
+	}
+
+	deviceNodes, err := m.getControlDeviceNodes()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
+	}
+
+	for gpu := 0; gpu < count; gpu++ {
+		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
+		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
+	}
+
+	return deviceNodes, nil
+}
+
+// getControlDeviceNodes generates a list of control devices
+func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
+	var deviceNodes []deviceNode
+
+	// Define the control devices for standard GPUs.
+	controlDevices := []deviceNode{
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
+	}
+
+	deviceNodes = append(deviceNodes, controlDevices...)
+
+	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
+		migControlMinor, exist := m.migCaps[migControlDevice]
+		if !exist {
+			continue
+		}
+
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			migControlMinor.DevicePath(),
+			int(migControlMinor),
+		)
+
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes, nil
+}
+
+// getGPUDeviceNodes generates a list of device nodes for a given GPU.
+func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
+	d := m.newDeviceNode(
+		devices.NVIDIAGPU,
+		fmt.Sprintf("/dev/nvidia%d", gpu),
+		gpu,
+	)
+
+	return []deviceNode{d}
+}
+
+// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
+func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
+	var selectedCapMinors []nvcaps.MigMinor
+	for gi := 0; ; gi++ {
+		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+		giMinor, exist := m.migCaps[giCap]
+		if !exist {
+			break
+		}
+		selectedCapMinors = append(selectedCapMinors, giMinor)
+		for ci := 0; ; ci++ {
+			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+			ciMinor, exist := m.migCaps[ciCap]
+			if !exist {
+				break
+			}
+			selectedCapMinors = append(selectedCapMinors, ciMinor)
+		}
+	}
+
+	var deviceNodes []deviceNode
+	for _, capMinor := range selectedCapMinors {
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			capMinor.DevicePath(),
+			int(capMinor),
+		)
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes
+}
+
+// newDeviceNode creates a new device node with the specified path and major/minor numbers.
+// The path is adjusted for the specified driver root.
+func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
+	major, _ := m.deviceMajors.Get(deviceName)
+
+	return deviceNode{
+		path:  filepath.Join(m.devRoot, path),
+		major: uint32(major),
+		minor: uint32(minor),
+	}
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -0,0 +1,424 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+	"github.com/fsnotify/fsnotify"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultDevCharPath = "/dev/char"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	devCharPath       string
+	driverRoot        string
+	dryRun            bool
+	watch             bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'create-dev-char-symlinks' command
+	c := cli.Command{
+		Name:  "create-dev-char-symlinks",
+		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "dev-char-path",
+			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
+			Value:       defaultDevCharPath,
+			Destination: &cfg.devCharPath,
+			EnvVars:     []string{"DEV_CHAR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
+			Value:       "/",
+			Destination: &cfg.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "watch",
+			Usage:       "If set, the command will watch for changes to the driver root and recreate the symlinks when changes are detected.",
+			Value:       false,
+			Destination: &cfg.watch,
+			EnvVars:     []string{"WATCH"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-all",
+			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
+			Destination: &cfg.createAll,
+			EnvVars:     []string{"CREATE_ALL"},
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "Load the NVIDIA kernel modules before creating symlinks. This is only applicable when --create-all is set.",
+			Destination: &cfg.loadKernelModules,
+			EnvVars:     []string{"LOAD_KERNEL_MODULES"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-device-nodes",
+			Usage:       "Create the NVIDIA control device nodes in the driver root if they do not exist. This is only applicable when --create-all is set",
+			Destination: &cfg.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "If set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &cfg.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	if cfg.createAll && cfg.watch {
+		return fmt.Errorf("create-all and watch are mutually exclusive")
+	}
+
+	if cfg.loadKernelModules && !cfg.createAll {
+		m.logger.Warning("load-kernel-modules is only applicable when create-all is set; ignoring")
+		cfg.loadKernelModules = false
+	}
+
+	if cfg.createDeviceNodes && !cfg.createAll {
+		m.logger.Warning("create-device-nodes is only applicable when create-all is set; ignoring")
+		cfg.createDeviceNodes = false
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	var watcher *fsnotify.Watcher
+	var sigs chan os.Signal
+
+	if cfg.watch {
+		watcher, err := newFSWatcher(filepath.Join(cfg.driverRoot, "dev"))
+		if err != nil {
+			return fmt.Errorf("failed to create FS watcher: %v", err)
+		}
+		defer watcher.Close()
+
+		sigs = newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	}
+
+	l, err := NewSymlinkCreator(
+		WithLogger(m.logger),
+		WithDevCharPath(cfg.devCharPath),
+		WithDriverRoot(cfg.driverRoot),
+		WithDryRun(cfg.dryRun),
+		WithCreateAll(cfg.createAll),
+		WithLoadKernelModules(cfg.loadKernelModules),
+		WithCreateDeviceNodes(cfg.createDeviceNodes),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink creator: %v", err)
+	}
+
+create:
+	err = l.CreateLinks()
+	if err != nil {
+		return fmt.Errorf("failed to create links: %v", err)
+	}
+	if !cfg.watch {
+		return nil
+	}
+	for {
+		select {
+
+		case event := <-watcher.Events:
+			deviceNode := filepath.Base(event.Name)
+			if !strings.HasPrefix(deviceNode, "nvidia") {
+				continue
+			}
+			if event.Op&fsnotify.Create == fsnotify.Create {
+				m.logger.Infof("%s created, restarting.", event.Name)
+				goto create
+			}
+			if event.Op&fsnotify.Create == fsnotify.Remove {
+				m.logger.Infof("%s removed. Ignoring", event.Name)
+
+			}
+
+		// Watch for any other fs errors and log them.
+		case err := <-watcher.Errors:
+			m.logger.Errorf("inotify: %s", err)
+
+		// React to signals
+		case s := <-sigs:
+			switch s {
+			case syscall.SIGHUP:
+				m.logger.Infof("Received SIGHUP, recreating symlinks.")
+				goto create
+			default:
+				m.logger.Infof("Received signal %q, shutting down.", s)
+				return nil
+			}
+		}
+	}
+}
+
+type linkCreator struct {
+	logger            logger.Interface
+	lister            nodeLister
+	driverRoot        string
+	devRoot           string
+	devCharPath       string
+	dryRun            bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
+}
+
+// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
+type Creator interface {
+	CreateLinks() error
+}
+
+// Option is a functional option for configuring the linkCreator.
+type Option func(*linkCreator)
+
+// NewSymlinkCreator creates a new linkCreator.
+func NewSymlinkCreator(opts ...Option) (Creator, error) {
+	c := linkCreator{}
+	for _, opt := range opts {
+		opt(&c)
+	}
+	if c.logger == nil {
+		c.logger = logger.New()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+	if c.devRoot == "" {
+		c.devRoot = "/"
+	}
+	if c.devCharPath == "" {
+		c.devCharPath = defaultDevCharPath
+	}
+
+	if err := c.setup(); err != nil {
+		return nil, err
+	}
+
+	if c.createAll {
+		lister, err := newAllPossible(c.logger, c.devRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
+		}
+		c.lister = lister
+	} else {
+		c.lister = existing{c.logger, c.devRoot}
+	}
+	return c, nil
+}
+
+func (m linkCreator) setup() error {
+	if !m.loadKernelModules && !m.createDeviceNodes {
+		return nil
+	}
+
+	if m.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(m.dryRun),
+			nvmodules.WithRoot(m.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if m.createDeviceNodes {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(m.dryRun),
+			nvdevices.WithDevRoot(m.devRoot),
+		)
+		if err != nil {
+			return err
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA device nodes: %v", err)
+		}
+	}
+	return nil
+}
+
+// WithDriverRoot sets the driver root path.
+// This is the path in which kernel modules must be loaded.
+func WithDriverRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.driverRoot = root
+	}
+}
+
+// WithDevRoot sets the root path for the /dev directory.
+func WithDevRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.devRoot = root
+	}
+}
+
+// WithDevCharPath sets the path at which the symlinks will be created.
+func WithDevCharPath(path string) Option {
+	return func(c *linkCreator) {
+		c.devCharPath = path
+	}
+}
+
+// WithDryRun sets the dry run flag.
+func WithDryRun(dryRun bool) Option {
+	return func(c *linkCreator) {
+		c.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger.
+func WithLogger(logger logger.Interface) Option {
+	return func(c *linkCreator) {
+		c.logger = logger
+	}
+}
+
+// WithCreateAll sets the createAll flag for the linkCreator.
+func WithCreateAll(createAll bool) Option {
+	return func(lc *linkCreator) {
+		lc.createAll = createAll
+	}
+}
+
+// WithLoadKernelModules sets the loadKernelModules flag for the linkCreator.
+func WithLoadKernelModules(loadKernelModules bool) Option {
+	return func(lc *linkCreator) {
+		lc.loadKernelModules = loadKernelModules
+	}
+}
+
+// WithCreateDeviceNodes sets the createDeviceNodes flag for the linkCreator.
+func WithCreateDeviceNodes(createDeviceNodes bool) Option {
+	return func(lc *linkCreator) {
+		lc.createDeviceNodes = createDeviceNodes
+	}
+}
+
+// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
+func (m linkCreator) CreateLinks() error {
+	deviceNodes, err := m.lister.DeviceNodes()
+	if err != nil {
+		return fmt.Errorf("failed to get device nodes: %v", err)
+	}
+
+	if len(deviceNodes) != 0 && !m.dryRun {
+		err := os.MkdirAll(m.devCharPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
+		}
+	}
+
+	for _, deviceNode := range deviceNodes {
+		target := deviceNode.path
+		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
+
+		m.logger.Infof("Creating link %s => %s", linkPath, target)
+		if m.dryRun {
+			continue
+		}
+
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warningf("Could not create symlink: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type deviceNode struct {
+	path  string
+	major uint32
+	minor uint32
+}
+
+func (d deviceNode) devCharName() string {
+	return fmt.Sprintf("%d:%d", d.major, d.minor)
+}
+
+func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, f := range files {
+		err = watcher.Add(f)
+		if err != nil {
+			watcher.Close()
+			return nil, err
+		}
+	}
+
+	return watcher, nil
+}
+
+func newOSWatcher(sigs ...os.Signal) chan os.Signal {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, sigs...)
+
+	return sigChan
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -0,0 +1,95 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"golang.org/x/sys/unix"
+)
+
+type nodeLister interface {
+	DeviceNodes() ([]deviceNode, error)
+}
+
+type existing struct {
+	logger  logger.Interface
+	devRoot string
+}
+
+// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
+// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
+func (m existing) DeviceNodes() ([]deviceNode, error) {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(m.devRoot),
+		lookup.WithOptional(true),
+	)
+
+	devices, err := locator.Locate("/dev/nvidia*")
+	if err != nil {
+		m.logger.Warningf("Error while locating device: %v", err)
+	}
+
+	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
+	if err != nil {
+		m.logger.Warningf("Error while locating caps device: %v", err)
+	}
+
+	if len(devices) == 0 && len(capDevices) == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
+		return nil, nil
+	}
+
+	var deviceNodes []deviceNode
+	for _, d := range append(devices, capDevices...) {
+		if m.nodeIsBlocked(d) {
+			continue
+		}
+
+		var stat unix.Stat_t
+		err := unix.Stat(d, &stat)
+		if err != nil {
+			m.logger.Warningf("Could not stat device: %v", err)
+			continue
+		}
+		deviceNode := deviceNode{
+			path:  d,
+			major: unix.Major(uint64(stat.Rdev)),
+			minor: unix.Minor(uint64(stat.Rdev)),
+		}
+
+		deviceNodes = append(deviceNodes, deviceNode)
+	}
+
+	return deviceNodes, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m existing) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,126 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverRoot string
+
+	dryRun bool
+
+	control bool
+
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device nodes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "load the NVIDIA Kernel Modules before creating devices nodes",
+			Destination: &opts.loadKernelModules,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	if opts.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(opts.dryRun),
+			nvmodules.WithRoot(opts.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if opts.control {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(opts.dryRun),
+			nvdevices.WithDevRoot(opts.driverRoot),
+		)
+		if err != nil {
+			return err
+		}
+		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/print-ldcache/print-ldcache.go

@@ -0,0 +1,101 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverRoot string
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "print-ldcache",
+		Usage: "A utility to print the contents of the ldcache",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	cache, err := ldcache.New(m.logger, opts.driverRoot)
+	if err != nil {
+		return fmt.Errorf("failed to create ldcache: %v", err)
+	}
+
+	lib32, lib64 := cache.List()
+
+	if len(lib32) == 0 {
+		m.logger.Info("No 32-bit libraries found")
+	} else {
+		m.logger.Infof("%d 32-bit libraries found", len(lib32))
+		for _, lib := range lib32 {
+			m.logger.Infof("%v", lib)
+		}
+	}
+	if len(lib64) == 0 {
+		m.logger.Info("No 64-bit libraries found")
+	} else {
+		m.logger.Infof("%d 64-bit libraries found", len(lib64))
+		for _, lib := range lib64 {
+			m.logger.Infof("%v", lib)
+		}
+	}
+
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/print-ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m command) build() *cli.Command {
+	// Create the 'system' command
+	system := cli.Command{
+		Name:  "system",
+		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	system.Subcommands = []*cli.Command{
+		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
+		ldcache.NewCommand(m.logger),
+	}
+
+	return &system
+}

config/config.toml.amazonlinux

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.centos

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.debian

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.opensuse-leap

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig.real"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

docker/Dockerfile.amazonlinux

@@ -1,73 +0,0 @@
-ARG BASEIMAGE
-FROM ${BASEIMAGE}
-
-RUN yum install -y \
-        ca-certificates \
-        gcc \
-        wget \
-        git \
-        rpm-build \
-        make && \
-    rm -rf /var/cache/yum/*
-
-ARG GOLANG_VERSION=0.0.0
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture"; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
-
-# packaging
-ARG PKG_NAME
-ARG PKG_VERS
-ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
-
-# output directory
-ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
-RUN mkdir -p $DIST_DIR /dist
-
-# nvidia-container-toolkit
-WORKDIR $GOPATH/src/nvidia-container-toolkit
-COPY . .
-
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
-RUN make PREFIX=${DIST_DIR} cmds
-
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-# This might not be useful on Amazon Linux, but it's simpler to keep the RHEL
-# and Amazon Linux packages identical.
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
-WORKDIR $DIST_DIR/..
-COPY packaging/rpm .
-
-CMD arch=$(uname -m) && \
-    rpmbuild --clean --target=$arch -bb \
-             -D "_topdir $PWD" \
-             -D "release_date $(date +'%a %b %d %Y')" \
-             -D "git_commit ${GIT_COMMIT}" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
-             SPECS/nvidia-container-toolkit.spec && \
-    mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -53,26 +53,20 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Debian Jessie still had ldconfig.real
-RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
-       sed -i 's;"@/sbin/ldconfig";"@/sbin/ldconfig.real";' $DIST_DIR/config.toml; \
-    fi
-
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \
             "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
-    dch --append "Bump libnvidia-container dependency to ${REVISION}" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER1_VERSION}" && \
     dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
-    mv /tmp/nvidia-container-toolkit_*.deb /dist
+    mv /tmp/*.deb /dist

docker/Dockerfile.devel

@@ -14,7 +14,8 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go install golang.org/x/lint/golint@latest
+RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
 RUN go install github.com/matryer/moq@latest
-RUN go install github.com/gordonklaus/ineffassign@latest
+RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
 RUN go install github.com/client9/misspell/cmd/misspell@latest
+RUN go install github.com/google/go-licenses@latest

docker/Dockerfile.opensuse-leap

@@ -28,9 +28,9 @@ ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -44,26 +44,19 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "release_date $(date +'%a %b %d %Y')" \
              -D "git_commit ${GIT_COMMIT}" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.rpm-yum

@@ -1,3 +1,19 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the dockerfile for building packages on yum-based RPM systems.
+
 ARG BASEIMAGE
 FROM ${BASEIMAGE}
 
@@ -30,9 +46,9 @@ ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -46,26 +62,19 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "release_date $(date +'%a %b %d %Y')" \
              -D "git_commit ${GIT_COMMIT}" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -51,21 +51,20 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \
             "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
-    dch --append "Bump libnvidia-container dependency to ${REVISION}" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" && \
     dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -17,7 +17,7 @@ AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
 X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := centos8 rhel8 amazonlinux2
+AARCH64_TARGETS := centos7 centos8 rhel8 amazonlinux2
 
 # Define top-level build targets
 docker%: SHELL:=/bin/bash
@@ -85,39 +85,37 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 --%: docker-build-%
 	@
 
+LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
+LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
+
+LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
---ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
---debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
+--rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
 
-# We allow the CONFIG_TOML_SUFFIX to be overridden.
-CONFIG_TOML_SUFFIX ?= $(OS)
 
 docker-build-%:
 	@echo "Building for $(TARGET_PLATFORM)"
@@ -129,10 +127,10 @@ docker-build-%:
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 	    --build-arg PKG_NAME="$(LIB_NAME)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
-	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
-		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
+	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
+	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
+	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \

go.mod

@@ -1,18 +1,36 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.14
+go 1.20
 
 require (
-	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.11.6-0
-	github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
-	github.com/containers/podman/v4 v4.0.3
-	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
-	github.com/pelletier/go-toml v1.9.4
-	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
-	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
-	github.com/urfave/cli/v2 v2.3.0
-	golang.org/x/mod v0.5.0
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
+	github.com/NVIDIA/go-nvml v0.12.0-2
+	github.com/fsnotify/fsnotify v1.7.0
+	github.com/opencontainers/runtime-spec v1.2.0
+	github.com/pelletier/go-toml v1.9.5
+	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.8.4
+	github.com/urfave/cli/v2 v2.27.1
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230818092907-09424fdc8884
+	golang.org/x/mod v0.15.0
+	golang.org/x/sys v0.17.0
+	tags.cncf.io/container-device-interface v0.6.2
+	tags.cncf.io/container-device-interface/specs-go v0.6.0
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

internal/config/cli.go

@@ -16,33 +16,18 @@
 
 package config
 
-import (
-	"github.com/pelletier/go-toml"
-)
-
 // ContainerCLIConfig stores the options for the nvidia-container-cli
 type ContainerCLIConfig struct {
-	Root string
-}
-
-// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
-	cfg := getDefaultContainerCLIConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
-
-	return cfg
-}
-
-// getDefaultContainerCLIConfig defines the default values for the config
-func getDefaultContainerCLIConfig() *ContainerCLIConfig {
-	c := ContainerCLIConfig{
-		Root: "",
-	}
-
-	return &c
+	Root        string   `toml:"root"`
+	Path        string   `toml:"path"`
+	Environment []string `toml:"environment"`
+	Debug       string   `toml:"debug"`
+	Ldcache     string   `toml:"ldcache"`
+	LoadKmods   bool     `toml:"load-kmods"`
+	// NoPivot disables the pivot root operation in the NVIDIA Container CLI.
+	// This is not exposed in the config if not set.
+	NoPivot   bool   `toml:"no-pivot,omitempty"`
+	NoCgroups bool   `toml:"no-cgroups"`
+	User      string `toml:"user"`
+	Ldconfig  string `toml:"ldconfig"`
 }

internal/config/config.go

@@ -17,17 +17,27 @@
 package config
 
 import (
-	"fmt"
-	"io"
+	"bufio"
 	"os"
-	"path"
+	"path/filepath"
+	"strings"
 
-	"github.com/pelletier/go-toml"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 const (
 	configOverride = "XDG_CONFIG_HOME"
 	configFilePath = "nvidia-container-runtime/config.toml"
+
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+
+	nvidiaContainerRuntimeHookExecutable  = "nvidia-container-runtime-hook"
+	nvidiaContainerRuntimeHookDefaultPath = "/usr/bin/nvidia-container-runtime-hook"
 )
 
 var (
@@ -38,77 +48,178 @@ var (
 	NVIDIAContainerRuntimeHookExecutable = "nvidia-container-runtime-hook"
 	// NVIDIAContainerToolkitExecutable is the executable name for the NVIDIA Container Toolkit (an alias for the NVIDIA Container Runtime Hook)
 	NVIDIAContainerToolkitExecutable = "nvidia-container-toolkit"
-
-	configDir = "/etc/"
 )
 
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
-	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+	DisableRequire                 bool   `toml:"disable-require"`
+	SwarmResource                  string `toml:"swarm-resource"`
+	AcceptEnvvarUnprivileged       bool   `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+	AcceptDeviceListAsVolumeMounts bool   `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	SupportedDriverCapabilities    string `toml:"supported-driver-capabilities"`
+
+	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
+}
+
+// GetConfigFilePath returns the path to the config file for the configured system
+func GetConfigFilePath() string {
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		return filepath.Join(XDGConfigDir, configFilePath)
+	}
+
+	return filepath.Join("/etc", configFilePath)
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file
 // or set via the environment.
 func GetConfig() (*Config, error) {
-	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
-		configDir = XDGConfigDir
-	}
-
-	configFilePath := path.Join(configDir, configFilePath)
-
-	tomlFile, err := os.Open(configFilePath)
-	if err != nil {
-		return getDefaultConfig(), nil
-	}
-	defer tomlFile.Close()
-
-	cfg, err := loadConfigFrom(tomlFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read config values: %v", err)
-	}
-
-	return cfg, nil
-}
-
-// loadRuntimeConfigFrom reads the config from the specified Reader
-func loadConfigFrom(reader io.Reader) (*Config, error) {
-	toml, err := toml.LoadReader(reader)
+	cfg, err := New(
+		WithConfigFile(GetConfigFilePath()),
+	)
 	if err != nil {
 		return nil, err
 	}
 
-	return getConfigFrom(toml)
+	return cfg.Config()
 }
 
-// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getConfigFrom(toml *toml.Tree) (*Config, error) {
-	cfg := getDefaultConfig()
+// GetDefault defines the default values for the config
+func GetDefault() (*Config, error) {
+	d := Config{
+		AcceptEnvvarUnprivileged:    true,
+		SupportedDriverCapabilities: image.SupportedDriverCapabilities.String(),
+		NVIDIAContainerCLIConfig: ContainerCLIConfig{
+			LoadKmods: true,
+			Ldconfig:  getLdConfigPath(),
+			User:      getUserGroup(),
+		},
+		NVIDIACTKConfig: CTKConfig{
+			Path: nvidiaCTKExecutable,
+		},
+		NVIDIAContainerRuntimeConfig: RuntimeConfig{
+			DebugFilePath: "/dev/null",
+			LogLevel:      "info",
+			Runtimes:      []string{"docker-runc", "runc", "crun"},
+			Mode:          "auto",
+			Modes: modesConfig{
+				CSV: csvModeConfig{
+					MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+				},
+				CDI: cdiModeConfig{
+					DefaultKind:        "nvidia.com/gpu",
+					AnnotationPrefixes: []string{cdi.AnnotationPrefix},
+					SpecDirs:           cdi.DefaultSpecDirs,
+				},
+			},
+		},
+		NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+			Path: NVIDIAContainerRuntimeHookExecutable,
+		},
+	}
+	return &d, nil
+}
 
-	if toml == nil {
-		return cfg, nil
+func getLdConfigPath() string {
+	if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
+		return "@/sbin/ldconfig.real"
+	}
+	return "@/sbin/ldconfig"
+}
+
+func getUserGroup() string {
+	if isSuse() {
+		return "root:video"
+	}
+	return ""
+}
+
+// isSuse returns whether a SUSE-based distribution was detected.
+func isSuse() bool {
+	suseDists := map[string]bool{
+		"suse":     true,
+		"opensuse": true,
 	}
 
-	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
-	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
-	runtimeConfig, err := getRuntimeConfigFrom(toml)
+	idsLike := getDistIDLike()
+	for _, id := range idsLike {
+		if suseDists[id] {
+			return true
+		}
+	}
+	return false
+}
+
+// getDistIDLike returns the ID_LIKE field from /etc/os-release.
+// We can override this for testing.
+var getDistIDLike = func() []string {
+	releaseFile, err := os.Open("/etc/os-release")
 	if err != nil {
-		return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
+		return nil
 	}
-	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
+	defer releaseFile.Close()
 
-	return cfg, nil
+	scanner := bufio.NewScanner(releaseFile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "ID_LIKE=") {
+			value := strings.Trim(strings.TrimPrefix(line, "ID_LIKE="), "\"")
+			return strings.Split(value, " ")
+		}
+	}
+	return nil
 }
 
-// getDefaultConfig defines the default values for the config
-func getDefaultConfig() *Config {
-	c := Config{
-		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
-		NVIDIACTKConfig:              *getDefaultCTKConfig(),
-		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
+// ResolveNVIDIACTKPath resolves the path to the nvidia-ctk binary.
+// This executable is used in hooks and needs to be an absolute path.
+// If the path is specified as an absolute path, it is used directly
+// without checking for existence of an executable at that path.
+func ResolveNVIDIACTKPath(logger logger.Interface, nvidiaCTKPath string) string {
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Toolkit CLI",
+		nvidiaCTKPath,
+		nvidiaCTKDefaultFilePath,
+	)
+}
+
+// ResolveNVIDIAContainerRuntimeHookPath resolves the path the nvidia-container-runtime-hook binary.
+func ResolveNVIDIAContainerRuntimeHookPath(logger logger.Interface, nvidiaContainerRuntimeHookPath string) string {
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Runtime Hook",
+		nvidiaContainerRuntimeHookPath,
+		nvidiaContainerRuntimeHookDefaultPath,
+	)
+}
+
+// resolveWithDefault resolves the path to the specified binary.
+// If an absolute path is specified, it is used directly without searching for the binary.
+// If the binary cannot be found in the path, the specified default is used instead.
+func resolveWithDefault(logger logger.Interface, label string, path string, defaultPath string) string {
+	if filepath.IsAbs(path) {
+		logger.Debugf("Using specified %v path %v", label, path)
+		return path
 	}
 
-	return &c
+	if path == "" {
+		path = filepath.Base(defaultPath)
+	}
+	logger.Debugf("Locating %v as %v", label, path)
+	lookup := lookup.NewExecutableLocator(logger, "")
+
+	resolvedPath := defaultPath
+	targets, err := lookup.Locate(path)
+	if err != nil {
+		logger.Warningf("Failed to locate %v: %v", path, err)
+	} else {
+		logger.Debugf("Found %v candidates: %v", path, targets)
+		resolvedPath = targets[0]
+	}
+	logger.Debugf("Using %v path %v", label, path)
+
+	return resolvedPath
 }

internal/config/config_test.go

@@ -17,7 +17,6 @@
 package config
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -27,50 +26,61 @@ import (
 )
 
 func TestGetConfigWithCustomConfig(t *testing.T) {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
+	testDir := t.TempDir()
+	t.Setenv(configOverride, testDir)
+
+	filename := filepath.Join(testDir, configFilePath)
 
 	// By default debug is disabled
 	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
-	testDir := filepath.Join(wd, "test")
-	filename := filepath.Join(testDir, configFilePath)
-
-	os.Setenv(configOverride, testDir)
 
 	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
-	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
-
-	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+	require.NoError(t, os.WriteFile(filename, contents, 0766))
 
 	cfg, err := GetConfig()
 	require.NoError(t, err)
-	require.Equal(t, cfg.NVIDIAContainerRuntimeConfig.DebugFilePath, "/nvidia-container-toolkit.log")
+	require.Equal(t, "/nvidia-container-toolkit.log", cfg.NVIDIAContainerRuntimeConfig.DebugFilePath)
 }
 
 func TestGetConfig(t *testing.T) {
 	testCases := []struct {
-		description    string
-		contents       []string
-		expectedError  error
-		expectedConfig *Config
+		description     string
+		contents        []string
+		expectedError   error
+		inspectLdconfig bool
+		distIdsLike     []string
+		expectedConfig  *Config
 	}{
 		{
-			description: "empty config is default",
+			description:     "empty config is default",
+			inspectLdconfig: true,
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "",
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/dev/null",
 					LogLevel:      "info",
-					Runtimes:      []string{"docker-runc", "runc"},
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
 					Mode:          "auto",
 					Modes: modesConfig{
 						CSV: csvModeConfig{
 							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "nvidia-ctk",
 				},
@@ -79,19 +89,32 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set inline",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
+				"supported-driver-capabilities = \"compute,utility\"",
 				"nvidia-container-cli.root = \"/bar/baz\"",
+				"nvidia-container-cli.load-kmods = false",
+				"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
+				"nvidia-container-cli.user = \"foo:bar\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
-				"nvidia-container-runtime.experimental = true",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
+				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"nvidia-container-runtime.modes.cdi.spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"nvidia-container-runtime-hook.path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    false,
+				SupportedDriverCapabilities: "compute,utility",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
+					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -102,8 +125,22 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "/foo/bar/nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
 				},
@@ -112,23 +149,38 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set in section",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
+				"supported-driver-capabilities = \"compute,utility\"",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
+				"load-kmods = false",
+				"ldconfig = \"/foo/bar/ldconfig\"",
+				"user = \"foo:bar\"",
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
-				"experimental = true",
 				"discover-mode = \"not-legacy\"",
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
 				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.cdi]",
+				"default-kind = \"example.vendor.com/device\"",
+				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"[nvidia-container-runtime-hook]",
+				"path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    false,
+				SupportedDriverCapabilities: "compute,utility",
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
+					User:      "foo:bar",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -139,27 +191,147 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "/foo/bar/nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
 				},
 			},
 		},
+		{
+			description:     "suse config",
+			distIdsLike:     []string{"suse", "opensuse"},
+			inspectLdconfig: true,
+			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
+					User:      "root:video",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
+					},
+				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+			},
+		},
+		{
+			description:     "suse config overrides user",
+			distIdsLike:     []string{"suse", "opensuse"},
+			inspectLdconfig: true,
+			contents: []string{
+				"nvidia-container-cli.user = \"foo:bar\"",
+			},
+			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged:    true,
+				SupportedDriverCapabilities: "compat32,compute,display,graphics,ngx,utility,video",
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
+					User:      "foo:bar",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc", "crun"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
+					},
+				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
+			defer setGetDistIDLikeForTest(tc.distIdsLike)()
 			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
 
-			cfg, err := loadConfigFrom(reader)
+			tomlCfg, err := loadConfigTomlFrom(reader)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
 			}
+			cfg, err := tomlCfg.Config()
+			require.NoError(t, err)
+
+			// We first handle the ldconfig path since this is currently system-dependent.
+			if tc.inspectLdconfig {
+				ldconfig := cfg.NVIDIAContainerCLIConfig.Ldconfig
+				require.True(t, strings.HasPrefix(ldconfig, "@/sbin/ldconfig"))
+				remaining := strings.TrimPrefix(ldconfig, "@/sbin/ldconfig")
+				require.True(t, remaining == ".real" || remaining == "")
+
+				cfg.NVIDIAContainerCLIConfig.Ldconfig = "WAS_CHECKED"
+			}
 
 			require.EqualValues(t, tc.expectedConfig, cfg)
 		})
 	}
 }
+
+// setGetDistIDsLikeForTest overrides the distribution IDs that would normally be read from the /etc/os-release file.
+func setGetDistIDLikeForTest(ids []string) func() {
+	if ids == nil {
+		return func() {}
+	}
+	original := getDistIDLike
+
+	getDistIDLike = func() []string {
+		return ids
+	}
+
+	return func() {
+		getDistIDLike = original
+	}
+}

internal/config/hook.go

@@ -0,0 +1,36 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeHookConfig struct {
+	// Path specifies the path to the NVIDIA Container Runtime hook binary.
+	// If an executable name is specified, this will be resolved in the path.
+	Path string `toml:"path"`
+	// SkipModeDetection disables the mode check for the runtime hook.
+	SkipModeDetection bool `toml:"skip-mode-detection"`
+}
+
+// GetDefaultRuntimeHookConfig defines the default values for the config
+func GetDefaultRuntimeHookConfig() (*RuntimeHookConfig, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg.NVIDIAContainerRuntimeHookConfig, nil
+}

internal/config/image/builder.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"strings"
+)
+
+type builder struct {
+	env            []string
+	disableRequire bool
+}
+
+// New creates a new CUDA image from the input options.
+func New(opt ...Option) (CUDA, error) {
+	b := &builder{}
+	for _, o := range opt {
+		o(b)
+	}
+
+	return b.build()
+}
+
+// build creates a CUDA image from the builder.
+func (b builder) build() (CUDA, error) {
+	c := make(CUDA)
+
+	for _, e := range b.env {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid environment variable: %v", e)
+		}
+		c[parts[0]] = parts[1]
+	}
+
+	if b.disableRequire {
+		c[envNVDisableRequire] = "true"
+	}
+
+	return c, nil
+}
+
+// Option is a functional option for creating a CUDA image.
+type Option func(*builder)
+
+// WithDisableRequire sets the disable require option.
+func WithDisableRequire(disableRequire bool) Option {
+	return func(b *builder) {
+		b.disableRequire = disableRequire
+	}
+}
+
+// WithEnv sets the environment variables to use when creating the CUDA image.
+func WithEnv(env []string) Option {
+	return func(b *builder) {
+		b.env = env
+	}
+}

internal/config/image/capabilities.go

@@ -0,0 +1,146 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"sort"
+	"strings"
+)
+
+// DriverCapability represents the possible values of NVIDIA_DRIVER_CAPABILITIES
+type DriverCapability string
+
+// Constants for the supported driver capabilities
+const (
+	DriverCapabilityAll      DriverCapability = "all"
+	DriverCapabilityNone     DriverCapability = "none"
+	DriverCapabilityCompat32 DriverCapability = "compat32"
+	DriverCapabilityCompute  DriverCapability = "compute"
+	DriverCapabilityDisplay  DriverCapability = "display"
+	DriverCapabilityGraphics DriverCapability = "graphics"
+	DriverCapabilityNgx      DriverCapability = "ngx"
+	DriverCapabilityUtility  DriverCapability = "utility"
+	DriverCapabilityVideo    DriverCapability = "video"
+)
+
+var (
+	driverCapabilitiesNone = NewDriverCapabilities()
+	driverCapabilitiesAll  = NewDriverCapabilities("all")
+
+	// DefaultDriverCapabilities sets the value for driver capabilities if no value is set.
+	DefaultDriverCapabilities = NewDriverCapabilities("utility,compute")
+	// SupportedDriverCapabilities defines the set of all supported driver capabilities.
+	SupportedDriverCapabilities = NewDriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
+)
+
+// NewDriverCapabilities creates a set of driver capabilities from the specified capabilities
+func NewDriverCapabilities(capabilities ...string) DriverCapabilities {
+	dc := make(DriverCapabilities)
+	for _, capability := range capabilities {
+		for _, c := range strings.Split(capability, ",") {
+			trimmed := strings.TrimSpace(c)
+			if trimmed == "" {
+				continue
+			}
+			dc[DriverCapability(trimmed)] = true
+		}
+	}
+	return dc
+}
+
+// DriverCapabilities represents the NVIDIA_DRIVER_CAPABILITIES set for the specified image.
+type DriverCapabilities map[DriverCapability]bool
+
+// Has check whether the specified capability is selected.
+func (c DriverCapabilities) Has(capability DriverCapability) bool {
+	if c.IsAll() {
+		return true
+	}
+	return c[capability]
+}
+
+// Any checks whether any of the specified capabilites are set
+func (c DriverCapabilities) Any(capabilities ...DriverCapability) bool {
+	if c.IsAll() {
+		return true
+	}
+	for _, cap := range capabilities {
+		if c.Has(cap) {
+			return true
+		}
+	}
+	return false
+}
+
+// List returns the list of driver capabilities.
+// The list is sorted.
+func (c DriverCapabilities) List() []string {
+	var capabilities []string
+	for capability := range c {
+		capabilities = append(capabilities, string(capability))
+	}
+	sort.Strings(capabilities)
+	return capabilities
+}
+
+// String returns the string repesentation of the driver capabilities.
+func (c DriverCapabilities) String() string {
+	if c.IsAll() {
+		return "all"
+	}
+	return strings.Join(c.List(), ",")
+}
+
+// IsAll indicates whether the set of capabilities is `all`
+func (c DriverCapabilities) IsAll() bool {
+	return c[DriverCapabilityAll]
+}
+
+// Intersection returns a new set which includes the item in BOTH d and s2.
+// For example: d = {a1, a2} s2 = {a2, a3} s1.Intersection(s2) = {a2}
+func (c DriverCapabilities) Intersection(s2 DriverCapabilities) DriverCapabilities {
+	if s2.IsAll() {
+		return c
+	}
+	if c.IsAll() {
+		return s2
+	}
+
+	intersection := make(DriverCapabilities)
+	for capability := range s2 {
+		if c[capability] {
+			intersection[capability] = true
+		}
+	}
+
+	return intersection
+}
+
+// IsSuperset returns true if and only if d is a superset of s2.
+func (c DriverCapabilities) IsSuperset(s2 DriverCapabilities) bool {
+	if c.IsAll() {
+		return true
+	}
+
+	for capability := range s2 {
+		if !c[capability] {
+			return false
+		}
+	}
+
+	return true
+}

internal/config/image/capabilities_test.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDriverCapabilitiesIntersection(t *testing.T) {
+	testCases := []struct {
+		capabilities          DriverCapabilities
+		supportedCapabilities DriverCapabilities
+		expectedIntersection  DriverCapabilities
+	}{
+		{
+			capabilities:          driverCapabilitiesNone,
+			supportedCapabilities: driverCapabilitiesNone,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          driverCapabilitiesAll,
+			supportedCapabilities: driverCapabilitiesNone,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          driverCapabilitiesAll,
+			supportedCapabilities: SupportedDriverCapabilities,
+			expectedIntersection:  SupportedDriverCapabilities,
+		},
+		{
+			capabilities:          SupportedDriverCapabilities,
+			supportedCapabilities: driverCapabilitiesAll,
+			expectedIntersection:  SupportedDriverCapabilities,
+		},
+		{
+			capabilities:          driverCapabilitiesNone,
+			supportedCapabilities: driverCapabilitiesAll,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          driverCapabilitiesNone,
+			supportedCapabilities: NewDriverCapabilities("cap1"),
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          NewDriverCapabilities("cap0,cap1"),
+			supportedCapabilities: NewDriverCapabilities("cap1,cap0"),
+			expectedIntersection:  NewDriverCapabilities("cap0,cap1"),
+		},
+		{
+			capabilities:          DefaultDriverCapabilities,
+			supportedCapabilities: SupportedDriverCapabilities,
+			expectedIntersection:  DefaultDriverCapabilities,
+		},
+		{
+			capabilities:          NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			supportedCapabilities: NewDriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			expectedIntersection:  NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+		{
+			capabilities:          NewDriverCapabilities("cap1"),
+			supportedCapabilities: driverCapabilitiesNone,
+			expectedIntersection:  driverCapabilitiesNone,
+		},
+		{
+			capabilities:          NewDriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			supportedCapabilities: NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			expectedIntersection:  NewDriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
+			require.EqualValues(t, tc.expectedIntersection, intersection)
+		})
+	}
+}
+
+func TestDriverCapabilitiesList(t *testing.T) {
+	testCases := []struct {
+		capabilities DriverCapabilities
+		expected     []string
+	}{
+		{
+			capabilities: NewDriverCapabilities(""),
+		},
+		{
+			capabilities: NewDriverCapabilities("  "),
+		},
+		{
+			capabilities: NewDriverCapabilities(","),
+		},
+		{
+			capabilities: NewDriverCapabilities(",cap"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: NewDriverCapabilities("cap,"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: NewDriverCapabilities("cap0,,cap1"),
+			expected:     []string{"cap0", "cap1"},
+		},
+		{
+			capabilities: NewDriverCapabilities("cap1,cap0,cap3"),
+			expected:     []string{"cap0", "cap1", "cap3"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			require.EqualValues(t, tc.expected, tc.capabilities.List())
+		})
+	}
+}

internal/config/image/cuda_image.go

@@ -26,11 +26,12 @@ import (
 )
 
 const (
-	envCUDAVersion      = "CUDA_VERSION"
-	envNVRequirePrefix  = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA    = envNVRequirePrefix + "CUDA"
-	envNVRequireJetpack = envNVRequirePrefix + "JETPACK"
-	envNVDisableRequire = "NVIDIA_DISABLE_REQUIRE"
+	envCUDAVersion          = "CUDA_VERSION"
+	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
+	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
+	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
 // CUDA represents a CUDA image that can be used for GPU computing. This wraps
@@ -41,27 +42,18 @@ type CUDA map[string]string
 // NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
 // The process environment is read (if present) to construc the CUDA Image.
 func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
-	if spec == nil || spec.Process == nil {
-		return NewCUDAImageFromEnv(nil)
+	var env []string
+	if spec != nil && spec.Process != nil {
+		env = spec.Process.Env
 	}
 
-	return NewCUDAImageFromEnv(spec.Process.Env)
+	return New(WithEnv(env))
 }
 
 // NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
 // is a list of strings of the form ENVAR=VALUE.
 func NewCUDAImageFromEnv(env []string) (CUDA, error) {
-	c := make(CUDA)
-
-	for _, e := range env {
-		parts := strings.SplitN(e, "=", 2)
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("invalid environment variable: %v", e)
-		}
-		c[parts[0]] = parts[1]
-	}
-
-	return c, nil
+	return New(WithEnv(env))
 }
 
 // IsLegacy returns whether the associated CUDA image is a "legacy" image. An
@@ -76,11 +68,9 @@ func (i CUDA) IsLegacy() bool {
 // GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
 // variables.
 func (i CUDA) GetRequirements() ([]string, error) {
-	// TODO: We need not process this if disable require is set, but this will be done
-	// in a single follow-up to ensure that the behavioural change is accurately captured.
-	// if i.HasDisableRequire() {
-	// 	return nil, nil
-	// }
+	if i.HasDisableRequire() {
+		return nil, nil
+	}
 
 	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
 	var requirements []string
@@ -112,10 +102,56 @@ func (i CUDA) HasDisableRequire() bool {
 	return false
 }
 
+// DevicesFromEnvvars returns the devices requested by the image through environment variables
+func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+	// We concantenate all the devices from the specified envvars.
+	var isSet bool
+	var devices []string
+	requested := make(map[string]bool)
+	for _, envVar := range envVars {
+		if devs, ok := i[envVar]; ok {
+			isSet = true
+			for _, d := range strings.Split(devs, ",") {
+				trimmed := strings.TrimSpace(d)
+				if len(trimmed) == 0 {
+					continue
+				}
+				devices = append(devices, trimmed)
+				requested[trimmed] = true
+			}
+		}
+	}
+
+	// Environment variable unset with legacy image: default to "all".
+	if !isSet && len(devices) == 0 && i.IsLegacy() {
+		return NewVisibleDevices("all")
+	}
+
+	// Environment variable unset or empty or "void": return nil
+	if len(devices) == 0 || requested["void"] {
+		return NewVisibleDevices("void")
+	}
+
+	return NewVisibleDevices(devices...)
+}
+
+// GetDriverCapabilities returns the requested driver capabilities.
+func (i CUDA) GetDriverCapabilities() DriverCapabilities {
+	env := i[envNVDriverCapabilities]
+
+	capabilites := make(DriverCapabilities)
+	for _, c := range strings.Split(env, ",") {
+		capabilites[DriverCapability(c)] = true
+	}
+
+	return capabilites
+}
+
 func (i CUDA) legacyVersion() (string, error) {
-	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
+	cudaVersion := i[envCUDAVersion]
+	majorMinor, err := parseMajorMinorVersion(cudaVersion)
 	if err != nil {
-		return "", fmt.Errorf("invalid CUDA version: %v", err)
+		return "", fmt.Errorf("invalid CUDA version %v: %v", cudaVersion, err)
 	}
 
 	return majorMinor, nil

internal/config/image/cuda_image_test.go

@@ -106,6 +106,16 @@ func TestGetRequirements(t *testing.T) {
 			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
 			requirements: []string{"cuda>=11.6", "brand=tesla"},
 		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores requirements",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores legacy image requirements",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
 	}
 
 	for _, tc := range testCases {

internal/config/image/devices.go

@@ -0,0 +1,127 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"strings"
+)
+
+// VisibleDevices represents the devices selected in a container image
+// through the NVIDIA_VISIBLE_DEVICES or other environment variables
+type VisibleDevices interface {
+	List() []string
+	Has(string) bool
+}
+
+var _ VisibleDevices = (*all)(nil)
+var _ VisibleDevices = (*none)(nil)
+var _ VisibleDevices = (*void)(nil)
+var _ VisibleDevices = (*devices)(nil)
+
+// NewVisibleDevices creates a VisibleDevices based on the value of the specified envvar.
+func NewVisibleDevices(envvars ...string) VisibleDevices {
+	for _, envvar := range envvars {
+		if envvar == "all" {
+			return all{}
+		}
+		if envvar == "none" {
+			return none{}
+		}
+		if envvar == "" || envvar == "void" {
+			return void{}
+		}
+	}
+
+	return newDevices(envvars...)
+}
+
+type all struct{}
+
+// List returns ["all"] for all devices
+func (a all) List() []string {
+	return []string{"all"}
+}
+
+// Has for all devices is true for any id except the empty ID
+func (a all) Has(id string) bool {
+	return id != ""
+}
+
+type none struct{}
+
+// List returns [""] for the none devices
+func (n none) List() []string {
+	return []string{""}
+}
+
+// Has for none devices is false for any id
+func (n none) Has(id string) bool {
+	return false
+}
+
+type void struct {
+	none
+}
+
+// List returns nil for the void devices
+func (v void) List() []string {
+	return nil
+}
+
+type devices struct {
+	len    int
+	lookup map[string]int
+}
+
+func newDevices(idOrCommaSeparated ...string) devices {
+	lookup := make(map[string]int)
+
+	i := 0
+	for _, commaSeparated := range idOrCommaSeparated {
+		for _, id := range strings.Split(commaSeparated, ",") {
+			lookup[id] = i
+			i++
+		}
+	}
+
+	d := devices{
+		len:    i,
+		lookup: lookup,
+	}
+	return d
+}
+
+// List returns the list of requested devices
+func (d devices) List() []string {
+	list := make([]string, d.len)
+
+	for id, i := range d.lookup {
+		list[i] = id
+	}
+
+	return list
+}
+
+// Has checks whether the specified ID is in the set of requested devices
+func (d devices) Has(id string) bool {
+	if id == "" {
+		return false
+	}
+
+	_, exist := d.lookup[id]
+	return exist
+}

internal/config/image/privileged.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged NVIDIA capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

@@ -16,20 +16,6 @@
 
 package config
 
-import (
-	"fmt"
-
-	"github.com/pelletier/go-toml"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	dockerRuncExecutableName = "docker-runc"
-	runcExecutableName       = "runc"
-
-	auto = "auto"
-)
-
 // RuntimeConfig stores the config options for the NVIDIA Container Runtime
 type RuntimeConfig struct {
 	DebugFilePath string `toml:"debug"`
@@ -44,52 +30,28 @@ type RuntimeConfig struct {
 // modesConfig defines (optional) per-mode configs
 type modesConfig struct {
 	CSV csvModeConfig `toml:"csv"`
+	CDI cdiModeConfig `toml:"cdi"`
+}
+
+type cdiModeConfig struct {
+	// SpecDirs allows for the default spec dirs for CDI to be overridden
+	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
 }
 
 type csvModeConfig struct {
 	MountSpecPath string `toml:"mount-spec-path"`
 }
 
-// dummy allows us to unmarshal only a RuntimeConfig from a *toml.Tree
-type dummy struct {
-	Runtime RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-// getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getRuntimeConfigFrom(toml *toml.Tree) (*RuntimeConfig, error) {
-	cfg := GetDefaultRuntimeConfig()
-
-	if toml == nil {
-		return cfg, nil
-	}
-
-	d := dummy{
-		Runtime: *cfg,
-	}
-
-	if err := toml.Unmarshal(&d); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
-	}
-
-	return &d.Runtime, nil
-}
-
 // GetDefaultRuntimeConfig defines the default values for the config
-func GetDefaultRuntimeConfig() *RuntimeConfig {
-	c := RuntimeConfig{
-		DebugFilePath: "/dev/null",
-		LogLevel:      logrus.InfoLevel.String(),
-		Runtimes: []string{
-			dockerRuncExecutableName,
-			runcExecutableName,
-		},
-		Mode: auto,
-		Modes: modesConfig{
-			CSV: csvModeConfig{
-				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
-			},
-		},
+func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
 	}
 
-	return &c
+	return &cfg.NVIDIAContainerRuntimeConfig, nil
 }

internal/config/toml.go

@@ -0,0 +1,218 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+
+	"github.com/pelletier/go-toml"
+)
+
+// Toml is a type for the TOML representation of a config.
+type Toml toml.Tree
+
+type options struct {
+	configFile string
+	required   bool
+}
+
+// Option is a functional option for loading TOML config files.
+type Option func(*options)
+
+// WithConfigFile sets the config file option.
+func WithConfigFile(configFile string) Option {
+	return func(o *options) {
+		o.configFile = configFile
+	}
+}
+
+// WithRequired sets the required option.
+// If this is set to true, a failure to open the specified file is treated as an error
+func WithRequired(required bool) Option {
+	return func(o *options) {
+		o.required = required
+	}
+}
+
+// New creates a new toml tree based on the provided options
+func New(opts ...Option) (*Toml, error) {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return o.loadConfigToml()
+}
+
+func (o options) loadConfigToml() (*Toml, error) {
+	filename := o.configFile
+	if filename == "" {
+		return defaultToml()
+	}
+
+	_, err := os.Stat(filename)
+	if os.IsNotExist(err) && o.required {
+		return nil, os.ErrNotExist
+	}
+
+	tomlFile, err := os.Open(filename)
+	if os.IsNotExist(err) {
+		return defaultToml()
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to load specified config file: %w", err)
+	}
+	defer tomlFile.Close()
+
+	return loadConfigTomlFrom(tomlFile)
+
+}
+
+func defaultToml() (*Toml, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	contents, err := toml.Marshal(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return loadConfigTomlFrom(bytes.NewReader(contents))
+}
+
+func loadConfigTomlFrom(reader io.Reader) (*Toml, error) {
+	tree, err := toml.LoadReader(reader)
+	if err != nil {
+		return nil, err
+	}
+	return (*Toml)(tree), nil
+}
+
+// Config returns the typed config associated with the toml tree.
+func (t *Toml) Config() (*Config, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	if t == nil {
+		return cfg, nil
+	}
+	if err := t.Unmarshal(cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal config: %v", err)
+	}
+	return cfg, nil
+}
+
+// Unmarshal wraps the toml.Tree Unmarshal function.
+func (t *Toml) Unmarshal(v interface{}) error {
+	return (*toml.Tree)(t).Unmarshal(v)
+}
+
+// Save saves the config to the specified Writer.
+func (t *Toml) Save(w io.Writer) (int64, error) {
+	contents, err := t.contents()
+	if err != nil {
+		return 0, err
+	}
+
+	n, err := w.Write(contents)
+	return int64(n), err
+}
+
+// contents returns the config TOML as a byte slice.
+// Any required formatting is applied.
+func (t Toml) contents() ([]byte, error) {
+	commented := t.commentDefaults()
+
+	buffer := bytes.NewBuffer(nil)
+
+	enc := toml.NewEncoder(buffer).Indentation("")
+	if err := enc.Encode((*toml.Tree)(commented)); err != nil {
+		return nil, fmt.Errorf("invalid config: %v", err)
+	}
+	return t.format(buffer.Bytes())
+}
+
+// format fixes the comments for the config to ensure that they start in column
+// 1 and are not followed by a space.
+func (t Toml) format(contents []byte) ([]byte, error) {
+	r, err := regexp.Compile(`(\n*)\s*?#\s*(\S.*)`)
+	if err != nil {
+		return nil, fmt.Errorf("unable to compile regexp: %v", err)
+	}
+	replaced := r.ReplaceAll(contents, []byte("$1#$2"))
+
+	return replaced, nil
+}
+
+// Delete deletes the specified key from the TOML config.
+func (t *Toml) Delete(key string) error {
+	return (*toml.Tree)(t).Delete(key)
+}
+
+// Get returns the value for the specified key.
+func (t *Toml) Get(key string) interface{} {
+	return (*toml.Tree)(t).Get(key)
+}
+
+// Set sets the specified key to the specified value in the TOML config.
+func (t *Toml) Set(key string, value interface{}) {
+	(*toml.Tree)(t).Set(key, value)
+}
+
+// commentDefaults applies the required comments for default values to the Toml.
+func (t *Toml) commentDefaults() *Toml {
+	asToml := (*toml.Tree)(t)
+	commentedDefaults := map[string]interface{}{
+		"swarm-resource": "DOCKER_RESOURCE_GPU",
+		"accept-nvidia-visible-devices-envvar-when-unprivileged": true,
+		"accept-nvidia-visible-devices-as-volume-mounts":         false,
+		"nvidia-container-cli.root":                              "/run/nvidia/driver",
+		"nvidia-container-cli.path":                              "/usr/bin/nvidia-container-cli",
+		"nvidia-container-cli.debug":                             "/var/log/nvidia-container-toolkit.log",
+		"nvidia-container-cli.ldcache":                           "/etc/ld.so.cache",
+		"nvidia-container-cli.no-cgroups":                        false,
+		"nvidia-container-cli.user":                              "root:video",
+		"nvidia-container-runtime.debug":                         "/var/log/nvidia-container-runtime.log",
+	}
+	for k, v := range commentedDefaults {
+		set := asToml.Get(k)
+		if !shouldComment(k, v, set) {
+			continue
+		}
+		asToml.SetWithComment(k, "", true, v)
+	}
+	return (*Toml)(asToml)
+}
+
+func shouldComment(key string, defaultValue interface{}, setTo interface{}) bool {
+	if key == "nvidia-container-cli.user" && defaultValue == setTo && isSuse() {
+		return false
+	}
+	if key == "nvidia-container-runtime.debug" && setTo == "/dev/null" {
+		return true
+	}
+	if setTo == nil || defaultValue == setTo || setTo == "" {
+		return true
+	}
+
+	return false
+}

internal/config/toml_test.go

@@ -0,0 +1,248 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTomlSave(t *testing.T) {
+	testCases := []struct {
+		description string
+		config      *Toml
+		expected    string
+	}{
+		{
+			description: "defaultConfig",
+			config: func() *Toml {
+				t, _ := defaultToml()
+				// TODO: We handle the ldconfig path specifically, since this is platform
+				// dependent.
+				(*toml.Tree)(t).Set("nvidia-container-cli.ldconfig", "OVERRIDDEN")
+				return t
+			}(),
+			expected: `
+#accept-nvidia-visible-devices-as-volume-mounts = false
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+#debug = "/var/log/nvidia-container-toolkit.log"
+environment = []
+#ldcache = "/etc/ld.so.cache"
+ldconfig = "OVERRIDDEN"
+load-kmods = true
+#no-cgroups = false
+#path = "/usr/bin/nvidia-container-cli"
+#root = "/run/nvidia/driver"
+#user = "root:video"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"
+log-level = "info"
+mode = "auto"
+runtimes = ["docker-runc", "runc", "crun"]
+
+[nvidia-container-runtime.modes]
+
+[nvidia-container-runtime.modes.cdi]
+annotation-prefixes = ["cdi.k8s.io/"]
+default-kind = "nvidia.com/gpu"
+spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+[nvidia-container-runtime.modes.csv]
+mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+path = "nvidia-container-runtime-hook"
+skip-mode-detection = false
+
+[nvidia-ctk]
+path = "nvidia-ctk"
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			buffer := new(bytes.Buffer)
+			_, err := tc.config.Save(buffer)
+			require.NoError(t, err)
+
+			require.EqualValues(t,
+				strings.TrimSpace(tc.expected),
+				strings.TrimSpace(buffer.String()),
+			)
+		})
+	}
+}
+
+func TestFormat(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "# comment",
+			expected: "#comment",
+		},
+		{
+			input:    " #comment",
+			expected: "#comment",
+		},
+		{
+			input:    " # comment",
+			expected: "#comment",
+		},
+		{
+			input: strings.Join([]string{
+				"some",
+				"# comment",
+				" # comment",
+				" #comment",
+				"other"}, "\n"),
+			expected: strings.Join([]string{
+				"some",
+				"#comment",
+				"#comment",
+				"#comment",
+				"other"}, "\n"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			actual, _ := (Toml{}).format([]byte(tc.input))
+			require.Equal(t, tc.expected, string(actual))
+		})
+	}
+}
+
+func TestGetFormattedConfig(t *testing.T) {
+	expectedLines := []string{
+		"#no-cgroups = false",
+		"#debug = \"/var/log/nvidia-container-toolkit.log\"",
+		"#debug = \"/var/log/nvidia-container-runtime.log\"",
+	}
+
+	contents, err := createEmpty().contents()
+	require.NoError(t, err)
+	lines := strings.Split(string(contents), "\n")
+
+	for _, line := range expectedLines {
+		require.Contains(t, lines, line)
+	}
+}
+
+func TestTomlContents(t *testing.T) {
+	testCases := []struct {
+		description string
+		contents    map[string]interface{}
+		expected    string
+	}{
+		{
+			description: "empty config returns commented defaults",
+			expected: `
+#accept-nvidia-visible-devices-as-volume-mounts = false
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+#no-cgroups = false
+#path = "/usr/bin/nvidia-container-cli"
+#root = "/run/nvidia/driver"
+#user = "root:video"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tree, err := toml.TreeFromMap(tc.contents)
+			require.NoError(t, err)
+			cfg := (*Toml)(tree)
+			contents, err := cfg.contents()
+			require.NoError(t, err)
+
+			require.EqualValues(t,
+				strings.TrimSpace(tc.expected),
+				strings.TrimSpace(string(contents)),
+			)
+		})
+	}
+}
+
+func TestConfigFromToml(t *testing.T) {
+	testCases := []struct {
+		description    string
+		contents       map[string]interface{}
+		expectedConfig *Config
+	}{
+		{
+			description: "empty config returns default config",
+			contents:    nil,
+			expectedConfig: func() *Config {
+				c, _ := GetDefault()
+				return c
+			}(),
+		},
+		{
+			description: "contents overrides default",
+			contents: map[string]interface{}{
+				"nvidia-container-runtime": map[string]interface{}{
+					"debug": "/some/log/file.log",
+					"mode":  "csv",
+				},
+			},
+			expectedConfig: func() *Config {
+				c, _ := GetDefault()
+				c.NVIDIAContainerRuntimeConfig.DebugFilePath = "/some/log/file.log"
+				c.NVIDIAContainerRuntimeConfig.Mode = "csv"
+				return c
+			}(),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tomlCfg := fromMap(tc.contents)
+			config, err := tomlCfg.Config()
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expectedConfig, config)
+		})
+	}
+}
+
+func fromMap(c map[string]interface{}) *Toml {
+	tree, _ := toml.TreeFromMap(c)
+	return (*Toml)(tree)
+}
+
+func createEmpty() *Toml {
+	return fromMap(nil)
+}

internal/config/toolkit-cli.go

@@ -16,31 +16,7 @@
 
 package config
 
-import "github.com/pelletier/go-toml"
-
 // CTKConfig stores the config options for the NVIDIA Container Toolkit CLI (nvidia-ctk)
 type CTKConfig struct {
 	Path string `toml:"path"`
 }
-
-// getCTKConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getCTKConfigFrom(toml *toml.Tree) *CTKConfig {
-	cfg := getDefaultCTKConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Path = toml.GetDefault("nvidia-ctk.path", cfg.Path).(string)
-
-	return cfg
-}
-
-// getDefaultCTKConfig defines the default values for the config
-func getDefaultCTKConfig() *CTKConfig {
-	c := CTKConfig{
-		Path: "nvidia-ctk",
-	}
-
-	return &c
-}

internal/discover/char_devices.go

@@ -0,0 +1,69 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
+func NewCharDeviceDiscoverer(logger logger.Interface, devices []string, root string) Discover {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(root),
+	)
+
+	return NewDeviceDiscoverer(logger, locator, root, devices)
+}
+
+// NewDeviceDiscoverer creates a discoverer which locates the specified set of device nodes using the specified locator.
+func NewDeviceDiscoverer(logger logger.Interface, locator lookup.Locator, root string, devices []string) Discover {
+	m := NewMounts(logger, locator, root, devices).(*mounts)
+
+	return (*charDevices)(m)
+}
+
+// Mounts returns the discovered mounts for the charDevices.
+// Since this explicitly specifies a device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices.
+// Here the device nodes are first discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	devicesAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range devicesAsMounts {
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
+	}
+
+	return devices, nil
+}

internal/discover/char_devices_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
@@ -49,7 +48,7 @@ func TestCharDevices(t *testing.T) {
 					required: []string{"required"},
 				},
 			),
-			expectedDevices: []Device{{Path: "located"}},
+			expectedDevices: []Device{{Path: "located", HostPath: "located"}},
 		},
 		{
 			description:          "dev devices returns error for nil lookup",
@@ -82,79 +81,3 @@ func TestCharDevices(t *testing.T) {
 		})
 	}
 }
-
-func TestNewFromMountSpec(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	locators := map[csv.MountSpecType]lookup.Locator{
-		"dev": &lookup.LocatorMock{},
-		"lib": &lookup.LocatorMock{},
-	}
-
-	testCases := []struct {
-		description        string
-		targets            []*csv.MountSpec
-		expectedError      error
-		expectedDiscoverer Discover
-	}{
-		{
-			description:        "empty targets returns None discoverer list",
-			expectedDiscoverer: &None{},
-		},
-		{
-			description: "unexpected locator returns error",
-			targets: []*csv.MountSpec{
-				{
-					Type: "foo",
-					Path: "bar",
-				},
-			},
-			expectedError: fmt.Errorf("no locator defined for foo"),
-		},
-		{
-			description: "creates discoverers based on type",
-			targets: []*csv.MountSpec{
-				{
-					Type: "dev",
-					Path: "dev0",
-				},
-				{
-					Type: "lib",
-					Path: "lib0",
-				},
-				{
-					Type: "dev",
-					Path: "dev1",
-				},
-			},
-			expectedDiscoverer: &list{
-				discoverers: []Discover{
-					(*charDevices)(
-						&mounts{
-							logger:   logger,
-							lookup:   locators["dev"],
-							required: []string{"dev0", "dev1"},
-						},
-					),
-					&mounts{
-						logger:   logger,
-						lookup:   locators["lib"],
-						required: []string{"lib0"},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			discoverer, err := newFromMountSpecs(logger, locators, tc.targets)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-				return
-			}
-			require.NoError(t, err)
-			require.EqualValues(t, tc.expectedDiscoverer, discoverer)
-		})
-	}
-}

internal/discover/csv.go

@@ -1,139 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package discover
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
-)
-
-// charDevices is a discover for a list of character devices
-type charDevices mounts
-
-var _ Discover = (*charDevices)(nil)
-
-// NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
-// The constructed discoverer is comprised of a list, with each element in the list being associated with a
-// single CSV files.
-func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
-	if len(files) == 0 {
-		logger.Warnf("No CSV files specified")
-		return None{}, nil
-	}
-
-	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
-	locators := map[csv.MountSpecType]lookup.Locator{
-		csv.MountSpecDev: lookup.NewCharDeviceLocator(logger, root),
-		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
-		// Libraries and symlinks are handled in the same way
-		csv.MountSpecLib: symlinkLocator,
-		csv.MountSpecSym: symlinkLocator,
-	}
-
-	var discoverers []Discover
-	for _, filename := range files {
-		d, err := NewFromCSVFile(logger, locators, filename)
-		if err != nil {
-			logger.Warnf("Skipping CSV file %v: %v", filename, err)
-			continue
-		}
-		discoverers = append(discoverers, d)
-	}
-
-	return &list{discoverers: discoverers}, nil
-}
-
-// NewFromCSVFile creates a discoverer for the specified CSV file. A logger is also supplied.
-// The constructed discoverer is comprised of a list, with each element in the list being associated with a particular
-// MountSpecType.
-func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, filename string) (Discover, error) {
-	// Create a discoverer for each file-kind combination
-	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse CSV file: %v", err)
-	}
-	if len(targets) == 0 {
-		return nil, fmt.Errorf("CSV file is empty")
-	}
-
-	return newFromMountSpecs(logger, locators, targets)
-}
-
-// newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
-// A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
-func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, targets []*csv.MountSpec) (Discover, error) {
-	if len(targets) == 0 {
-		return &None{}, nil
-	}
-
-	var discoverers []Discover
-	var mountSpecTypes []csv.MountSpecType
-	candidatesByType := make(map[csv.MountSpecType][]string)
-	for _, t := range targets {
-		if _, exists := candidatesByType[t.Type]; !exists {
-			mountSpecTypes = append(mountSpecTypes, t.Type)
-		}
-		candidatesByType[t.Type] = append(candidatesByType[t.Type], t.Path)
-	}
-
-	for _, t := range mountSpecTypes {
-		locator, exists := locators[t]
-		if !exists {
-			return nil, fmt.Errorf("no locator defined for '%v'", t)
-		}
-
-		m := &mounts{
-			logger:   logger,
-			lookup:   locator,
-			required: candidatesByType[t],
-		}
-
-		switch t {
-		case csv.MountSpecDev:
-			// For device mount specs, we insert a charDevices into the list of discoverers.
-			discoverers = append(discoverers, (*charDevices)(m))
-		default:
-			discoverers = append(discoverers, m)
-		}
-	}
-
-	return &list{discoverers: discoverers}, nil
-}
-
-// Mounts returns the discovered mounts for the charDevices. Since this explicitly specifies a
-// device list, the mounts are nil.
-func (d *charDevices) Mounts() ([]Mount, error) {
-	return nil, nil
-}
-
-// Devices returns the discovered devices for the charDevices. Here the device nodes are first
-// discovered as mounts and these are converted to devices.
-func (d *charDevices) Devices() ([]Device, error) {
-	devicesAsMounts, err := (*mounts)(d).Mounts()
-	if err != nil {
-		return nil, err
-	}
-	var devices []Device
-	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
-	}
-
-	return devices, nil
-}

internal/discover/discover.go

@@ -16,20 +16,17 @@
 
 package discover
 
-// Config represents the configuration options for discovery
-type Config struct {
-	Root                                    string
-	NVIDIAContainerToolkitCLIExecutablePath string
-}
-
 // Device represents a discovered character device.
 type Device struct {
-	Path string
+	HostPath string
+	Path     string
 }
 
 // Mount represents a discovered mount.
 type Mount struct {
-	Path string
+	HostPath string
+	Path     string
+	Options  []string
 }
 
 // Hook represents a discovered hook.
@@ -39,8 +36,9 @@ type Hook struct {
 	Args      []string
 }
 
-//go:generate moq -stub -out discover_mock.go . Discover
 // Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+//
+//go:generate moq -stub -out discover_mock.go . Discover
 type Discover interface {
 	Devices() ([]Device, error)
 	Mounts() ([]Mount, error)

internal/discover/discover_mock.go

@@ -13,25 +13,25 @@ var _ Discover = &DiscoverMock{}
 
 // DiscoverMock is a mock implementation of Discover.
 //
-// 	func TestSomethingThatUsesDiscover(t *testing.T) {
+//	func TestSomethingThatUsesDiscover(t *testing.T) {
 //
-// 		// make and configure a mocked Discover
-// 		mockedDiscover := &DiscoverMock{
-// 			DevicesFunc: func() ([]Device, error) {
-// 				panic("mock out the Devices method")
-// 			},
-// 			HooksFunc: func() ([]Hook, error) {
-// 				panic("mock out the Hooks method")
-// 			},
-// 			MountsFunc: func() ([]Mount, error) {
-// 				panic("mock out the Mounts method")
-// 			},
-// 		}
+//		// make and configure a mocked Discover
+//		mockedDiscover := &DiscoverMock{
+//			DevicesFunc: func() ([]Device, error) {
+//				panic("mock out the Devices method")
+//			},
+//			HooksFunc: func() ([]Hook, error) {
+//				panic("mock out the Hooks method")
+//			},
+//			MountsFunc: func() ([]Mount, error) {
+//				panic("mock out the Mounts method")
+//			},
+//		}
 //
-// 		// use mockedDiscover in code that requires Discover
-// 		// and then make assertions.
+//		// use mockedDiscover in code that requires Discover
+//		// and then make assertions.
 //
-// 	}
+//	}
 type DiscoverMock struct {
 	// DevicesFunc mocks the Devices method.
 	DevicesFunc func() ([]Device, error)
@@ -78,7 +78,8 @@ func (mock *DiscoverMock) Devices() ([]Device, error) {
 
 // DevicesCalls gets all the calls that were made to Devices.
 // Check the length with:
-//     len(mockedDiscover.DevicesCalls())
+//
+//	len(mockedDiscover.DevicesCalls())
 func (mock *DiscoverMock) DevicesCalls() []struct {
 } {
 	var calls []struct {
@@ -108,7 +109,8 @@ func (mock *DiscoverMock) Hooks() ([]Hook, error) {
 
 // HooksCalls gets all the calls that were made to Hooks.
 // Check the length with:
-//     len(mockedDiscover.HooksCalls())
+//
+//	len(mockedDiscover.HooksCalls())
 func (mock *DiscoverMock) HooksCalls() []struct {
 } {
 	var calls []struct {
@@ -138,7 +140,8 @@ func (mock *DiscoverMock) Mounts() ([]Mount, error) {
 
 // MountsCalls gets all the calls that were made to Mounts.
 // Check the length with:
-//     len(mockedDiscover.MountsCalls())
+//
+//	len(mockedDiscover.MountsCalls())
 func (mock *DiscoverMock) MountsCalls() []struct {
 } {
 	var calls []struct {

internal/discover/filter.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// Filter defines an interface for filtering discovered entities
+type Filter interface {
+	DeviceIsSelected(device Device) bool
+}
+
+// filtered represents a filtered discoverer
+type filtered struct {
+	Discover
+	logger logger.Interface
+	filter Filter
+}
+
+// newFilteredDisoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
+func newFilteredDisoverer(logger logger.Interface, applyTo Discover, filter Filter) Discover {
+	return filtered{
+		Discover: applyTo,
+		logger:   logger,
+		filter:   filter,
+	}
+}
+
+// Devices returns a filtered list of devices based on the specified filter.
+func (d filtered) Devices() ([]Device, error) {
+	devices, err := d.Discover.Devices()
+	if err != nil {
+		return nil, err
+	}
+
+	if d.filter == nil {
+		return devices, nil
+	}
+
+	var selected []Device
+	for _, device := range devices {
+		if d.filter.DeviceIsSelected(device) {
+			selected = append(selected, device)
+		}
+		d.logger.Debugf("skipping device %v", device)
+	}
+
+	return selected, nil
+}

internal/discover/gds.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type gdsDeviceDiscoverer struct {
+	None
+	logger  logger.Interface
+	devices Discover
+	mounts  Discover
+}
+
+// NewGDSDiscoverer creates a discoverer for GPUDirect Storage devices and mounts.
+func NewGDSDiscoverer(logger logger.Interface, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{"/dev/nvidia-fs*"},
+		root,
+	)
+
+	udev := NewMounts(
+		logger,
+		lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(root)),
+		root,
+		[]string{"/run/udev"},
+	)
+
+	cufile := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{"/etc/cufile.json"},
+	)
+
+	d := gdsDeviceDiscoverer{
+		logger:  logger,
+		devices: devices,
+		mounts:  Merge(udev, cufile),
+	}
+
+	return &d, nil
+}
+
+// Devices discovers the nvidia-fs device nodes for use with GPUDirect Storage
+func (d *gdsDeviceDiscoverer) Devices() ([]Device, error) {
+	return d.devices.Devices()
+}
+
+// Mounts discovers the required mounts for GPUDirect Storage.
+// If no devices are discovered the discovered mounts are empty
+func (d *gdsDeviceDiscoverer) Mounts() ([]Mount, error) {
+	devices, err := d.Devices()
+	if err != nil || len(devices) == 0 {
+		d.logger.Debugf("No nvidia-fs devices detected; skipping detection of mounts")
+		return nil, nil
+	}
+
+	return d.mounts.Mounts()
+}

internal/discover/graphics.go

@@ -0,0 +1,386 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
+)
+
+// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
+func NewGraphicsDiscoverer(logger logger.Interface, devices image.VisibleDevices, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	mounts, err := NewGraphicsMountsDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
+	}
+
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, driverRoot, nvidiaCTKPath)
+
+	discover := Merge(
+		Merge(drmDeviceNodes, drmByPathSymlinks),
+		mounts,
+	)
+
+	return discover, nil
+}
+
+// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
+func NewGraphicsMountsDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct library locator: %v", err)
+	}
+	libraries := NewMounts(
+		logger,
+		locator,
+		driverRoot,
+		[]string{
+			"libnvidia-egl-gbm.so",
+		},
+	)
+
+	jsonMounts := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/etc", "/usr/share"),
+		),
+		driverRoot,
+		[]string{
+			"glvnd/egl_vendor.d/10_nvidia.json",
+			"vulkan/icd.d/nvidia_icd.json",
+			"vulkan/icd.d/nvidia_layers.json",
+			"vulkan/implicit_layer.d/nvidia_layers.json",
+			"egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"egl/egl_external_platform.d/10_nvidia_wayland.json",
+			"nvidia/nvoptix.bin",
+		},
+	)
+
+	xorg := optionalXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+
+	discover := Merge(
+		libraries,
+		jsonMounts,
+		xorg,
+	)
+
+	return discover, nil
+}
+
+type drmDevicesByPath struct {
+	None
+	logger        logger.Interface
+	nvidiaCTKPath string
+	driverRoot    string
+	devicesFrom   Discover
+}
+
+// newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
+func newCreateDRMByPathSymlinks(logger logger.Interface, devices Discover, driverRoot string, nvidiaCTKPath string) Discover {
+	d := drmDevicesByPath{
+		logger:        logger,
+		nvidiaCTKPath: nvidiaCTKPath,
+		driverRoot:    driverRoot,
+		devicesFrom:   devices,
+	}
+
+	return &d
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d drmDevicesByPath) Hooks() ([]Hook, error) {
+	devices, err := d.devicesFrom.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices for by-path symlinks: %v", err)
+	}
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	links, err := d.getSpecificLinkArgs(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []Hook{hook}, nil
+}
+
+// getSpecificLinkArgs returns the required specic links that need to be created
+func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error) {
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[filepath.Base(d.HostPath)] = true
+	}
+
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.driverRoot),
+	)
+	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
+	if err != nil {
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
+	}
+
+	var links []string
+	for _, c := range candidates {
+		device, err := os.Readlink(c)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", c)
+			continue
+		}
+
+		if selectedDevices[filepath.Base(device)] {
+			d.logger.Debugf("adding device symlink %v -> %v", c, device)
+			links = append(links, fmt.Sprintf("%v::%v", device, c))
+		}
+	}
+
+	return links, nil
+}
+
+// newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
+func newDRMDeviceDiscoverer(logger logger.Interface, devices image.VisibleDevices, driverRoot string) (Discover, error) {
+	allDevices := NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+		),
+		driverRoot,
+		[]string{
+			"/dev/dri/card*",
+			"/dev/dri/renderD*",
+		},
+	)
+
+	filter, err := newDRMDeviceFilter(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
+	}
+
+	// We return a discoverer that applies the DRM device filter created above to all discovered DRM device nodes.
+	d := newFilteredDisoverer(
+		logger,
+		allDevices,
+		filter,
+	)
+
+	return d, err
+}
+
+// newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
+func newDRMDeviceFilter(logger logger.Interface, devices image.VisibleDevices, driverRoot string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read GPU information: %v", err)
+	}
+
+	var selectedBusIds []string
+	for _, f := range gpuInformationPaths {
+		info, err := proc.ParseGPUInformationFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %v: %v", f, err)
+		}
+		uuid := info[proc.GPUInfoGPUUUID]
+		busID := info[proc.GPUInfoBusLocation]
+		minor := info[proc.GPUInfoDeviceMinor]
+
+		if devices.Has(minor) || devices.Has(uuid) || devices.Has(busID) {
+			selectedBusIds = append(selectedBusIds, busID)
+		}
+	}
+
+	filter := make(selectDeviceByPath)
+	for _, busID := range selectedBusIds {
+		drmDeviceNodes, err := drm.GetDeviceNodesByBusID(busID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", busID, err)
+		}
+		for _, drmDeviceNode := range drmDeviceNodes {
+			filter[filepath.Join(drmDeviceNode)] = true
+		}
+	}
+
+	return filter, nil
+}
+
+type xorgHooks struct {
+	libraries     Discover
+	driverVersion string
+	nvidiaCTKPath string
+}
+
+var _ Discover = (*xorgHooks)(nil)
+
+// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
+// If the creation of the discoverer fails, a None discoverer is returned.
+func optionalXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		logger.Warningf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
+		return None{}
+	}
+	return xorg
+}
+
+func newXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate(".*.*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+	libcudaPath := libCudaPaths[0]
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	libRoot := filepath.Dir(libcudaPath)
+	xorgLibs := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"nvidia/xorg/nvidia_drv.so",
+			fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
+		},
+	)
+	xorgHooks := xorgHooks{
+		libraries:     xorgLibs,
+		driverVersion: version,
+		nvidiaCTKPath: nvidiaCTKPath,
+	}
+
+	xorgConfg := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/usr/share"),
+		),
+		driverRoot,
+		[]string{"X11/xorg.conf.d/10-nvidia.conf"},
+	)
+
+	d := Merge(
+		xorgLibs,
+		xorgConfg,
+		xorgHooks,
+	)
+
+	return d, nil
+}
+
+// Devices returns no devices for Xorg
+func (m xorgHooks) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a hook to create symlinks for Xorg libraries
+func (m xorgHooks) Hooks() ([]Hook, error) {
+	mounts, err := m.libraries.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get mounts: %v", err)
+	}
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+
+	var target string
+	for _, mount := range mounts {
+		filename := filepath.Base(mount.HostPath)
+		if filename == "libglxserver_nvidia.so."+m.driverVersion {
+			target = mount.Path
+		}
+	}
+
+	if target == "" {
+		return nil, nil
+	}
+
+	link := strings.TrimSuffix(target, "."+m.driverVersion)
+	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
+	symlinkHook := CreateCreateSymlinkHook(
+		m.nvidiaCTKPath,
+		links,
+	)
+
+	return symlinkHook.Hooks()
+}
+
+// Mounts returns the libraries required for Xorg
+func (m xorgHooks) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// selectDeviceByPath is a filter that allows devices to be selected by the path
+type selectDeviceByPath map[string]bool
+
+var _ Filter = (*selectDeviceByPath)(nil)
+
+// DeviceIsSelected determines whether the device's path has been selected
+func (s selectDeviceByPath) DeviceIsSelected(device Device) bool {
+	return s[device.Path]
+}
+
+// MountIsSelected is always true
+func (s selectDeviceByPath) MountIsSelected(Mount) bool {
+	return true
+}
+
+// HookIsSelected is always true
+func (s selectDeviceByPath) HookIsSelected(Hook) bool {
+	return true
+}

internal/discover/hooks.go

@@ -0,0 +1,67 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+)
+
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+// CreateCreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateCreateSymlinkHook(nvidiaCTKPath string, links []string) Discover {
+	if len(links) == 0 {
+		return None{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateNvidiaCTKHook(
+		nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+}
+
+// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
+func CreateNvidiaCTKHook(nvidiaCTKPath string, hookName string, additionalArgs ...string) Hook {
+	return Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      nvidiaCTKPath,
+		Args:      append([]string{filepath.Base(nvidiaCTKPath), "hook", hookName}, additionalArgs...),
+	}
+}

internal/discover/icp_test.go

@@ -0,0 +1,61 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIPCMounts(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	l := ipcMounts(
+		mounts{
+			logger: logger,
+			lookup: &lookup.LocatorMock{
+				LocateFunc: func(path string) ([]string, error) {
+					return []string{"/host/path"}, nil
+				},
+			},
+			required: []string{"target"},
+		},
+	)
+
+	mounts, err := l.Mounts()
+	require.NoError(t, err)
+
+	require.EqualValues(
+		t,
+		[]Mount{
+			{
+				HostPath: "/host/path",
+				Path:     "/host/path",
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+					"noexec",
+				},
+			},
+		},
+		mounts,
+	)
+}

internal/discover/ipc.go

@@ -0,0 +1,78 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type ipcMounts mounts
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger logger.Interface, driverRoot string) (Discover, error) {
+	sockets := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/run", "/var/run"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/nvidia-persistenced/socket",
+			"/nvidia-fabricmanager/socket",
+		},
+	)
+
+	mps := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	d := Merge(
+		(*ipcMounts)(sockets),
+		(*ipcMounts)(mps),
+	)
+	return d, nil
+}
+
+// Mounts returns the discovered mounts with "noexec" added to the mount options.
+func (d *ipcMounts) Mounts() ([]Mount, error) {
+	mounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+
+	var modifiedMounts []Mount
+	for _, m := range mounts {
+		mount := m
+		mount.Options = append(m.Options, "noexec")
+		modifiedMounts = append(modifiedMounts, mount)
+	}
+
+	return modifiedMounts, nil
+}

internal/discover/ldconfig.go

@@ -19,36 +19,27 @@ package discover
 import (
 	"fmt"
 	"path/filepath"
-	"sort"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
 // NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
-func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
+func NewLDCacheUpdateHook(logger logger.Interface, mounts Discover, nvidiaCTKPath string) (Discover, error) {
 	d := ldconfig{
-		logger:                  logger,
-		mountsFrom:              mounts,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+		logger:        logger,
+		nvidiaCTKPath: nvidiaCTKPath,
+		mountsFrom:    mounts,
 	}
 
 	return &d, nil
 }
 
-const (
-	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
-)
-
 type ldconfig struct {
 	None
-	logger                  *logrus.Logger
-	mountsFrom              Discover
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
+	logger        logger.Interface
+	nvidiaCTKPath string
+	mountsFrom    Discover
 }
 
 // Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
@@ -57,70 +48,75 @@ func (d ldconfig) Hooks() ([]Hook, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
 	}
-
-	libDirs := getLibDirs(mounts)
-
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{hookPath, "hook", "update-ldcache"}
-	for _, f := range libDirs {
-		args = append(args, "--folder", f)
-	}
-	h := Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
-
+	h := CreateLDCacheUpdateHook(
+		d.nvidiaCTKPath,
+		getLibraryPaths(mounts),
+	)
 	return []Hook{h}, nil
 }
 
-// getLibDirs extracts the library dirs from the specified mounts
-func getLibDirs(mounts []Mount) []string {
-	var paths []string
-	checked := make(map[string]bool)
-
-	for _, m := range mounts {
-		dir := filepath.Dir(m.Path)
-		if dir == "" {
-			continue
-		}
-
-		_, exists := checked[dir]
-		if exists {
-			continue
-		}
-		checked[dir] = isLibName(filepath.Base(m.Path))
-
-		if checked[dir] {
-			paths = append(paths, dir)
-		}
+// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
+func CreateLDCacheUpdateHook(executable string, libraries []string) Hook {
+	var args []string
+	for _, f := range uniqueFolders(libraries) {
+		args = append(args, "--folder", f)
 	}
 
-	sort.Strings(paths)
+	hook := CreateNvidiaCTKHook(
+		executable,
+		"update-ldcache",
+		args...,
+	)
 
+	return hook
+
+}
+
+// getLibraryPaths extracts the library dirs from the specified mounts
+func getLibraryPaths(mounts []Mount) []string {
+	var paths []string
+	for _, m := range mounts {
+		if !isLibName(m.Path) {
+			continue
+		}
+		paths = append(paths, m.Path)
+	}
 	return paths
 }
 
 // isLibName checks if the specified filename is a library (i.e. ends in `.so*`)
 func isLibName(filename string) bool {
-	parts := strings.Split(filename, ".")
 
-	for _, p := range parts {
-		if p == "so" {
-			return true
-		}
+	base := filepath.Base(filename)
+
+	isLib, err := filepath.Match("lib?*.so*", base)
+	if !isLib || err != nil {
+		return false
 	}
 
-	return false
+	parts := strings.Split(base, ".so")
+	if len(parts) == 1 {
+		return true
+	}
+
+	return parts[len(parts)-1] == "" || strings.HasPrefix(parts[len(parts)-1], ".")
+}
+
+// uniqueFolders returns the unique set of folders for the specified files
+func uniqueFolders(libraries []string) []string {
+	var paths []string
+	checked := make(map[string]bool)
+
+	for _, l := range libraries {
+		dir := filepath.Dir(l)
+		if dir == "" {
+			continue
+		}
+		if checked[dir] {
+			continue
+		}
+		checked[dir] = true
+		paths = append(paths, dir)
+	}
+	return paths
 }

internal/discover/ldconfig_test.go

@@ -0,0 +1,163 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testNvidiaCTKPath = "/foo/bar/nvidia-ctk"
+)
+
+func TestLDCacheUpdateHook(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		mounts        []Mount
+		mountError    error
+		expectedError error
+		expectedArgs  []string
+	}{
+		{
+			description:  "empty mounts",
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache"},
+		},
+		{
+			description:   "mount error",
+			mountError:    fmt.Errorf("mountError"),
+			expectedError: fmt.Errorf("mountError"),
+		},
+		{
+			description: "library folders are added to args",
+			mounts: []Mount{
+				{
+					Path: "/usr/local/lib/libfoo.so",
+				},
+				{
+					Path: "/usr/bin/notlib",
+				},
+				{
+					Path: "/usr/local/libother/libfoo.so",
+				},
+				{
+					Path: "/usr/local/lib/libbar.so",
+				},
+			},
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
+		},
+		{
+			description: "host paths are ignored",
+			mounts: []Mount{
+				{
+					HostPath: "/usr/local/other/libfoo.so",
+					Path:     "/usr/local/lib/libfoo.so",
+				},
+			},
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache", "--folder", "/usr/local/lib"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mountMock := &DiscoverMock{
+				MountsFunc: func() ([]Mount, error) {
+					return tc.mounts, tc.mountError
+				},
+			}
+			expectedHook := Hook{
+				Path:      testNvidiaCTKPath,
+				Args:      tc.expectedArgs,
+				Lifecycle: "createContainer",
+			}
+
+			d, err := NewLDCacheUpdateHook(logger, mountMock, testNvidiaCTKPath)
+			require.NoError(t, err)
+
+			hooks, err := d.Hooks()
+			require.Len(t, mountMock.MountsCalls(), 1)
+			require.Len(t, mountMock.DevicesCalls(), 0)
+			require.Len(t, mountMock.HooksCalls(), 0)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Len(t, hooks, 1)
+
+			require.EqualValues(t, hooks[0], expectedHook)
+
+			devices, err := d.Devices()
+			require.NoError(t, err)
+			require.Empty(t, devices)
+
+			mounts, err := d.Mounts()
+			require.NoError(t, err)
+			require.Empty(t, mounts)
+
+		})
+	}
+
+}
+
+func TestIsLibName(t *testing.T) {
+	testCases := []struct {
+		name  string
+		isLib bool
+	}{
+		{
+			name:  "",
+			isLib: false,
+		},
+		{
+			name:  "lib/not/.so",
+			isLib: false,
+		},
+		{
+			name:  "lib.so",
+			isLib: false,
+		},
+		{
+			name:  "notlibcuda.so",
+			isLib: false,
+		},
+		{
+			name:  "libcuda.so",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.so.1",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.soNOT",
+			isLib: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.isLib, isLibName(tc.name))
+		})
+	}
+}

internal/discover/list.go

@@ -27,8 +27,8 @@ type list struct {
 
 var _ Discover = (*list)(nil)
 
-// NewList creates a discoverer that is the composite of a list of discoveres.
-func NewList(d ...Discover) Discover {
+// Merge creates a discoverer that is the composite of a list of discoveres.
+func Merge(d ...Discover) Discover {
 	l := list{
 		discoverers: d,
 	}

internal/discover/mofed.go

@@ -0,0 +1,33 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// NewMOFEDDiscoverer creates a discoverer for MOFED devices.
+func NewMOFEDDiscoverer(logger logger.Interface, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{
+			"/dev/infiniband/uverbs*",
+			"/dev/infiniband/rdma_cm",
+		},
+		root,
+	)
+
+	return devices, nil
+}

internal/discover/mounts.go

@@ -18,10 +18,12 @@ package discover
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 	"sync"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
 )
 
 // mounts is a generic discoverer for Mounts. It is customized by specifying the
@@ -29,8 +31,9 @@ import (
 // based on the entry in the list.
 type mounts struct {
 	None
-	logger   *logrus.Logger
+	logger   logger.Interface
 	lookup   lookup.Locator
+	root     string
 	required []string
 	sync.Mutex
 	cache []Mount
@@ -38,6 +41,21 @@ type mounts struct {
 
 var _ Discover = (*mounts)(nil)
 
+// NewMounts creates a discoverer for the required mounts using the specified locator.
+func NewMounts(logger logger.Interface, lookup lookup.Locator, root string, required []string) Discover {
+	return newMounts(logger, lookup, root, required)
+}
+
+// newMounts creates a discoverer for the required mounts using the specified locator.
+func newMounts(logger logger.Interface, lookup lookup.Locator, root string, required []string) *mounts {
+	return &mounts{
+		logger:   logger,
+		lookup:   lookup,
+		root:     filepath.Join("/", root),
+		required: required,
+	}
+}
+
 func (d *mounts) Mounts() ([]Mount, error) {
 	if d.lookup == nil {
 		return nil, fmt.Errorf("no lookup defined")
@@ -51,35 +69,60 @@ func (d *mounts) Mounts() ([]Mount, error) {
 	d.Lock()
 	defer d.Unlock()
 
-	paths := make(map[string]bool)
+	uniqueMounts := make(map[string]Mount)
 
 	for _, candidate := range d.required {
 		d.logger.Debugf("Locating %v", candidate)
 		located, err := d.lookup.Locate(candidate)
 		if err != nil {
-			d.logger.Warnf("Could not locate %v: %v", candidate, err)
+			d.logger.Warningf("Could not locate %v: %v", candidate, err)
 			continue
 		}
 		if len(located) == 0 {
-			d.logger.Warnf("Missing %v", candidate)
+			d.logger.Warningf("Missing %v", candidate)
 			continue
 		}
 		d.logger.Debugf("Located %v as %v", candidate, located)
 		for _, p := range located {
-			paths[p] = true
+			if _, ok := uniqueMounts[p]; ok {
+				d.logger.Debugf("Skipping duplicate mount %v", p)
+				continue
+			}
+
+			r := d.relativeTo(p)
+			if r == "" {
+				r = p
+			}
+
+			d.logger.Infof("Selecting %v as %v", p, r)
+			uniqueMounts[p] = Mount{
+				HostPath: p,
+				Path:     r,
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+				},
+			}
 		}
 	}
 
 	var mounts []Mount
-	for path := range paths {
-		d.logger.Infof("Selecting %v", path)
-		mount := Mount{
-			Path: path,
-		}
-		mounts = append(mounts, mount)
+	for _, m := range uniqueMounts {
+		mounts = append(mounts, m)
 	}
 
 	d.cache = mounts
 
-	return mounts, nil
+	return d.cache, nil
+}
+
+// relativeTo returns the path relative to the root for the file locator
+func (d *mounts) relativeTo(path string) string {
+	if d.root == "/" {
+		return path
+	}
+
+	return strings.TrimPrefix(path, d.root)
 }