Merge branch 'skip-for-point-release' into 'main'

Skip components for patch releases

See merge request nvidia/container-toolkit/container-toolkit!374

.common-ci.yml

@@ -23,6 +23,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -34,13 +35,44 @@ stages:
   - scan
   - release
 
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
 .main-or-manual:
   rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+    - !reference [.pipeline-trigger-rules, rules]
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: manual
 
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
+  rules:
+    - !reference [.main-or-manual, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
+
 # Define the distribution targets
 .dist-amazonlinux2:
   rules:
@@ -70,20 +102,6 @@ stages:
     DIST: debian10
     PACKAGE_REPO_TYPE: debian
 
-.dist-debian9:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: debian9
-    PACKAGE_REPO_TYPE: debian
-
-.dist-fedora35:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: fedora35
-    PACKAGE_REPO_TYPE: rpm
-
 .dist-opensuse-leap15.1:
   rules:
     - !reference [.main-or-manual, rules]
@@ -99,13 +117,6 @@ stages:
     CVE_UPDATES: "cyrus-sasl-lib"
     PACKAGE_REPO_TYPE: rpm
 
-.dist-ubuntu16.04:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: ubuntu16.04
-    PACKAGE_REPO_TYPE: debian
-
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
@@ -113,8 +124,6 @@ stages:
     PACKAGE_REPO_TYPE: debian
 
 .dist-ubuntu20.04:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubuntu20.04
     CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
@@ -259,22 +268,15 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
 release:staging-ubuntu20.04:
   extends:
     - .release:staging
     - .dist-ubuntu20.04
   needs:
-    - image-ubuntu20.04
+    - test-toolkit-ubuntu20.04
+    - test-containerd-ubuntu20.04
+    - test-crio-ubuntu20.04
+    - test-docker-ubuntu20.04
 
 release:staging-packaging:
   extends:

.github/workflows/blossom-ci.yaml

@@ -0,0 +1,113 @@
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to trigger ci on hybrid infra (github + self hosted runner)
+name: Blossom-CI
+on:
+  issue_comment:
+    types: [created]
+  workflow_dispatch:
+      inputs:
+          platform:
+            description: 'runs-on argument'     
+            required: false
+          args:
+            description: 'argument'     
+            required: false
+jobs:
+  Authorization:
+    name: Authorization
+    runs-on: blossom 
+    outputs:
+      args: ${{ env.args }}
+     
+    # This job only runs for pull request comments
+    if: |
+         contains( '\
+         anstockatnv,\
+         rohitrajani2018,\
+         cdesiniotis,\
+         shivamerla,\
+         ArangoGutierrez,\
+         elezar,\
+         klueska,\
+         zvonkok,\
+         ', format('{0},', github.actor)) && 
+         github.event.comment.body == '/blossom-ci'  
+    steps:
+      - name: Check if comment is issued by authorized person
+        run: blossom-ci
+        env:
+          OPERATION: 'AUTH'
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+      
+  Vulnerability-scan:
+    name: Vulnerability scan
+    needs: [Authorization]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
+          ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
+          lfs: 'true'
+      
+      # repo specific steps 
+      #- name: Setup java
+      #  uses: actions/setup-java@v1
+      #  with:
+      #    java-version: 1.8
+      
+      # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file
+      #- name: Setup blackduck properties
+      #  run: |
+      #       PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g')
+      #       echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties
+      #       echo detect.maven.included.scopes=compile >> application.properties
+          
+      - name: Run blossom action
+        uses: NVIDIA/blossom-action@main
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+        with:
+          args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }}
+          args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }}
+          args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }}
+          
+  Job-trigger:
+    name: Start ci job
+    needs: [Vulnerability-scan]
+    runs-on: blossom
+    steps:
+      - name: Start ci job
+        run: blossom-ci
+        env:
+          OPERATION: 'START-CI-JOB'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              
+  Upload-Log:
+    name: Upload log
+    runs-on: blossom
+    if : github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here)
+        run: blossom-ci
+        env:
+          OPERATION: 'POST-PROCESSING'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,9 +1,11 @@
 dist
+artifacts
 *.swp
 *.swo
 /coverage.out*
 /test/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
 /nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk

.gitlab-ci.yml

@@ -116,12 +116,6 @@ package-amazonlinux2-x86_64:
     - .dist-amazonlinux2
     - .arch-x86_64
 
-package-centos7-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-ppc64le
-
 package-centos7-x86_64:
   extends:
     - .package-build
@@ -152,42 +146,12 @@ package-debian10-amd64:
     - .dist-debian10
     - .arch-amd64
 
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-fedora35-aarch64:
-  extends:
-    - .package-build
-    - .dist-fedora35
-    - .arch-aarch64
-
-package-fedora35-x86_64:
-  extends:
-    - .package-build
-    - .dist-fedora35
-    - .arch-x86_64
-
 package-opensuse-leap15.1-x86_64:
   extends:
     - .package-build
     - .dist-opensuse-leap15.1
     - .arch-x86_64
 
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -228,7 +192,7 @@ package-ubuntu18.04-ppc64le:
   before_script:
     - !reference [.buildx-setup, before_script]
 
-    - apk add --no-cache bash make
+    - apk add --no-cache bash make git
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
@@ -240,7 +204,6 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
-    - package-centos7-ppc64le
     - package-centos7-x86_64
 
 image-ubi8:
@@ -254,17 +217,6 @@ image-ubi8:
     - package-centos8-x86_64
     - package-centos8-ppc64le
 
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
-
 image-ubuntu20.04:
   extends:
     - .image-build
@@ -273,7 +225,8 @@ image-ubuntu20.04:
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -290,26 +243,14 @@ image-packaging:
       optional: true
     - job: package-amazonlinux2-x86_64
       optional: true
-    - job: package-centos7-ppc64le
-      optional: true
     - job: package-centos7-x86_64
       optional: true
     - job: package-centos8-ppc64le
       optional: true
     - job: package-debian10-amd64
       optional: true
-    - job: package-debian9-amd64
-      optional: true
-    - job: package-fedora35-aarch64
-      optional: true
-    - job: package-fedora35-x86_64
-      optional: true
     - job: package-opensuse-leap15.1-x86_64
       optional: true
-    - job: package-ubuntu16.04-amd64
-      optional: true
-    - job: package-ubuntu16.04-ppc64le
-      optional: true
     - job: package-ubuntu18.04-ppc64le
       optional: true
 
@@ -343,31 +284,31 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu18.04:
+test-toolkit-ubuntu20.04:
   extends:
     - .test:toolkit
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
-test-containerd-ubuntu18.04:
+test-containerd-ubuntu20.04:
   extends:
     - .test:containerd
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
-test-crio-ubuntu18.04:
+test-crio-ubuntu20.04:
   extends:
     - .test:crio
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
-test-docker-ubuntu18.04:
+test-docker-ubuntu20.04:
   extends:
     - .test:docker
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 

.nvidia-ci.yml

@@ -36,8 +36,7 @@ variables:
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
   ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
-  # TODO: We should set the kitmaker release folder here once we have the end-to-end workflow set up
-  KITMAKER_RELEASE_FOLDER: "testing"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
 
 .image-pull:
   stage: image-build
@@ -79,11 +78,6 @@ image-ubi8:
     - .dist-ubi8
     - .image-pull
 
-image-ubuntu18.04:
-  extends:
-    - .dist-ubuntu18.04
-    - .image-pull
-
 image-ubuntu20.04:
   extends:
     - .dist-ubuntu20.04
@@ -95,109 +89,6 @@ image-packaging:
     - .dist-packaging
     - .image-pull
 
-# Define the package release targets
-release:packages:amazonlinux2-aarch64:
-  extends:
-    - .release:packages
-    - .dist-amazonlinux2
-    - .arch-aarch64
-
-release:packages:amazonlinux2-x86_64:
-  extends:
-    - .release:packages
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-release:packages:centos7-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-centos7
-    - .arch-ppc64le
-
-release:packages:centos7-x86_64:
-  extends:
-    - .release:packages
-    - .dist-centos7
-    - .arch-x86_64
-
-release:packages:centos8-aarch64:
-  extends:
-    - .release:packages
-    - .dist-centos8
-    - .arch-aarch64
-
-release:packages:centos8-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-centos8
-    - .arch-ppc64le
-
-release:packages:centos8-x86_64:
-  extends:
-    - .release:packages
-    - .dist-centos8
-    - .arch-x86_64
-
-release:packages:debian10-amd64:
-  extends:
-    - .release:packages
-    - .dist-debian10
-    - .arch-amd64
-
-release:packages:debian9-amd64:
-  extends:
-    - .release:packages
-    - .dist-debian9
-    - .arch-amd64
-
-release:packages:fedora35-aarch64:
-  extends:
-    - .release:packages
-    - .dist-fedora35
-    - .arch-aarch64
-
-release:packages:fedora35-x86_64:
-  extends:
-    - .release:packages
-    - .dist-fedora35
-    - .arch-x86_64
-
-release:packages:opensuse-leap15.1-x86_64:
-  extends:
-    - .release:packages
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-release:packages:ubuntu16.04-amd64:
-  extends:
-    - .release:packages
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-release:packages:ubuntu16.04-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
-release:packages:ubuntu18.04-amd64:
-  extends:
-    - .release:packages
-    - .dist-ubuntu18.04
-    - .arch-amd64
-
-release:packages:ubuntu18.04-arm64:
-  extends:
-    - .release:packages
-    - .dist-ubuntu18.04
-    - .arch-arm64
-
-release:packages:ubuntu18.04-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-ubuntu18.04
-    - .arch-ppc64le
-
 # We skip the integration tests for the internal CI:
 .integration:
   stage: test
@@ -213,7 +104,7 @@ release:packages:ubuntu18.04-ppc64le:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
+    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
   rules:
     - if: $SKIP_SCANS != "yes"
     - when: manual
@@ -256,14 +147,6 @@ scan-centos7-arm64:
     - image-centos7
     - scan-centos7-amd64
 
-scan-ubuntu18.04-amd64:
-  extends:
-    - .dist-ubuntu18.04
-    - .platform-amd64
-    - .scan
-  needs:
-    - image-ubuntu18.04
-
 scan-ubuntu20.04-amd64:
   extends:
     - .dist-ubuntu20.04
@@ -319,22 +202,26 @@ scan-ubi8-arm64:
     PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
     PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
-    PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local"
     KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
   script:
     - !reference [.regctl-setup, before_script]
-    - apk add --no-cache bash
+    - apk add --no-cache bash git
     - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
-    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
     # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
-    - ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
 
-release:staging-ubuntu18.04:
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
+
+release:staging-ubuntu20.04:
   extends:
     - .release:staging
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
 # Define the external release targets
 # Release to NGC
@@ -343,11 +230,6 @@ release:ngc-centos7:
     - .dist-centos7
     - .release:ngc
 
-release:ngc-ubuntu18.04:
-  extends:
-    - .dist-ubuntu18.04
-    - .release:ngc
-
 release:ngc-ubuntu20.04:
   extends:
     - .dist-ubuntu20.04

CHANGELOG.md

@@ -1,5 +1,104 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
 ## v1.12.0-rc.2
 
 * Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime

Makefile

@@ -61,7 +61,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-extldflags=-Wl,-z,lazy -s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...

build/container/Dockerfile.packaging

@@ -15,17 +15,27 @@
 ARG BASE_DIST
 ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
-ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
-
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
+
+# build-args are added to the manifest.txt file below.
+ARG BASE_DIST
+ARG PACKAGE_DIST
+ARG PACKAGE_VERSION
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
+ARG SOURCE_DATE_EPOCH
+ARG VERSION
+
 # Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
-RUN find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' > /artifacts/manifest.txt
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
 
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

build/container/Makefile

@@ -44,13 +44,13 @@ OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
 DEFAULT_PUSH_TARGET := ubuntu20.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7
+DISTRIBUTIONS := ubuntu20.04 ubi8 centos7
 
 META_TARGETS := packaging
 
 BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
 PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
 
 .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -94,6 +94,10 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
+		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
+		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
+		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
 		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
@@ -102,24 +106,20 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
 build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
 
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
 build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-centos7: BASE_DIST = $(*)
 build-centos7: DOCKERFILE_SUFFIX := centos
 build-centos7: PACKAGE_DIST = $(BASE_DIST)
-build-centos7: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
@@ -143,10 +143,7 @@ test-packaging:
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

cmd/nvidia-container-runtime-hook/container_config.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
 )
 
@@ -130,7 +131,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -139,28 +140,31 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		for _, c := range caps {
+			if c == capSysAdmin {
+				return true
+			}
 		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
+		return false
 	}
 
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	process := specs.Process{
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
 func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -43,8 +43,9 @@ type HookConfig struct {
 	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
 	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
+	NvidiaContainerCLI         CLIConfig                `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime     config.RuntimeConfig     `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHook config.RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
 }
 
 func getDefaultHookConfig() HookConfig {
@@ -66,7 +67,8 @@ func getDefaultHookConfig() HookConfig {
 			User:        nil,
 			Ldconfig:    nil,
 		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntime:     *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntimeHook: *config.GetDefaultRuntimeHookConfig(),
 	}
 }
 

cmd/nvidia-container-runtime-hook/main.go

@@ -74,7 +74,7 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
+	if !hook.NVIDIAContainerRuntimeHook.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
 		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
 	}
 

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/main.go

@@ -1,89 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer func() {
-		if rerr != nil {
-			logger.Errorf("%v", rerr)
-		}
-		logger.Reset()
-	}()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,7 +42,7 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		logrus.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
@@ -53,11 +54,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		logrus.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		logrus.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,7 +78,7 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -91,9 +92,10 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -170,7 +172,7 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	m := modifier.NewStableRuntimeModifier(logrus.StandardLogger())
 	return m.Modify(spec)
 }
 

cmd/nvidia-ctk/cdi/cdi.go

@@ -18,6 +18,7 @@ package cdi
 
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -44,6 +45,7 @@ func (m command) build() *cli.Command {
 
 	hook.Subcommands = []*cli.Command{
 		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -18,27 +18,22 @@ package generate
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-	"sigs.k8s.io/yaml"
 )
 
 const (
-	nvidiaCTKExecutable      = "nvidia-ctk"
-	nvidiaCTKDefaultFilePath = "/usr/bin/" + nvidiaCTKExecutable
+	allDeviceName = "all"
 )
 
 type command struct {
@@ -46,8 +41,14 @@ type command struct {
 }
 
 type config struct {
-	output   string
-	jsonMode bool
+	output             string
+	format             string
+	deviceNameStrategy string
+	driverRoot         string
+	nvidiaCTKPath      string
+	mode               string
+	vendor             string
+	class              string
 }
 
 // NewCommand constructs a generate-cdi command with the specified logger
@@ -66,6 +67,9 @@ func (m command) build() *cli.Command {
 	c := cli.Command{
 		Name:  "generate",
 		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
 		Action: func(c *cli.Context) error {
 			return m.run(c, &cfg)
 		},
@@ -74,486 +78,206 @@ func (m command) build() *cli.Command {
 	c.Flags = []cli.Flag{
 		&cli.StringFlag{
 			Name:        "output",
-			Usage:       "Specify the file to output the generated CDI specification to. If this is '-' or '' the specification is output to STDOUT",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
 			Destination: &cfg.output,
 		},
-		&cli.BoolFlag{
-			Name:        "json",
-			Usage:       "Output the generated CDI spec in JSON mode instead of YAML",
-			Destination: &cfg.jsonMode,
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       spec.FormatYAML,
+			Destination: &cfg.format,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Aliases:     []string{"discovery-mode"},
+			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       nvcdi.ModeAuto,
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
+			Value:       nvcdi.DeviceNameStrategyIndex,
+			Destination: &cfg.deviceNameStrategy,
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &cfg.driverRoot,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-ctk-path",
+			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
+			Destination: &cfg.nvidiaCTKPath,
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &cfg.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &cfg.class,
 		},
 	}
 
 	return &c
 }
 
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+
+	cfg.format = strings.ToLower(cfg.format)
+	switch cfg.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", cfg.format)
+	}
+
+	cfg.mode = strings.ToLower(cfg.mode)
+	switch cfg.mode {
+	case nvcdi.ModeAuto:
+	case nvcdi.ModeNvml:
+	case nvcdi.ModeWsl:
+	case nvcdi.ModeManagement:
+	default:
+		return fmt.Errorf("invalid discovery mode: %v", cfg.mode)
+	}
+
+	_, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
+	if err != nil {
+		return err
+	}
+
+	cfg.nvidiaCTKPath = discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath)
+
+	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			cfg.format = outputFileFormat
+		} else if outputFileFormat != cfg.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", cfg.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(cfg.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(cfg.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	return nil
+}
+
 func (m command) run(c *cli.Context, cfg *config) error {
-	spec, err := m.generateSpec()
+	spec, err := m.generateSpec(cfg)
 	if err != nil {
 		return fmt.Errorf("failed to generate CDI spec: %v", err)
 	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
 
-	var outputTo io.Writer
-	if cfg.output == "" || cfg.output == "-" {
-		outputTo = os.Stdout
-	} else {
-		outputFile, err := os.Create(cfg.output)
+	if cfg.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
 		if err != nil {
-			return fmt.Errorf("failed to create output file: %v", err)
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
 		}
-		defer outputFile.Close()
-		outputTo = outputFile
-	}
-
-	if filepath.Ext(cfg.output) == ".json" {
-		cfg.jsonMode = true
-	} else if filepath.Ext(cfg.output) == ".yaml" || filepath.Ext(cfg.output) == ".yml" {
-		cfg.jsonMode = false
-	}
-
-	data, err := yaml.Marshal(spec)
-	if err != nil {
-		return fmt.Errorf("failed to marshal CDI spec: %v", err)
-	}
-
-	if cfg.jsonMode {
-		data, err = yaml.YAMLToJSONStrict(data)
-		if err != nil {
-			return fmt.Errorf("failed to convert CDI spec from YAML to JSON: %v", err)
-		}
-	}
-
-	err = writeToOutput(cfg.jsonMode, data, outputTo)
-	if err != nil {
-		return fmt.Errorf("failed to write output: %v", err)
-	}
-
-	return nil
-}
-
-func writeToOutput(jsonMode bool, data []byte, output io.Writer) error {
-	if !jsonMode {
-		_, err := output.Write([]byte("---\n"))
-		if err != nil {
-			return fmt.Errorf("failed to write YAML separator: %v", err)
-		}
-
-	}
-	_, err := output.Write(data)
-	if err != nil {
-		return fmt.Errorf("failed to write data: %v", err)
-	}
-
-	return nil
-}
-
-func (m command) generateSpec() (*specs.Spec, error) {
-	nvmllib := nvml.New()
-	if r := nvmllib.Init(); r != nvml.SUCCESS {
-		return nil, r
-	}
-	defer nvmllib.Shutdown()
-
-	devicelib := device.New(device.WithNvml(nvmllib))
-
-	spec := specs.Spec{
-		Version:        "0.4.0",
-		Kind:           "nvidia.com/gpu",
-		ContainerEdits: specs.ContainerEdits{},
-	}
-	err := devicelib.VisitDevices(func(i int, d device.Device) error {
-		isMig, err := d.IsMigEnabled()
-		if err != nil {
-			return fmt.Errorf("failed to check whether device is MIG device: %v", err)
-		}
-		if isMig {
-			return nil
-		}
-		device, err := generateEditsForDevice(newGPUDevice(i, d))
-		if err != nil {
-			return fmt.Errorf("failed to generate CDI spec for device %v: %v", i, err)
-		}
-
-		graphicsEdits, err := m.editsForGraphicsDevice(d)
-		if err != nil {
-			return fmt.Errorf("failed to generate CDI spec for DRM devices associated with device %v: %v", i, err)
-		}
-
-		// We add the device nodes and hooks edits for the DRM devices; Mounts are added globally
-		for _, dn := range graphicsEdits.DeviceNodes {
-			device.ContainerEdits.DeviceNodes = append(device.ContainerEdits.DeviceNodes, dn)
-		}
-		for _, h := range graphicsEdits.Hooks {
-			device.ContainerEdits.Hooks = append(device.ContainerEdits.Hooks, h)
-		}
-
-		spec.Devices = append(spec.Devices, device)
 		return nil
-	})
+	}
+
+	return spec.Save(cfg.output)
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
+	}
+
+	return ""
+}
+
+func (m command) generateSpec(cfg *config) (spec.Interface, error) {
+	deviceNamer, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
 	if err != nil {
-		return nil, fmt.Errorf("failed to generate CDI spec for GPU devices: %v", err)
+		return nil, fmt.Errorf("failed to create device namer: %v", err)
 	}
 
-	err = devicelib.VisitMigDevices(func(i int, d device.Device, j int, m device.MigDevice) error {
-		device, err := generateEditsForDevice(newMigDevice(i, j, m))
-		if err != nil {
-			return fmt.Errorf("failed to generate CDI spec for device %v: %v", i, err)
-		}
-
-		spec.Devices = append(spec.Devices, device)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("falied to generate CDI spec for MIG devices: %v", err)
-	}
-
-	// We create an "all" device with all the discovered device nodes
-	var allDeviceNodes []*specs.DeviceNode
-	for _, d := range spec.Devices {
-		for _, dn := range d.ContainerEdits.DeviceNodes {
-			allDeviceNodes = append(allDeviceNodes, dn)
-		}
-	}
-	all := specs.Device{
-		Name: "all",
-		ContainerEdits: specs.ContainerEdits{
-			DeviceNodes: allDeviceNodes,
-		},
-	}
-
-	spec.Devices = append(spec.Devices, all)
-	spec.ContainerEdits.DeviceNodes = m.getExistingMetaDeviceNodes()
-
-	libraries, err := m.findLibs(nvmllib)
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate driver libraries: %v", err)
-	}
-
-	binaries, err := m.findBinaries()
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate driver binaries: %v", err)
-	}
-
-	ipcs, err := m.findIPC()
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate driver IPC sockets: %v", err)
-	}
-
-	graphicsEdits, err := m.editsForGraphicsDevice(nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generated edits for graphics libraries")
-	}
-
-	libOptions := []string{
-		"ro",
-		"nosuid",
-		"nodev",
-		"bind",
-	}
-	ipcOptions := append(libOptions, "noexec")
-
-	spec.ContainerEdits.Mounts = append(
-		generateMountsForPaths(libOptions, libraries, binaries),
-		generateMountsForPaths(ipcOptions, ipcs)...,
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(cfg.driverRoot),
+		nvcdi.WithNVIDIACTKPath(cfg.nvidiaCTKPath),
+		nvcdi.WithDeviceNamer(deviceNamer),
+		nvcdi.WithMode(string(cfg.mode)),
 	)
-
-	spec.ContainerEdits.Mounts = append(spec.ContainerEdits.Mounts, graphicsEdits.Mounts...)
-
-	ldcacheUpdateHook := m.generateUpdateLdCacheHook(libraries)
-
-	deviceFolderPermissionHooks, err := m.generateDeviceFolderPermissionHooks(ldcacheUpdateHook.Path, allDeviceNodes)
 	if err != nil {
-		return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
 	}
 
-	spec.ContainerEdits.Hooks = append([]*specs.Hook{ldcacheUpdateHook}, deviceFolderPermissionHooks...)
-
-	return &spec, nil
-}
-
-func generateEditsForDevice(name string, d deviceInfo) (specs.Device, error) {
-	deviceNodePaths, err := d.GetDeviceNodes()
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
 	if err != nil {
-		return specs.Device{}, fmt.Errorf("failed to get paths for device: %v", err)
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
 	}
-
-	deviceNodes := getDeviceNodesFromPaths(deviceNodePaths)
-
-	device := specs.Device{
-		Name: name,
-		ContainerEdits: specs.ContainerEdits{
-			DeviceNodes: deviceNodes,
-		},
-	}
-
-	return device, nil
-}
-
-func (m command) editsForGraphicsDevice(device device.Device) (*specs.ContainerEdits, error) {
-	selectedDevice := image.NewVisibleDevices("none")
-	if device != nil {
-		uuid, ret := device.GetUUID()
-		if ret != nvml.SUCCESS {
-			return nil, fmt.Errorf("error getting device UUID: %v", ret)
-		}
-		selectedDevice = image.NewVisibleDevices(uuid)
-	}
-	cfg := discover.Config{
-		Root:                                    "",
-		NVIDIAContainerToolkitCLIExecutablePath: "nvidia-ctk",
-	}
-	// Create a discoverer for the single device:
-	d, err := discover.NewGraphicsDiscoverer(m.logger, selectedDevice, &cfg)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing discoverer: %v", err)
-	}
-
-	devices, err := d.Devices()
-	if err != nil {
-		return nil, fmt.Errorf("error getting DRM devices: %v", err)
-	}
-
-	var deviceNodes []*specs.DeviceNode
-	for _, d := range devices {
-		dn := specs.DeviceNode{
-			Path:     d.Path,
-			HostPath: d.HostPath,
-		}
-		deviceNodes = append(deviceNodes, &dn)
-	}
-
-	hooks, err := d.Hooks()
-	if err != nil {
-		return nil, fmt.Errorf("error getting hooks: %v", err)
-	}
-
-	var cdiHooks []*specs.Hook
-	for _, h := range hooks {
-		cdiHook := specs.Hook{
-			HookName: h.Lifecycle,
-			Path:     h.Path,
-			Args:     h.Args,
-		}
-		cdiHooks = append(cdiHooks, &cdiHook)
-	}
-
-	mounts, err := d.Mounts()
-	if err != nil {
-		return nil, fmt.Errorf("error getting mounts: %v", err)
-	}
-
-	var cdiMounts []*specs.Mount
-	for _, m := range mounts {
-		cdiMount := specs.Mount{
-			ContainerPath: m.Path,
-			HostPath:      m.HostPath,
-			Options: []string{
-				"ro",
-				"nosuid",
-				"nodev",
-				"bind",
-			},
-			Type: "bind",
-		}
-		cdiMounts = append(cdiMounts, &cdiMount)
-	}
-
-	edits := specs.ContainerEdits{
-		DeviceNodes: deviceNodes,
-		Hooks:       cdiHooks,
-		Mounts:      cdiMounts,
-	}
-
-	return &edits, nil
-}
-
-func (m command) getExistingMetaDeviceNodes() []*specs.DeviceNode {
-	metaDeviceNodePaths := []string{
-		"/dev/nvidia-modeset",
-		"/dev/nvidia-uvm-tools",
-		"/dev/nvidia-uvm",
-		"/dev/nvidiactl",
-	}
-
-	var existingDeviceNodePaths []string
-	for _, p := range metaDeviceNodePaths {
-		if _, err := os.Stat(p); err != nil {
-			m.logger.Infof("Ignoring missing meta device %v", p)
-			continue
-		}
-		existingDeviceNodePaths = append(existingDeviceNodePaths, p)
-	}
-
-	return getDeviceNodesFromPaths(existingDeviceNodePaths)
-}
-
-func getDeviceNodesFromPaths(deviceNodePaths []string) []*specs.DeviceNode {
-	var deviceNodes []*specs.DeviceNode
-	for _, p := range deviceNodePaths {
-		deviceNode := specs.DeviceNode{
-			Path: p,
-		}
-		deviceNodes = append(deviceNodes, &deviceNode)
-	}
-
-	return deviceNodes
-}
-
-func (m command) findLibs(nvmllib nvml.Interface) ([]string, error) {
-	version, r := nvmllib.SystemGetDriverVersion()
-	if r != nvml.SUCCESS {
-		return nil, fmt.Errorf("failed to determine driver version: %v", r)
-	}
-	m.logger.Infof("Using driver version %v", version)
-
-	cache, err := ldcache.New(m.logger, "")
-	if err != nil {
-		return nil, fmt.Errorf("failed to load ldcache: %v", err)
-	}
-
-	libs32, libs64 := cache.List()
-
-	var libs []string
-	for _, l := range libs64 {
-		if strings.HasSuffix(l, version) {
-			m.logger.Infof("found 64-bit driver lib: %v", l)
-			libs = append(libs, l)
+	var hasAll bool
+	for _, deviceSpec := range deviceSpecs {
+		if deviceSpec.Name == allDeviceName {
+			hasAll = true
+			break
 		}
 	}
-
-	for _, l := range libs32 {
-		if strings.HasSuffix(l, version) {
-			m.logger.Infof("found 32-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
-	}
-
-	return libs, nil
-}
-
-func (m command) findBinaries() ([]string, error) {
-	candidates := []string{
-		"nvidia-smi",              /* System management interface */
-		"nvidia-debugdump",        /* GPU coredump utility */
-		"nvidia-persistenced",     /* Persistence mode utility */
-		"nvidia-cuda-mps-control", /* Multi process service CLI */
-		"nvidia-cuda-mps-server",  /* Multi process service server */
-	}
-
-	locator := lookup.NewExecutableLocator(m.logger, "")
-
-	var binaries []string
-	for _, c := range candidates {
-		targets, err := locator.Locate(c)
+	if !hasAll {
+		allDevice, err := MergeDeviceSpecs(deviceSpecs, allDeviceName)
 		if err != nil {
-			m.logger.Warningf("skipping %v: %v", c, err)
-			continue
+			return nil, fmt.Errorf("failed to create CDI specification for %q device: %v", allDeviceName, err)
 		}
-
-		binaries = append(binaries, targets[0])
-	}
-	return binaries, nil
-}
-
-func (m command) findIPC() ([]string, error) {
-	candidates := []string{
-		"/var/run/nvidia-persistenced/socket",
-		"/var/run/nvidia-fabricmanager/socket",
-		// TODO: This can be controlled by the NV_MPS_PIPE_DIR envvar
-		"/tmp/nvidia-mps",
+		deviceSpecs = append(deviceSpecs, allDevice)
 	}
 
-	locator := lookup.NewFileLocator(m.logger, "")
-
-	var ipcs []string
-	for _, c := range candidates {
-		targets, err := locator.Locate(c)
-		if err != nil {
-			m.logger.Warningf("skipping %v: %v", c, err)
-			continue
-		}
-
-		ipcs = append(ipcs, targets[0])
+	commonEdits, err := cdilib.GetCommonEdits()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
 	}
-	return ipcs, nil
-}
 
-func generateMountsForPaths(options []string, pathSets ...[]string) []*specs.Mount {
-	var mounts []*specs.Mount
-	for _, paths := range pathSets {
-		for _, p := range paths {
-			mount := specs.Mount{
-				HostPath: p,
-				// We may want to adjust the container path
-				ContainerPath: p,
-				Type:          "bind",
-				Options:       options,
-			}
-			mounts = append(mounts, &mount)
-		}
-	}
-	return mounts
-}
-
-func (m command) generateUpdateLdCacheHook(libraries []string) *specs.Hook {
-	locator := lookup.NewExecutableLocator(m.logger, "")
-
-	hook := discover.CreateLDCacheUpdateHook(
-		m.logger,
-		locator,
-		nvidiaCTKExecutable,
-		nvidiaCTKDefaultFilePath,
-		libraries,
+	return spec.New(
+		spec.WithVendor(cfg.vendor),
+		spec.WithClass(cfg.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(cfg.format),
 	)
-	return &specs.Hook{
-		HookName: hook.Lifecycle,
-		Path:     hook.Path,
-		Args:     hook.Args,
-	}
 }
 
-func (m command) generateDeviceFolderPermissionHooks(nvidiaCTKPath string, deviceNodes []*specs.DeviceNode) ([]*specs.Hook, error) {
-	var deviceFolders []string
-	seen := make(map[string]bool)
-
-	for _, dn := range deviceNodes {
-		if !strings.HasPrefix(dn.Path, "/dev") {
-			m.logger.Warningf("Skipping unexpected device folder path for device %v", dn.Path)
-			continue
-		}
-		for df := filepath.Dir(dn.Path); df != "/dev"; df = filepath.Dir(df) {
-			if seen[df] {
-				continue
-			}
-			deviceFolders = append(deviceFolders, df)
-			seen[df] = true
+// MergeDeviceSpecs creates a device with the specified name which combines the edits from the previous devices.
+// If a device of the specified name already exists, an error is returned.
+func MergeDeviceSpecs(deviceSpecs []specs.Device, mergedDeviceName string) (specs.Device, error) {
+	if err := cdi.ValidateDeviceName(mergedDeviceName); err != nil {
+		return specs.Device{}, fmt.Errorf("invalid device name %q: %v", mergedDeviceName, err)
+	}
+	for _, d := range deviceSpecs {
+		if d.Name == mergedDeviceName {
+			return specs.Device{}, fmt.Errorf("device %q already exists", mergedDeviceName)
 		}
 	}
 
-	foldersByMode := make(map[string][]string)
-	for _, p := range deviceFolders {
-		info, err := os.Stat(p)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get info for path %v: %v", p, err)
+	mergedEdits := edits.NewContainerEdits()
+
+	for _, d := range deviceSpecs {
+		edit := cdi.ContainerEdits{
+			ContainerEdits: &d.ContainerEdits,
 		}
-		mode := fmt.Sprintf("%o", info.Mode().Perm())
-		foldersByMode[mode] = append(foldersByMode[mode], p)
+		mergedEdits.Append(&edit)
 	}
 
-	var hooks []*specs.Hook
-	for mode, folders := range foldersByMode {
-		args := []string{filepath.Base(nvidiaCTKPath), "hook", "chmod", "--mode", mode}
-		for _, folder := range folders {
-			args = append(args, "--path", folder)
-		}
-		hook := specs.Hook{
-			HookName: cdi.CreateContainerHook,
-			Path:     nvidiaCTKPath,
-			Args:     args,
-		}
-
-		hooks = append(hooks, &hook)
+	merged := specs.Device{
+		Name:           mergedDeviceName,
+		ContainerEdits: *mergedEdits.ContainerEdits,
 	}
-
-	return hooks, nil
+	return merged, nil
 }

cmd/nvidia-ctk/cdi/generate/generate_test.go

@@ -0,0 +1,117 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeDeviceSpecs(t *testing.T) {
+	testCases := []struct {
+		description      string
+		deviceSpecs      []specs.Device
+		mergedDeviceName string
+		expectedError    error
+		expected         specs.Device
+	}{
+		{
+			description:      "no devices",
+			mergedDeviceName: "all",
+			expected: specs.Device{
+				Name: "all",
+			},
+		},
+		{
+			description:      "one device",
+			mergedDeviceName: "all",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+			},
+			expected: specs.Device{
+				Name: "all",
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"GPU=0"},
+				},
+			},
+		},
+		{
+			description:      "two devices",
+			mergedDeviceName: "all",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+				{
+					Name: "gpu1",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=1"},
+					},
+				},
+			},
+			expected: specs.Device{
+				Name: "all",
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"GPU=0", "GPU=1"},
+				},
+			},
+		},
+		{
+			description:      "has merged device",
+			mergedDeviceName: "gpu0",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+			},
+			expectedError: fmt.Errorf("device %q already exists", "gpu0"),
+		},
+		{
+			description:      "invalid merged device name",
+			mergedDeviceName: ".-not-valid",
+			expectedError:    fmt.Errorf("invalid device name %q", ".-not-valid"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mergedDevice, err := MergeDeviceSpecs(tc.deviceSpecs, tc.mergedDeviceName)
+
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expected, mergedDevice)
+		})
+	}
+}

cmd/nvidia-ctk/cdi/generate/nvml_devices.go

@@ -1,123 +0,0 @@
-/*
- * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY Type, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package generate
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-)
-
-// nvmlDevice wraps an nvml.Device with more functions.
-type nvmlDevice struct {
-	nvml.Device
-}
-
-// nvmlMigDevice allows for specific functions of nvmlDevice to be overridden.
-type nvmlMigDevice nvmlDevice
-
-// deviceInfo defines the information the required to construct a Device
-type deviceInfo interface {
-	GetUUID() (string, error)
-	GetDeviceNodes() ([]string, error)
-}
-
-var _ deviceInfo = (*nvmlDevice)(nil)
-var _ deviceInfo = (*nvmlMigDevice)(nil)
-
-func newGPUDevice(i int, gpu device.Device) (string, nvmlDevice) {
-	return fmt.Sprintf("gpu%v", i), nvmlDevice{gpu}
-}
-
-func newMigDevice(i int, j int, mig device.MigDevice) (string, nvmlMigDevice) {
-	return fmt.Sprintf("mig%v:%v", i, j), nvmlMigDevice{mig}
-}
-
-// GetUUID returns the UUID of the device
-func (d nvmlDevice) GetUUID() (string, error) {
-	uuid, ret := d.Device.GetUUID()
-	if ret != nvml.SUCCESS {
-		return "", ret
-	}
-	return uuid, nil
-}
-
-// GetUUID returns the UUID of the device
-func (d nvmlMigDevice) GetUUID() (string, error) {
-	return nvmlDevice(d).GetUUID()
-}
-
-// GetDeviceNodes returns the device node paths for a GPU device
-func (d nvmlDevice) GetDeviceNodes() ([]string, error) {
-	minor, ret := d.GetMinorNumber()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
-	}
-	path := fmt.Sprintf("/dev/nvidia%d", minor)
-
-	return []string{path}, nil
-}
-
-// GetDeviceNodes returns the device node paths for a MIG device
-func (d nvmlMigDevice) GetDeviceNodes() ([]string, error) {
-	parent, ret := d.GetDeviceHandleFromMigDeviceHandle()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting parent device: %v", ret)
-	}
-	minor, ret := parent.GetMinorNumber()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
-	}
-	parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
-
-	migCaps, err := nvcaps.NewMigCaps()
-	if err != nil {
-		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
-	}
-
-	gi, ret := d.GetGpuInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
-	}
-
-	ci, ret := d.GetComputeInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
-	}
-
-	giCap := nvcaps.NewGPUInstanceCap(minor, gi)
-	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
-	}
-
-	ciCap := nvcaps.NewComputeInstanceCap(minor, gi, ci)
-	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
-	}
-
-	devicePaths := []string{
-		parentPath,
-		giCapDevicePath,
-		ciCapDevicePath,
-	}
-
-	return devicePaths, nil
-}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,159 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type loadSaver interface {
+	Load() (spec.Interface, error)
+	Save(spec.Interface) error
+}
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from string
+	to   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transform.NewRootTransformer(
+		opts.from,
+		opts.to,
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -18,6 +18,7 @@ package chmod
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
@@ -133,7 +134,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 func (m command) getPaths(root string, paths []string) []string {
 	var pathsInRoot []string
 	for _, f := range paths {
-		pathsInRoot = append(pathsInRoot, filepath.Join(root, f))
+		path := filepath.Join(root, f)
+		if _, err := os.Stat(path); err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
 	}
 
 	return pathsInRoot

cmd/nvidia-ctk/hook/hook.go

@@ -18,6 +18,7 @@ package hook
 
 import (
 	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
 	"github.com/sirupsen/logrus"

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -84,6 +84,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
+	_, err = os.Stat(filepath.Join(containerRoot, "/etc/ld.so.cache"))
+	if err != nil && os.IsNotExist(err) {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		return nil
+	}
+
 	err = m.createConfig(containerRoot, cfg.folders.Value())
 	if err != nil {
 		return fmt.Errorf("failed to update ld.so.conf: %v", err)
@@ -105,6 +111,10 @@ func (m command) createConfig(root string, folders []string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
 	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
 	if err != nil {
 		return fmt.Errorf("failed to create config file: %v", err)

cmd/nvidia-ctk/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
 	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 
 	log "github.com/sirupsen/logrus"
@@ -77,6 +78,7 @@ func main() {
 		runtime.NewCommand(logger),
 		infoCLI.NewCommand(logger),
 		cdi.NewCommand(logger),
+		system.NewCommand(logger),
 	}
 
 	// Run the CLI

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -22,8 +22,8 @@ import (
 	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -127,13 +127,14 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
 		configFilePath = defaultDockerConfigFilePath
 	}
 
-	cfg, err := docker.LoadConfig(configFilePath)
+	cfg, err := docker.New(
+		docker.WithPath(configFilePath),
+	)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
 	}
 
-	err = docker.UpdateConfig(
-		cfg,
+	err = cfg.AddRuntime(
 		config.nvidiaOptions.RuntimeName,
 		config.nvidiaOptions.RuntimePath,
 		config.nvidiaOptions.SetAsDefault,
@@ -150,12 +151,16 @@ func (m command) configureDocker(c *cli.Context, config *config) error {
 		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
 		return nil
 	}
-	err = docker.FlushConfig(cfg, configFilePath)
+	n, err := cfg.Save(configFilePath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)
 	}
 
-	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	if n == 0 {
+		m.logger.Infof("Removed empty config from %v", configFilePath)
+	} else {
+		m.logger.Infof("Wrote updated config to %v", configFilePath)
+	}
 	m.logger.Infof("It is recommended that the docker daemon be restarted.")
 
 	return nil
@@ -168,13 +173,14 @@ func (m command) configureCrio(c *cli.Context, config *config) error {
 		configFilePath = defaultCrioConfigFilePath
 	}
 
-	cfg, err := crio.LoadConfig(configFilePath)
+	cfg, err := crio.New(
+		crio.WithPath(configFilePath),
+	)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
 	}
 
-	err = crio.UpdateConfig(
-		cfg,
+	err = cfg.AddRuntime(
 		config.nvidiaOptions.RuntimeName,
 		config.nvidiaOptions.RuntimePath,
 		config.nvidiaOptions.SetAsDefault,
@@ -191,12 +197,16 @@ func (m command) configureCrio(c *cli.Context, config *config) error {
 		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
 		return nil
 	}
-	err = crio.FlushConfig(configFilePath, cfg)
+	n, err := cfg.Save(configFilePath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)
 	}
 
-	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	if n == 0 {
+		m.logger.Infof("Removed empty config from %v", configFilePath)
+	} else {
+		m.logger.Infof("Wrote updated config to %v", configFilePath)
+	}
 	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
 
 	return nil

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -0,0 +1,175 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
+)
+
+type allPossible struct {
+	logger       *logrus.Logger
+	driverRoot   string
+	deviceMajors devices.Devices
+	migCaps      nvcaps.MigCaps
+}
+
+// newAllPossible returns a new allPossible device node lister.
+// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error) {
+	deviceMajors, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed reading device majors: %v", err)
+	}
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
+	}
+	if migCaps == nil {
+		migCaps = make(nvcaps.MigCaps)
+	}
+
+	l := allPossible{
+		logger:       logger,
+		driverRoot:   driverRoot,
+		deviceMajors: deviceMajors,
+		migCaps:      migCaps,
+	}
+
+	return l, nil
+}
+
+// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func (m allPossible) DeviceNodes() ([]deviceNode, error) {
+	gpus, err := nvpci.NewFrom(
+		filepath.Join(m.driverRoot, nvpci.PCIDevicesRoot),
+	).GetGPUs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GPU information: %v", err)
+	}
+
+	count := len(gpus)
+	if count == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		return nil, nil
+	}
+
+	deviceNodes, err := m.getControlDeviceNodes()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
+	}
+
+	for gpu := 0; gpu < count; gpu++ {
+		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
+		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
+	}
+
+	return deviceNodes, nil
+}
+
+// getControlDeviceNodes generates a list of control devices
+func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
+	var deviceNodes []deviceNode
+
+	// Define the control devices for standard GPUs.
+	controlDevices := []deviceNode{
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
+	}
+
+	deviceNodes = append(deviceNodes, controlDevices...)
+
+	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
+		migControlMinor, exist := m.migCaps[migControlDevice]
+		if !exist {
+			continue
+		}
+
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			migControlMinor.DevicePath(),
+			int(migControlMinor),
+		)
+
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes, nil
+}
+
+// getGPUDeviceNodes generates a list of device nodes for a given GPU.
+func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
+	d := m.newDeviceNode(
+		devices.NVIDIAGPU,
+		fmt.Sprintf("/dev/nvidia%d", gpu),
+		gpu,
+	)
+
+	return []deviceNode{d}
+}
+
+// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
+func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
+	var selectedCapMinors []nvcaps.MigMinor
+	for gi := 0; ; gi++ {
+		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+		giMinor, exist := m.migCaps[giCap]
+		if !exist {
+			break
+		}
+		selectedCapMinors = append(selectedCapMinors, giMinor)
+		for ci := 0; ; ci++ {
+			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+			ciMinor, exist := m.migCaps[ciCap]
+			if !exist {
+				break
+			}
+			selectedCapMinors = append(selectedCapMinors, ciMinor)
+		}
+	}
+
+	var deviceNodes []deviceNode
+	for _, capMinor := range selectedCapMinors {
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			capMinor.DevicePath(),
+			int(capMinor),
+		)
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes
+}
+
+// newDeviceNode creates a new device node with the specified path and major/minor numbers.
+// The path is adjusted for the specified driver root.
+func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
+	major, _ := m.deviceMajors.Get(deviceName)
+
+	return deviceNode{
+		path:  filepath.Join(m.driverRoot, path),
+		major: uint32(major),
+		minor: uint32(minor),
+	}
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -0,0 +1,332 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultDevCharPath = "/dev/char"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	devCharPath string
+	driverRoot  string
+	dryRun      bool
+	watch       bool
+	createAll   bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'create-dev-char-symlinks' command
+	c := cli.Command{
+		Name:  "create-dev-char-symlinks",
+		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "dev-char-path",
+			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
+			Value:       defaultDevCharPath,
+			Destination: &cfg.devCharPath,
+			EnvVars:     []string{"DEV_CHAR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
+			Value:       "/",
+			Destination: &cfg.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "watch",
+			Usage:       "If set, the command will watch for changes to the driver root and recreate the symlinks when changes are detected.",
+			Value:       false,
+			Destination: &cfg.watch,
+			EnvVars:     []string{"WATCH"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-all",
+			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
+			Destination: &cfg.createAll,
+			EnvVars:     []string{"CREATE_ALL"},
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "If set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &cfg.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	if cfg.createAll && cfg.watch {
+		return fmt.Errorf("create-all and watch are mutually exclusive")
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	var watcher *fsnotify.Watcher
+	var sigs chan os.Signal
+
+	if cfg.watch {
+		watcher, err := newFSWatcher(filepath.Join(cfg.driverRoot, "dev"))
+		if err != nil {
+			return fmt.Errorf("failed to create FS watcher: %v", err)
+		}
+		defer watcher.Close()
+
+		sigs = newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	}
+
+	l, err := NewSymlinkCreator(
+		WithLogger(m.logger),
+		WithDevCharPath(cfg.devCharPath),
+		WithDriverRoot(cfg.driverRoot),
+		WithDryRun(cfg.dryRun),
+		WithCreateAll(cfg.createAll),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink creator: %v", err)
+	}
+
+create:
+	err = l.CreateLinks()
+	if err != nil {
+		return fmt.Errorf("failed to create links: %v", err)
+	}
+	if !cfg.watch {
+		return nil
+	}
+	for {
+		select {
+
+		case event := <-watcher.Events:
+			deviceNode := filepath.Base(event.Name)
+			if !strings.HasPrefix(deviceNode, "nvidia") {
+				continue
+			}
+			if event.Op&fsnotify.Create == fsnotify.Create {
+				m.logger.Infof("%s created, restarting.", event.Name)
+				goto create
+			}
+			if event.Op&fsnotify.Create == fsnotify.Remove {
+				m.logger.Infof("%s removed. Ignoring", event.Name)
+
+			}
+
+		// Watch for any other fs errors and log them.
+		case err := <-watcher.Errors:
+			m.logger.Errorf("inotify: %s", err)
+
+		// React to signals
+		case s := <-sigs:
+			switch s {
+			case syscall.SIGHUP:
+				m.logger.Infof("Received SIGHUP, recreating symlinks.")
+				goto create
+			default:
+				m.logger.Infof("Received signal %q, shutting down.", s)
+				return nil
+			}
+		}
+	}
+}
+
+type linkCreator struct {
+	logger      *logrus.Logger
+	lister      nodeLister
+	driverRoot  string
+	devCharPath string
+	dryRun      bool
+	createAll   bool
+}
+
+// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
+type Creator interface {
+	CreateLinks() error
+}
+
+// Option is a functional option for configuring the linkCreator.
+type Option func(*linkCreator)
+
+// NewSymlinkCreator creates a new linkCreator.
+func NewSymlinkCreator(opts ...Option) (Creator, error) {
+	c := linkCreator{}
+	for _, opt := range opts {
+		opt(&c)
+	}
+	if c.logger == nil {
+		c.logger = logrus.StandardLogger()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+	if c.devCharPath == "" {
+		c.devCharPath = defaultDevCharPath
+	}
+
+	if c.createAll {
+		lister, err := newAllPossible(c.logger, c.driverRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
+		}
+		c.lister = lister
+	} else {
+		c.lister = existing{c.logger, c.driverRoot}
+	}
+	return c, nil
+}
+
+// WithDriverRoot sets the driver root path.
+func WithDriverRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.driverRoot = root
+	}
+}
+
+// WithDevCharPath sets the path at which the symlinks will be created.
+func WithDevCharPath(path string) Option {
+	return func(c *linkCreator) {
+		c.devCharPath = path
+	}
+}
+
+// WithDryRun sets the dry run flag.
+func WithDryRun(dryRun bool) Option {
+	return func(c *linkCreator) {
+		c.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger.
+func WithLogger(logger *logrus.Logger) Option {
+	return func(c *linkCreator) {
+		c.logger = logger
+	}
+}
+
+// WithCreateAll sets the createAll flag for the linkCreator.
+func WithCreateAll(createAll bool) Option {
+	return func(lc *linkCreator) {
+		lc.createAll = createAll
+	}
+}
+
+// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
+func (m linkCreator) CreateLinks() error {
+	deviceNodes, err := m.lister.DeviceNodes()
+	if err != nil {
+		return fmt.Errorf("failed to get device nodes: %v", err)
+	}
+
+	if len(deviceNodes) != 0 && !m.dryRun {
+		err := os.MkdirAll(m.devCharPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
+		}
+	}
+
+	for _, deviceNode := range deviceNodes {
+		target := deviceNode.path
+		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
+
+		m.logger.Infof("Creating link %s => %s", linkPath, target)
+		if m.dryRun {
+			continue
+		}
+
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warnf("Could not create symlink: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type deviceNode struct {
+	path  string
+	major uint32
+	minor uint32
+}
+
+func (d deviceNode) devCharName() string {
+	return fmt.Sprintf("%d:%d", d.major, d.minor)
+}
+
+func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, f := range files {
+		err = watcher.Add(f)
+		if err != nil {
+			watcher.Close()
+			return nil, err
+		}
+	}
+
+	return watcher, nil
+}
+
+func newOSWatcher(sigs ...os.Signal) chan os.Signal {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, sigs...)
+
+	return sigChan
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -0,0 +1,95 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+type nodeLister interface {
+	DeviceNodes() ([]deviceNode, error)
+}
+
+type existing struct {
+	logger     *logrus.Logger
+	driverRoot string
+}
+
+// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
+// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
+func (m existing) DeviceNodes() ([]deviceNode, error) {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(m.driverRoot),
+		lookup.WithOptional(true),
+	)
+
+	devices, err := locator.Locate("/dev/nvidia*")
+	if err != nil {
+		m.logger.Warnf("Error while locating device: %v", err)
+	}
+
+	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
+	if err != nil {
+		m.logger.Warnf("Error while locating caps device: %v", err)
+	}
+
+	if len(devices) == 0 && len(capDevices) == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		return nil, nil
+	}
+
+	var deviceNodes []deviceNode
+	for _, d := range append(devices, capDevices...) {
+		if m.nodeIsBlocked(d) {
+			continue
+		}
+
+		var stat unix.Stat_t
+		err := unix.Stat(d, &stat)
+		if err != nil {
+			m.logger.Warnf("Could not stat device: %v", err)
+			continue
+		}
+		deviceNode := deviceNode{
+			path:  d,
+			major: unix.Major(uint64(stat.Rdev)),
+			minor: unix.Minor(uint64(stat.Rdev)),
+		}
+
+		deviceNodes = append(deviceNodes, deviceNode)
+	}
+
+	return deviceNodes, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m existing) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,107 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type options struct {
+	driverRoot string
+
+	dryRun bool
+
+	control bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device ndoes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	s, err := system.New(
+		system.WithLogger(m.logger),
+		system.WithDryRun(opts.dryRun),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create library: %v", err)
+	}
+
+	if opts.control {
+		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
+		if err := s.CreateNVIDIAControlDeviceNodesAt(opts.driverRoot); err != nil {
+			return fmt.Errorf("failed to create control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m command) build() *cli.Command {
+	// Create the 'system' command
+	system := cli.Command{
+		Name:  "system",
+		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	system.Subcommands = []*cli.Command{
+		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
+	}
+
+	return &system
+}

docker/Dockerfile.devel

@@ -14,8 +14,8 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go install golang.org/x/lint/golint@latest
+RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
 RUN go install github.com/matryer/moq@latest
-RUN go install github.com/gordonklaus/ineffassign@latest
+RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
 RUN go install github.com/client9/misspell/cmd/misspell@latest
 RUN go install github.com/google/go-licenses@latest

docker/docker.mk

@@ -14,10 +14,10 @@
 
 # Supported OSs by architecture
 AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
-X86_64_TARGETS := fedora35 centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
+X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := fedora35 centos8 rhel8 amazonlinux2
+AARCH64_TARGETS := centos8 rhel8 amazonlinux2
 
 # Define top-level build targets
 docker%: SHELL:=/bin/bash
@@ -88,53 +88,31 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
 LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
 
+LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
---ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---ubuntu%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
---ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
---debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---debian%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
---debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
---centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --centos%: CONFIG_TOML_SUFFIX := rpm-yum
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
-# private fedora target
---fedora%: OS := fedora
---fedora%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
---fedora%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---fedora%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---fedora%: CONFIG_TOML_SUFFIX := rpm-yum
-# The fedora(35) base image has very slow performance when building aarch64 packages.
-# Since our primary concern here is glibc versions, we use the older glibc version available in centos8.
---fedora35%: BASEIMAGE = quay.io/centos/centos:stream8
-
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
 --amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
 --rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
@@ -155,8 +133,8 @@ docker-build-%:
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 	    --build-arg PKG_NAME="$(LIB_NAME)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
+	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
 	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
 	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
 	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \

go.mod

@@ -4,38 +4,33 @@ go 1.18
 
 require (
 	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82
-	github.com/container-orchestrated-devices/container-device-interface v0.5.2
-	github.com/opencontainers/runc v1.1.4
-	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
+	github.com/NVIDIA/go-nvml v0.12.0-0
+	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
+	github.com/fsnotify/fsnotify v1.5.4
+	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
 	github.com/pelletier/go-toml v1.9.4
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
-	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220922133427-1049a7fa76a9
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438
 	golang.org/x/mod v0.5.0
 	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
-	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/opencontainers/runtime-tools v0.9.1-0.20220110225228-7e2d60f1e41f // indirect
+	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
 	github.com/opencontainers/selinux v1.10.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
-	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
-	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -3,13 +3,14 @@ github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82 h1:x751Xx1tdxkiA/sdkv2J769n21UbYKzVOpe9S/h1M3k=
 github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
-github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/NVIDIA/go-nvml v0.12.0-0 h1:eHYNHbzAsMgWYshf6dEmTY66/GCXnORJFnzm3TNH4mc=
+github.com/NVIDIA/go-nvml v0.12.0-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
 github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
-github.com/container-orchestrated-devices/container-device-interface v0.5.2 h1:Bf/Zq8UBhbSBtB+pFBVIQ2Rh7sNK/x2ZEr6uW5YjNv8=
-github.com/container-orchestrated-devices/container-device-interface v0.5.2/go.mod h1:ZToWfSyUH5l9Rk7/bjkUUkNLz4b1mE+CVUVafuikDPY=
+github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a h1:sP3PcgyIkRlHqfF3Jfpe/7G8kf/qpzG4C8r94y9hLbE=
+github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a/go.mod h1:xMRa4fJgXzSDFUCURSimOUgoSc+odohvO3uXT9xjqH0=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -46,20 +47,17 @@ github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg=
 github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
-github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab h1:YQZXa3elcHgKXAa2GjVFC9M3JeP7ZPyFD1YByDx/dgQ=
-github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.9.1-0.20220110225228-7e2d60f1e41f h1:MMcsVl0FAVEahmXTy+uXoDTw3yJq7nGrK8ITs/kkreo=
-github.com/opencontainers/runtime-tools v0.9.1-0.20220110225228-7e2d60f1e41f/go.mod h1:/tgP02fPXGHkU3/qKK1Y0Db4yqNyGm03vLq/mzHzcS4=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb h1:1xSVPOd7/UA+39/hXEGnBJ13p6JFB0E1EvQFlrRDOXI=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w=
 github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -90,8 +88,10 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220922133427-1049a7fa76a9 h1:3C1cK0/n1HGfH0MPCCPRntK8xPNlrVq9OWR4RA72jW0=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220922133427-1049a7fa76a9/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342 h1:083n9fJt2dWOpJd/X/q9Xgl5XtQLL22uSFYbzVqJssg=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438 h1:+qRai7XRl8omFQVCeHcaWzL542Yw64vfmuXG+79ZCIc=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
 golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -121,7 +121,7 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

internal/config/config.go

@@ -45,9 +45,12 @@ var (
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
-	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+	AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+
+	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file
@@ -91,6 +94,8 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 		return cfg, nil
 	}
 
+	cfg.AcceptEnvvarUnprivileged = toml.GetDefault("accept-nvidia-visible-devices-envvar-when-unprivileged", cfg.AcceptEnvvarUnprivileged).(bool)
+
 	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
 	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
 	runtimeConfig, err := getRuntimeConfigFrom(toml)
@@ -99,12 +104,19 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 	}
 	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
 
+	runtimeHookConfig, err := getRuntimeHookConfigFrom(toml)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load nvidia-container-runtime-hook config: %v", err)
+	}
+	cfg.NVIDIAContainerRuntimeHookConfig = *runtimeHookConfig
+
 	return cfg, nil
 }
 
 // getDefaultConfig defines the default values for the config
 func getDefaultConfig() *Config {
 	c := Config{
+		AcceptEnvvarUnprivileged:     true,
 		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
 		NVIDIACTKConfig:              *getDefaultCTKConfig(),
 		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),

internal/config/config_test.go

@@ -57,6 +57,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "empty config is default",
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: true,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "",
 				},
@@ -69,6 +70,10 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{
@@ -79,6 +84,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set inline",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"nvidia-container-cli.root = \"/bar/baz\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
 				"nvidia-container-runtime.experimental = true",
@@ -86,10 +92,13 @@ func TestGetConfig(t *testing.T) {
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
+				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "/bar/baz",
 				},
@@ -102,6 +111,13 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{
@@ -112,6 +128,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set in section",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
 				"[nvidia-container-runtime]",
@@ -121,12 +138,16 @@ func TestGetConfig(t *testing.T) {
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
 				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.cdi]",
+				"default-kind = \"example.vendor.com/device\"",
+				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "/bar/baz",
 				},
@@ -139,6 +160,13 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{

internal/config/crio/crio.go

@@ -1,125 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package crio
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pelletier/go-toml"
-	log "github.com/sirupsen/logrus"
-)
-
-// LoadConfig loads the cri-o config from disk
-func LoadConfig(config string) (*toml.Tree, error) {
-	log.Infof("Loading config: %v", config)
-
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	configFile := config
-	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		log.Infof("Config file does not exist, creating new one")
-	}
-
-	cfg, err := toml.LoadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-
-	return cfg, nil
-}
-
-// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
-func UpdateConfig(config *toml.Tree, runtimeClass string, runtimePath string, setAsDefault bool) error {
-	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
-	case *toml.Tree:
-		runc, _ = toml.Load(runc.String())
-		config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass}, runc)
-	}
-
-	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_path"}, runtimePath)
-	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_type"}, "oci")
-
-	if setAsDefault {
-		config.SetPath([]string{"crio", "runtime", "default_runtime"}, runtimeClass)
-	}
-
-	return nil
-}
-
-// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
-func RevertConfig(config *toml.Tree, runtimeClass string) error {
-	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
-		if runtimeClass == runtime {
-			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
-		}
-	}
-
-	runtimeClassPath := []string{"crio", "runtime", "runtimes", runtimeClass}
-	config.DeletePath(runtimeClassPath)
-	for i := 0; i < len(runtimeClassPath); i++ {
-		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
-		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
-			if len(entry.Keys()) != 0 {
-				break
-			}
-			config.DeletePath(remainingPath)
-		}
-	}
-
-	return nil
-}
-
-// FlushConfig flushes the updated/reverted config out to disk
-func FlushConfig(config string, cfg *toml.Tree) error {
-	log.Infof("Flushing config")
-
-	output, err := cfg.ToTomlString()
-	if err != nil {
-		return fmt.Errorf("unable to convert to TOML: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(config)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(config)
-		if err != nil {
-			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(output)
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
-}

internal/config/docker/docker.go

@@ -1,117 +0,0 @@
-/**
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package docker
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// LoadConfig loads the docker config from disk
-func LoadConfig(configFilePath string) (map[string]interface{}, error) {
-	log.Infof("Loading docker config from %v", configFilePath)
-
-	info, err := os.Stat(configFilePath)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	cfg := make(map[string]interface{})
-
-	if os.IsNotExist(err) {
-		log.Infof("Config file does not exist, creating new one")
-		return cfg, nil
-	}
-
-	readBytes, err := ioutil.ReadFile(configFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("unable to read config: %v", err)
-	}
-
-	reader := bytes.NewReader(readBytes)
-	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-	return cfg, nil
-}
-
-// UpdateConfig updates the docker config to include the nvidia runtimes
-func UpdateConfig(config map[string]interface{}, runtimeName string, runtimePath string, setAsDefault bool) error {
-	// Read the existing runtimes
-	runtimes := make(map[string]interface{})
-	if _, exists := config["runtimes"]; exists {
-		runtimes = config["runtimes"].(map[string]interface{})
-	}
-
-	// Add / update the runtime definitions
-	runtimes[runtimeName] = map[string]interface{}{
-		"path": runtimePath,
-		"args": []string{},
-	}
-
-	// Update the runtimes definition
-	if len(runtimes) > 0 {
-		config["runtimes"] = runtimes
-	}
-
-	if setAsDefault {
-		config["default-runtime"] = runtimeName
-	}
-
-	return nil
-}
-
-// FlushConfig flushes the updated/reverted config out to disk
-func FlushConfig(cfg map[string]interface{}, configFilePath string) error {
-	log.Infof("Flushing docker config to %v", configFilePath)
-
-	output, err := json.MarshalIndent(cfg, "", "    ")
-	if err != nil {
-		return fmt.Errorf("unable to convert to JSON: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(configFilePath)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(configFilePath)
-		if err != nil {
-			return fmt.Errorf("unable to open %v for writing: %v", configFilePath, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(string(output))
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
-}

internal/config/engine/api.go

@@ -0,0 +1,25 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package engine
+
+// Interface defines the API for a runtime config updater.
+type Interface interface {
+	DefaultRuntime() string
+	AddRuntime(string, string, bool) error
+	RemoveRuntime(string) error
+	Save(string) (int64, error)
+}

internal/config/engine/containerd/config_v1.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// ConfigV1 represents a version 1 containerd config
+type ConfigV1 Config
+
+var _ engine.Interface = (*ConfigV1)(nil)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := *c.Tree
+
+	config.Set("version", int64(1))
+
+	switch runc := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", "runc"}).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, runc)
+	}
+
+	if config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name}) == nil {
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := (*Config)(c).getRuntimeAnnotations([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
+	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}, path)
+
+	if setAsDefault && c.UseDefaultRuntimeName {
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
+	} else if setAsDefault {
+		// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
+		if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
+		}
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c ConfigV1) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *ConfigV1) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	// If the specified runtime was set as the default runtime we need to remove the default runtime too.
+	runtimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}).(string)
+	if !ok || runtimePath == "" {
+		runtimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}).(string)
+	}
+	defaultRuntimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}).(string)
+	if !ok || defaultRuntimePath == "" {
+		defaultRuntimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}).(string)
+	}
+	if runtimePath != "" && defaultRuntimePath != "" && runtimePath == defaultRuntimePath {
+		config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime"})
+	}
+
+	config.DeletePath([]string{"plugins", "cri", "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimeConfigPath := []string{"plugins", "cri", "containerd", "runtimes", name}
+	for i := 0; i < len(runtimeConfigPath); i++ {
+		if runtimes, ok := config.GetPath(runtimeConfigPath[:len(runtimeConfigPath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimeConfigPath[:len(runtimeConfigPath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// Save wrotes the config to a file
+func (c ConfigV1) Save(path string) (int64, error) {
+	return (Config)(c).Save(path)
+}

internal/config/engine/containerd/config_v2.go

@@ -0,0 +1,161 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+	config := *c.Tree
+
+	config.Set("version", int64(2))
+
+	switch runc := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc"}).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, runc)
+	}
+
+	if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := c.getRuntimeAnnotations([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
+
+	if setAsDefault {
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
+	if c == nil || c.Tree == nil {
+		return nil, nil
+	}
+
+	config := *c.Tree
+	if !config.HasPath(path) {
+		return nil, nil
+	}
+	annotationsI, ok := config.GetPath(path).([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("invalid annotations: %v", annotationsI)
+	}
+
+	var annotations []string
+	for _, annotation := range annotationsI {
+		a, ok := annotation.(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid annotation: %v", annotation)
+		}
+		annotations = append(annotations, a)
+	}
+
+	return annotations, nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
+	for i := 0; i < len(runtimePath); i++ {
+		if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimePath[:len(runtimePath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	config := c.Tree
+	output, err := config.ToTomlString()
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(output)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), err
+}

internal/config/engine/containerd/containerd.go

@@ -0,0 +1,40 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// Config represents the containerd config
+type Config struct {
+	*toml.Tree
+	RuntimeType           string
+	UseDefaultRuntimeName bool
+	ContainerAnnotations  []string
+}
+
+// New creates a containerd config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}

internal/config/engine/containerd/option.go

@@ -0,0 +1,149 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultRuntimeType = "io.containerd.runc.v2"
+)
+
+type builder struct {
+	path                 string
+	runtimeType          string
+	useLegacyConfig      bool
+	containerAnnotations []string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+// WithRuntimeType sets the runtime type for the config builder
+func WithRuntimeType(runtimeType string) Option {
+	return func(b *builder) {
+		b.runtimeType = runtimeType
+	}
+}
+
+// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
+func WithUseLegacyConfig(useLegacyConfig bool) Option {
+	return func(b *builder) {
+		b.useLegacyConfig = useLegacyConfig
+	}
+}
+
+// WithContainerAnnotations sets the container annotations for the config builder
+func WithContainerAnnotations(containerAnnotations ...string) Option {
+	return func(b *builder) {
+		b.containerAnnotations = containerAnnotations
+	}
+}
+
+func (b *builder) build() (engine.Interface, error) {
+	if b.path == "" {
+		return nil, fmt.Errorf("config path is empty")
+	}
+
+	if b.runtimeType == "" {
+		b.runtimeType = defaultRuntimeType
+	}
+
+	config, err := loadConfig(b.path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config.RuntimeType = b.runtimeType
+	config.UseDefaultRuntimeName = !b.useLegacyConfig
+	config.ContainerAnnotations = b.containerAnnotations
+
+	version, err := config.parseVersion(b.useLegacyConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config version: %v", err)
+	}
+	switch version {
+	case 1:
+		return (*ConfigV1)(config), nil
+	case 2:
+		return config, nil
+	}
+
+	return nil, fmt.Errorf("unsupported config version: %v", version)
+}
+
+// loadConfig loads the containerd config from disk
+func loadConfig(config string) (*Config, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	tomlConfig, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	cfg := Config{
+		Tree: tomlConfig,
+	}
+	return &cfg, nil
+}
+
+// parseVersion returns the version of the config
+func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
+	defaultVersion := 2
+	if useLegacyConfig {
+		defaultVersion = 1
+	}
+
+	switch v := c.Get("version").(type) {
+	case nil:
+		switch len(c.Keys()) {
+		case 0: // No config exists, or the config file is empty, use version inferred from containerd
+			return defaultVersion, nil
+		default: // A config file exists, has content, and no version is set
+			return 1, nil
+		}
+	case int64:
+		return int(v), nil
+	default:
+		return -1, fmt.Errorf("unsupported type for version field: %v", v)
+	}
+}

internal/config/engine/crio/crio.go

@@ -0,0 +1,131 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// Config represents the cri-o config
+type Config toml.Tree
+
+// New creates a cri-o config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}
+
+// AddRuntime adds a new runtime to the crio config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := (toml.Tree)(*c)
+
+	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"crio", "runtime", "runtimes", name}, runc)
+	}
+
+	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_path"}, path)
+	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_type"}, "oci")
+
+	if setAsDefault {
+		config.SetPath([]string{"crio", "runtime", "default_runtime"}, name)
+	}
+
+	*c = (Config)(config)
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	config := (toml.Tree)(c)
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the cri-o config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil {
+		return nil
+	}
+
+	config := (toml.Tree)(*c)
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
+		}
+	}
+
+	runtimeClassPath := []string{"crio", "runtime", "runtimes", name}
+	config.DeletePath(runtimeClassPath)
+	for i := 0; i < len(runtimeClassPath); i++ {
+		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
+		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
+			if len(entry.Keys()) != 0 {
+				break
+			}
+			config.DeletePath(remainingPath)
+		}
+	}
+
+	*c = (Config)(config)
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	config := (toml.Tree)(c)
+	output, err := config.ToTomlString()
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(output)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), err
+}

internal/config/engine/crio/option.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+type builder struct {
+	path string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+func (b *builder) build() (*Config, error) {
+	if b.path == "" {
+		empty := toml.Tree{}
+		return (*Config)(&empty), nil
+	}
+
+	return loadConfig(b.path)
+}
+
+// loadConfig loads the cri-o config from disk
+func loadConfig(config string) (*Config, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return (*Config)(cfg), nil
+}

internal/config/engine/docker/docker.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+)
+
+const (
+	defaultDockerRuntime = "runc"
+)
+
+// Config defines a docker config file.
+// TODO: This should not be public, but we need to access it from the tests in tools/container/docker
+type Config map[string]interface{}
+
+// New creates a docker config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}
+
+// AddRuntime adds a new runtime to the docker config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := *c
+
+	// Read the existing runtimes
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	// Add / update the runtime definitions
+	runtimes[name] = map[string]interface{}{
+		"path": path,
+		"args": []string{},
+	}
+
+	config["runtimes"] = runtimes
+
+	if setAsDefault {
+		config["default-runtime"] = name
+	}
+
+	*c = config
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the docker config
+func (c Config) DefaultRuntime() string {
+	r, ok := c["default-runtime"].(string)
+	if !ok {
+		return ""
+	}
+	return r
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil {
+		return nil
+	}
+	config := *c
+
+	if _, exists := config["default-runtime"]; exists {
+		defaultRuntime := config["default-runtime"].(string)
+		if defaultRuntime == name {
+			config["default-runtime"] = defaultDockerRuntime
+		}
+	}
+
+	if _, exists := config["runtimes"]; exists {
+		runtimes := config["runtimes"].(map[string]interface{})
+
+		delete(runtimes, name)
+
+		if len(runtimes) == 0 {
+			delete(config, "runtimes")
+		}
+	}
+
+	*c = config
+
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	output, err := json.MarshalIndent(c, "", "    ")
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(string(output))
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), nil
+}

internal/config/engine/docker/docker_test.go

@@ -26,7 +26,7 @@ import (
 
 func TestUpdateConfigDefaultRuntime(t *testing.T) {
 	testCases := []struct {
-		config                     map[string]interface{}
+		config                     Config
 		runtimeName                string
 		setAsDefault               bool
 		expectedDefaultRuntimeName interface{}
@@ -63,7 +63,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			if tc.config == nil {
 				tc.config = make(map[string]interface{})
 			}
-			err := UpdateConfig(tc.config, tc.runtimeName, "", tc.setAsDefault)
+			err := tc.config.AddRuntime(tc.runtimeName, "", tc.setAsDefault)
 			require.NoError(t, err)
 
 			defaultRuntimeName := tc.config["default-runtime"]
@@ -74,7 +74,7 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 
 func TestUpdateConfigRuntimes(t *testing.T) {
 	testCases := []struct {
-		config         map[string]interface{}
+		config         Config
 		runtimes       map[string]string
 		expectedConfig map[string]interface{}
 	}{
@@ -198,7 +198,7 @@ func TestUpdateConfigRuntimes(t *testing.T) {
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
 			for runtimeName, runtimePath := range tc.runtimes {
-				err := UpdateConfig(tc.config, runtimeName, runtimePath, false)
+				err := tc.config.AddRuntime(runtimeName, runtimePath, false)
 				require.NoError(t, err)
 			}
 

internal/config/engine/docker/option.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type builder struct {
+	path string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+func (b *builder) build() (*Config, error) {
+	if b.path == "" {
+		empty := make(Config)
+		return &empty, nil
+	}
+
+	return loadConfig(b.path)
+}
+
+// loadConfig loads the docker config from disk
+func loadConfig(configFilePath string) (*Config, error) {
+	log.Infof("Loading docker config from %v", configFilePath)
+
+	info, err := os.Stat(configFilePath)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(Config)
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return &cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return &cfg, nil
+}

internal/config/hook.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"fmt"
+
+	"github.com/pelletier/go-toml"
+)
+
+// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeHookConfig struct {
+	// SkipModeDetection disables the mode check for the runtime hook.
+	SkipModeDetection bool `toml:"skip-mode-detection"`
+}
+
+// dummyHookConfig allows us to unmarshal only a RuntimeHookConfig from a *toml.Tree
+type dummyHookConfig struct {
+	RuntimeHook RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
+}
+
+// getRuntimeHookConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getRuntimeHookConfigFrom(toml *toml.Tree) (*RuntimeHookConfig, error) {
+	cfg := GetDefaultRuntimeHookConfig()
+
+	if toml == nil {
+		return cfg, nil
+	}
+
+	d := dummyHookConfig{
+		RuntimeHook: *cfg,
+	}
+
+	if err := toml.Unmarshal(&d); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
+	}
+
+	return &d.RuntimeHook, nil
+}
+
+// GetDefaultRuntimeHookConfig defines the default values for the config
+func GetDefaultRuntimeHookConfig() *RuntimeHookConfig {
+	c := RuntimeHookConfig{
+		SkipModeDetection: false,
+	}
+
+	return &c
+}

internal/config/image/privileged.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged NVIDIA capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

@@ -19,6 +19,7 @@ package config
 import (
 	"fmt"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 )
@@ -50,6 +51,10 @@ type modesConfig struct {
 type cdiModeConfig struct {
 	// SpecDirs allows for the default spec dirs for CDI to be overridden
 	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
 }
 
 type csvModeConfig struct {
@@ -94,6 +99,12 @@ func GetDefaultRuntimeConfig() *RuntimeConfig {
 			CSV: csvModeConfig{
 				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 			},
+			CDI: cdiModeConfig{
+				DefaultKind: "nvidia.com/gpu",
+				AnnotationPrefixes: []string{
+					cdi.AnnotationPrefix,
+				},
+			},
 		},
 	}
 

internal/discover/char_devices.go

@@ -28,7 +28,10 @@ var _ Discover = (*charDevices)(nil)
 
 // NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
 func NewCharDeviceDiscoverer(logger *logrus.Logger, devices []string, root string) Discover {
-	locator := lookup.NewCharDeviceLocator(logger, root)
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(root),
+	)
 
 	return NewDeviceDiscoverer(logger, locator, root, devices)
 }
@@ -55,7 +58,11 @@ func (d *charDevices) Devices() ([]Device, error) {
 	}
 	var devices []Device
 	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
 	}
 
 	return devices, nil

internal/discover/csv.go

@@ -27,16 +27,16 @@ import (
 // NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
 // The constructed discoverer is comprised of a list, with each element in the list being associated with a
 // single CSV files.
-func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
+func NewFromCSVFiles(logger *logrus.Logger, files []string, driverRoot string) (Discover, error) {
 	if len(files) == 0 {
 		logger.Warnf("No CSV files specified")
 		return None{}, nil
 	}
 
-	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
+	symlinkLocator := lookup.NewSymlinkLocator(logger, driverRoot)
 	locators := map[csv.MountSpecType]lookup.Locator{
-		csv.MountSpecDev: lookup.NewCharDeviceLocator(logger, root),
-		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
+		csv.MountSpecDev: lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
+		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, driverRoot),
 		// Libraries and symlinks are handled in the same way
 		csv.MountSpecLib: symlinkLocator,
 		csv.MountSpecSym: symlinkLocator,
@@ -52,7 +52,7 @@ func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discov
 		mountSpecs = append(mountSpecs, targets...)
 	}
 
-	return newFromMountSpecs(logger, locators, root, mountSpecs)
+	return newFromMountSpecs(logger, locators, driverRoot, mountSpecs)
 }
 
 // loadCSVFile loads the specified CSV file and returns the list of mount specs
@@ -71,7 +71,7 @@ func loadCSVFile(logger *logrus.Logger, filename string) ([]*csv.MountSpec, erro
 
 // newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
 // A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
-func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, root string, targets []*csv.MountSpec) (Discover, error) {
+func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, driverRoot string, targets []*csv.MountSpec) (Discover, error) {
 	if len(targets) == 0 {
 		return &None{}, nil
 	}
@@ -95,9 +95,9 @@ func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]loo
 		var m Discover
 		switch t {
 		case csv.MountSpecDev:
-			m = NewDeviceDiscoverer(logger, locator, root, candidatesByType[t])
+			m = NewDeviceDiscoverer(logger, locator, driverRoot, candidatesByType[t])
 		default:
-			m = NewMounts(logger, locator, root, candidatesByType[t])
+			m = NewMounts(logger, locator, driverRoot, candidatesByType[t])
 		}
 		discoverers = append(discoverers, m)
 

internal/discover/discover.go

@@ -18,8 +18,8 @@ package discover
 
 // Config represents the configuration options for discovery
 type Config struct {
-	Root                                    string
-	NVIDIAContainerToolkitCLIExecutablePath string
+	DriverRoot    string
+	NvidiaCTKPath string
 }
 
 // Device represents a discovered character device.
@@ -32,6 +32,7 @@ type Device struct {
 type Mount struct {
 	HostPath string
 	Path     string
+	Options  []string
 }
 
 // Hook represents a discovered hook.
@@ -41,8 +42,9 @@ type Hook struct {
 	Args      []string
 }
 
-//go:generate moq -stub -out discover_mock.go . Discover
 // Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+//
+//go:generate moq -stub -out discover_mock.go . Discover
 type Discover interface {
 	Devices() ([]Device, error)
 	Mounts() ([]Mount, error)

internal/discover/gds.go

@@ -45,7 +45,10 @@ func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
 
 	cufile := NewMounts(
 		logger,
-		lookup.NewFileLocator(logger, root),
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
 		root,
 		[]string{"/etc/cufile.json"},
 	)

internal/discover/graphics.go

@@ -20,27 +20,53 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
 	"github.com/sirupsen/logrus"
 )
 
 // NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
 func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, cfg *Config) (Discover, error) {
-	root := cfg.Root
+	driverRoot := cfg.DriverRoot
 
-	locator, err := lookup.NewLibraryLocator(logger, root)
+	mounts, err := NewGraphicsMountsDiscoverer(logger, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
+	}
+
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
+
+	xorg := optionalXorgDiscoverer(logger, driverRoot, cfg.NvidiaCTKPath)
+
+	discover := Merge(
+		Merge(drmDeviceNodes, drmByPathSymlinks),
+		mounts,
+		xorg,
+	)
+
+	return discover, nil
+}
+
+// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
+func NewGraphicsMountsDiscoverer(logger *logrus.Logger, driverRoot string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct library locator: %v", err)
 	}
 	libraries := NewMounts(
 		logger,
 		locator,
-		root,
+		driverRoot,
 		[]string{
 			"libnvidia-egl-gbm.so",
 		},
@@ -48,29 +74,22 @@ func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices,
 
 	jsonMounts := NewMounts(
 		logger,
-		lookup.NewFileLocator(logger, root),
-		root,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/etc", "/usr/share"),
+		),
+		driverRoot,
 		[]string{
-			// TODO: We should handle this more cleanly
-			"/etc/glvnd/egl_vendor.d/10_nvidia.json",
-			"/etc/vulkan/icd.d/nvidia_icd.json",
-			"/etc/vulkan/implicit_layer.d/nvidia_layers.json",
-			"/usr/share/glvnd/egl_vendor.d/10_nvidia.json",
-			"/usr/share/vulkan/icd.d/nvidia_icd.json",
-			"/usr/share/vulkan/implicit_layer.d/nvidia_layers.json",
-			"/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"glvnd/egl_vendor.d/10_nvidia.json",
+			"vulkan/icd.d/nvidia_icd.json",
+			"vulkan/implicit_layer.d/nvidia_layers.json",
+			"egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"egl/egl_external_platform.d/10_nvidia_wayland.json",
 		},
 	)
 
-	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, root)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
-	}
-
-	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
-
 	discover := Merge(
-		Merge(drmDeviceNodes, drmByPathSymlinks),
 		libraries,
 		jsonMounts,
 	)
@@ -80,21 +99,19 @@ func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices,
 
 type drmDevicesByPath struct {
 	None
-	logger                  *logrus.Logger
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
-	root                    string
-	devicesFrom             Discover
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	driverRoot    string
+	devicesFrom   Discover
 }
 
 // newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
 func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover {
 	d := drmDevicesByPath{
-		logger:                  logger,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
-		root:                    cfg.Root,
-		devicesFrom:             devices,
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		driverRoot:    cfg.DriverRoot,
+		devicesFrom:   devices,
 	}
 
 	return &d
@@ -109,35 +126,26 @@ func (d drmDevicesByPath) Hooks() ([]Hook, error) {
 	if len(devices) == 0 {
 		return nil, nil
 	}
-
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{hookPath, "hook", "create-symlinks"}
 	links, err := d.getSpecificLinkArgs(devices)
 	if err != nil {
 		return nil, fmt.Errorf("failed to determine specific links: %v", err)
 	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var args []string
 	for _, l := range links {
 		args = append(args, "--link", l)
 	}
 
-	h := Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
 
-	return []Hook{h}, nil
+	return []Hook{hook}, nil
 }
 
 // getSpecificLinkArgs returns the required specic links that need to be created
@@ -147,10 +155,14 @@ func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error
 		selectedDevices[filepath.Base(d.HostPath)] = true
 	}
 
-	linkLocator := lookup.NewFileLocator(d.logger, d.root)
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.driverRoot),
+	)
 	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
 	if err != nil {
-		return nil, fmt.Errorf("failed to locate devices by path: %v", err)
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
 	}
 
 	var links []string
@@ -171,18 +183,21 @@ func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error
 }
 
 // newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
-func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, root string) (Discover, error) {
+func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, driverRoot string) (Discover, error) {
 	allDevices := NewDeviceDiscoverer(
 		logger,
-		lookup.NewCharDeviceLocator(logger, root),
-		root,
+		lookup.NewCharDeviceLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+		),
+		driverRoot,
 		[]string{
 			"/dev/dri/card*",
 			"/dev/dri/renderD*",
 		},
 	)
 
-	filter, err := newDRMDeviceFilter(logger, devices, root)
+	filter, err := newDRMDeviceFilter(logger, devices, driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
 	}
@@ -198,8 +213,8 @@ func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices,
 }
 
 // newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
-func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, root string) (Filter, error) {
-	gpuInformationPaths, err := proc.GetInformationFilePaths(root)
+func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, driverRoot string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read GPU information: %v", err)
 	}
@@ -233,6 +248,123 @@ func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, roo
 	return filter, nil
 }
 
+type xorgHooks struct {
+	libraries     Discover
+	driverVersion string
+	nvidiaCTKPath string
+}
+
+var _ Discover = (*xorgHooks)(nil)
+
+// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
+// If the creation of the discoverer fails, a None discoverer is returned.
+func optionalXorgDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		logger.Warnf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
+		return None{}
+	}
+	return xorg
+}
+
+func newXorgDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate(".*.*.*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+	libcudaPath := libCudaPaths[0]
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	libRoot := filepath.Dir(libcudaPath)
+	xorgLibs := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"nvidia/xorg/nvidia_drv.so",
+			fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
+		},
+	)
+	xorgHooks := xorgHooks{
+		libraries:     xorgLibs,
+		driverVersion: version,
+		nvidiaCTKPath: FindNvidiaCTK(logger, nvidiaCTKPath),
+	}
+
+	xorgConfg := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/usr/share"),
+		),
+		driverRoot,
+		[]string{"X11/xorg.conf.d/10-nvidia.conf"},
+	)
+
+	d := Merge(
+		xorgLibs,
+		xorgConfg,
+		xorgHooks,
+	)
+
+	return d, nil
+}
+
+// Devices returns no devices for Xorg
+func (m xorgHooks) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a hook to create symlinks for Xorg libraries
+func (m xorgHooks) Hooks() ([]Hook, error) {
+	mounts, err := m.libraries.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get mounts: %v", err)
+	}
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+
+	var target string
+	for _, mount := range mounts {
+		filename := filepath.Base(mount.HostPath)
+		if filename == "libglxserver_nvidia.so."+m.driverVersion {
+			target = mount.Path
+		}
+	}
+
+	if target == "" {
+		return nil, nil
+	}
+
+	link := strings.TrimSuffix(target, "."+m.driverVersion)
+	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
+	symlinkHook := CreateCreateSymlinkHook(
+		m.nvidiaCTKPath,
+		links,
+	)
+
+	return symlinkHook.Hooks()
+}
+
+// Mounts returns the libraries required for Xorg
+func (m xorgHooks) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
 // selectDeviceByPath is a filter that allows devices to be selected by the path
 type selectDeviceByPath map[string]bool
 

internal/discover/hooks.go

@@ -0,0 +1,103 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+)
+
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+// CreateCreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateCreateSymlinkHook(nvidiaCTKPath string, links []string) Discover {
+	if len(links) == 0 {
+		return None{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateNvidiaCTKHook(
+		nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+}
+
+// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
+func CreateNvidiaCTKHook(nvidiaCTKPath string, hookName string, additionalArgs ...string) Hook {
+	return Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      nvidiaCTKPath,
+		Args:      append([]string{filepath.Base(nvidiaCTKPath), "hook", hookName}, additionalArgs...),
+	}
+}
+
+// FindNvidiaCTK locates the nvidia-ctk executable to be used in hooks.
+// If an nvidia-ctk path is specified as an absolute path, it is used directly
+// without checking for existence of an executable at that path.
+func FindNvidiaCTK(logger *logrus.Logger, nvidiaCTKPath string) string {
+	if filepath.IsAbs(nvidiaCTKPath) {
+		logger.Debugf("Using specified NVIDIA Container Toolkit CLI path %v", nvidiaCTKPath)
+		return nvidiaCTKPath
+	}
+
+	if nvidiaCTKPath == "" {
+		nvidiaCTKPath = nvidiaCTKExecutable
+	}
+	logger.Debugf("Locating NVIDIA Container Toolkit CLI as %v", nvidiaCTKPath)
+	lookup := lookup.NewExecutableLocator(logger, "")
+	hookPath := nvidiaCTKDefaultFilePath
+	targets, err := lookup.Locate(nvidiaCTKPath)
+	if err != nil {
+		logger.Warnf("Failed to locate %v: %v", nvidiaCTKPath, err)
+	} else if len(targets) == 0 {
+		logger.Warnf("%v not found", nvidiaCTKPath)
+	} else {
+		logger.Debugf("Found %v candidates: %v", nvidiaCTKPath, targets)
+		hookPath = targets[0]
+	}
+	logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
+
+	return hookPath
+}

internal/discover/icp_test.go

@@ -0,0 +1,60 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIPCMounts(t *testing.T) {
+	l := ipcMounts(
+		mounts{
+			logger: logrus.New(),
+			lookup: &lookup.LocatorMock{
+				LocateFunc: func(path string) ([]string, error) {
+					return []string{"/host/path"}, nil
+				},
+			},
+			required: []string{"target"},
+		},
+	)
+
+	mounts, err := l.Mounts()
+	require.NoError(t, err)
+
+	require.EqualValues(
+		t,
+		[]Mount{
+			{
+				HostPath: "/host/path",
+				Path:     "/host/path",
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+					"noexec",
+				},
+			},
+		},
+		mounts,
+	)
+}

internal/discover/ipc.go

@@ -0,0 +1,78 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type ipcMounts mounts
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger *logrus.Logger, driverRoot string) (Discover, error) {
+	sockets := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/run", "/var/run"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/nvidia-persistenced/socket",
+			"/nvidia-fabricmanager/socket",
+		},
+	)
+
+	mps := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	d := Merge(
+		(*ipcMounts)(sockets),
+		(*ipcMounts)(mps),
+	)
+	return d, nil
+}
+
+// Mounts returns the discovered mounts with "noexec" added to the mount options.
+func (d *ipcMounts) Mounts() ([]Mount, error) {
+	mounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+
+	var modifiedMounts []Mount
+	for _, m := range mounts {
+		mount := m
+		mount.Options = append(m.Options, "noexec")
+		modifiedMounts = append(modifiedMounts, mount)
+	}
+
+	return modifiedMounts, nil
+}

internal/discover/ldconfig.go

@@ -21,33 +21,25 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/sirupsen/logrus"
 )
 
 // NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
 func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
 	d := ldconfig{
-		logger:                  logger,
-		mountsFrom:              mounts,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		mountsFrom:    mounts,
 	}
 
 	return &d, nil
 }
 
-const (
-	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
-)
-
 type ldconfig struct {
 	None
-	logger                  *logrus.Logger
-	mountsFrom              Discover
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	mountsFrom    Discover
 }
 
 // Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
@@ -57,38 +49,27 @@ func (d ldconfig) Hooks() ([]Hook, error) {
 		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
 	}
 	h := CreateLDCacheUpdateHook(
-		d.logger,
-		d.lookup,
-		d.nvidiaCTKExecutablePath,
-		nvidiaCTKDefaultFilePath,
+		d.nvidiaCTKPath,
 		getLibraryPaths(mounts),
 	)
 	return []Hook{h}, nil
 }
 
 // CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
-func CreateLDCacheUpdateHook(logger *logrus.Logger, lookup lookup.Locator, execuable string, defaultPath string, libraries []string) Hook {
-	hookPath := defaultPath
-	targets, err := lookup.Locate(execuable)
-	if err != nil {
-		logger.Warnf("Failed to locate %v: %v", execuable, err)
-	} else if len(targets) == 0 {
-		logger.Warnf("%v not found", execuable)
-	} else {
-		logger.Debugf("Found %v candidates: %v", execuable, targets)
-		hookPath = targets[0]
-	}
-	logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{filepath.Base(hookPath), "hook", "update-ldcache"}
+func CreateLDCacheUpdateHook(executable string, libraries []string) Hook {
+	var args []string
 	for _, f := range uniqueFolders(libraries) {
 		args = append(args, "--folder", f)
 	}
-	return Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
+
+	hook := CreateNvidiaCTKHook(
+		executable,
+		"update-ldcache",
+		args...,
+	)
+
+	return hook
+
 }
 
 // getLibraryPaths extracts the library dirs from the specified mounts

internal/discover/ldconfig_test.go

@@ -24,12 +24,16 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const (
+	testNvidiaCTKPath = "/foo/bar/nvidia-ctk"
+)
+
 func TestLDCacheUpdateHook(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
 	cfg := Config{
-		Root:                                    "/",
-		NVIDIAContainerToolkitCLIExecutablePath: "/foo/bar/nvidia-ctk",
+		DriverRoot:    "/",
+		NvidiaCTKPath: testNvidiaCTKPath,
 	}
 
 	testCases := []struct {
@@ -86,7 +90,7 @@ func TestLDCacheUpdateHook(t *testing.T) {
 				},
 			}
 			expectedHook := Hook{
-				Path:      "/usr/bin/nvidia-ctk",
+				Path:      testNvidiaCTKPath,
 				Args:      tc.expectedArgs,
 				Lifecycle: "createContainer",
 			}

internal/discover/mounts.go

@@ -43,6 +43,11 @@ var _ Discover = (*mounts)(nil)
 
 // NewMounts creates a discoverer for the required mounts using the specified locator.
 func NewMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) Discover {
+	return newMounts(logger, lookup, root, required)
+}
+
+// newMounts creates a discoverer for the required mounts using the specified locator.
+func newMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) *mounts {
 	return &mounts{
 		logger:   logger,
 		lookup:   lookup,
@@ -93,6 +98,12 @@ func (d *mounts) Mounts() ([]Mount, error) {
 			uniqueMounts[p] = Mount{
 				HostPath: p,
 				Path:     r,
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+				},
 			}
 		}
 	}

internal/discover/mounts_test.go

@@ -35,6 +35,14 @@ func TestMountsReturnsEmptyDevices(t *testing.T) {
 }
 
 func TestMounts(t *testing.T) {
+
+	mountOptions := []string{
+		"ro",
+		"nosuid",
+		"nodev",
+		"bind",
+	}
+
 	logger, logHook := testlog.NewNullLogger()
 
 	testCases := []struct {
@@ -70,7 +78,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required"},
 			},
-			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description:   "mounts removes located duplicates",
@@ -83,7 +91,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips located errors",
@@ -98,7 +106,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "error", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips unlocated",
@@ -113,7 +121,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "empty", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts adds multiple",
@@ -129,10 +137,10 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "required0", HostPath: "required0"},
-				{Path: "multiple0", HostPath: "multiple0"},
-				{Path: "multiple1", HostPath: "multiple1"},
-				{Path: "required1", HostPath: "required1"},
+				{Path: "required0", HostPath: "required0", Options: mountOptions},
+				{Path: "multiple0", HostPath: "multiple0", Options: mountOptions},
+				{Path: "multiple1", HostPath: "multiple1", Options: mountOptions},
+				{Path: "required1", HostPath: "required1", Options: mountOptions},
 			},
 		},
 		{
@@ -147,7 +155,7 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "/located", HostPath: "/some/root/located"},
+				{Path: "/located", HostPath: "/some/root/located", Options: mountOptions},
 			},
 		},
 	}

internal/discover/symlinks.go

@@ -21,28 +21,24 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/sirupsen/logrus"
 )
 
 type symlinks struct {
 	None
-	logger                  *logrus.Logger
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
-	csvFiles                []string
-	mountsFrom              Discover
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	csvFiles      []string
+	mountsFrom    Discover
 }
 
 // NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container
 func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) {
 	d := symlinks{
-		logger:                  logger,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
-		csvFiles:                csvFiles,
-		mountsFrom:              mounts,
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		csvFiles:      csvFiles,
+		mountsFrom:    mounts,
 	}
 
 	return &d, nil
@@ -50,19 +46,7 @@ func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Disc
 
 // Hooks returns a hook to create the symlinks from the required CSV files
 func (d symlinks) Hooks() ([]Hook, error) {
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{hookPath, "hook", "create-symlinks"}
+	var args []string
 	for _, f := range d.csvFiles {
 		args = append(args, "--csv-filename", f)
 	}
@@ -73,13 +57,13 @@ func (d symlinks) Hooks() ([]Hook, error) {
 	}
 	args = append(args, links...)
 
-	h := Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
 
-	return []Hook{h}, nil
+	return []Hook{hook}, nil
 }
 
 // getSpecificLinkArgs returns the required specic links that need to be created

internal/dxcore/api.go

@@ -0,0 +1,64 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+import (
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+const (
+	libraryName      = "libdxcore.so"
+	libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+// dxcore stores a reference the dxcore dynamic library
+var dxcore *context
+
+// Init initializes the dxcore dynamic library
+func Init() error {
+	c, err := initContext()
+	if err != nil {
+		return err
+	}
+	dxcore = c
+	return nil
+}
+
+// Shutdown closes the dxcore dynamic library
+func Shutdown() error {
+	if dxcore != nil && dxcore.initialized != 0 {
+		dxcore.deinitContext()
+	}
+	return nil
+}
+
+// GetDriverStorePaths returns the list of driver store paths
+func GetDriverStorePaths() []string {
+	var paths []string
+	selected := make(map[string]bool)
+
+	for i := 0; i < dxcore.getAdapterCount(); i++ {
+		path := dxcore.getAdapter(i).getDriverStorePath()
+		if selected[path] {
+			continue
+		}
+		selected[path] = true
+		paths = append(paths, path)
+	}
+
+	return paths
+}

internal/dxcore/dxcore.c

@@ -0,0 +1,334 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include "dxcore.h"
+
+// We define log_write as an empty macro to allow dxcore to remain unchanged.
+#define log_write(...)
+
+// We define the following macros to allow dxcore to remain largely unchanged.
+#define log_info(msg) log_write('I', __FILE__, __LINE__, msg)
+#define log_warn(msg) log_write('W', __FILE__, __LINE__, msg)
+#define log_err(msg)  log_write('E', __FILE__, __LINE__, msg)
+#define log_infof(fmt, ...) log_write('I', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_warnf(fmt, ...) log_write('W', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_errf(fmt, ...)  log_write('E', __FILE__, __LINE__, fmt, __VA_ARGS__)
+
+
+#define DXCORE_MAX_PATH 260
+
+/*
+ * List of components we expect to find in the driver store that we need to mount
+ */
+static const char * const dxcore_nvidia_driver_store_components[] = {
+        "libcuda.so.1.1",                   /* Core library for cuda support */
+        "libcuda_loader.so",                /* Core library for cuda support on WSL */
+        "libnvidia-ptxjitcompiler.so.1",    /* Core library for PTX Jit support */
+        "libnvidia-ml.so.1",                /* Core library for nvml */
+        "libnvidia-ml_loader.so",           /* Core library for nvml on WSL */
+        "nvidia-smi",                       /* nvidia-smi binary*/
+        "nvcubins.bin",                     /* Binary containing GPU code for cuda */
+};
+
+
+/*
+ * List of functions and structures we need to communicate with libdxcore.
+ * Documentation on these functions can be found on docs.microsoft.com in d3dkmthk.
+ */
+
+struct dxcore_enumAdapters2;
+struct dxcore_queryAdapterInfo;
+
+typedef int(*pfnDxcoreEnumAdapters2)(struct dxcore_enumAdapters2* pParams);
+typedef int(*pfnDxcoreQueryAdapterInfo)(struct dxcore_queryAdapterInfo* pParams);
+
+struct dxcore_lib {
+        void* hDxcoreLib;
+        pfnDxcoreEnumAdapters2 pDxcoreEnumAdapters2;
+        pfnDxcoreQueryAdapterInfo pDxcoreQueryAdapterInfo;
+};
+
+struct dxcore_adapterInfo
+{
+        unsigned int              hAdapter;
+        struct dxcore_luid        AdapterLuid;
+        unsigned int              NumOfSources;
+        unsigned int              bPresentMoveRegionsPreferred;
+};
+
+struct dxcore_enumAdapters2
+{
+        unsigned int                   NumAdapters;
+        struct dxcore_adapterInfo     *pAdapters;
+};
+
+enum dxcore_kmtqueryAdapterInfoType
+{
+        DXCORE_QUERYDRIVERVERSION = 13,
+        DXCORE_QUERYREGISTRY = 48,
+};
+
+enum dxcore_queryregistry_type {
+        DXCORE_QUERYREGISTRY_DRIVERSTOREPATH = 2,
+        DXCORE_QUERYREGISTRY_DRIVERIMAGEPATH = 3,
+};
+
+enum dxcore_queryregistry_status {
+        DXCORE_QUERYREGISTRY_STATUS_SUCCESS = 0,
+        DXCORE_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW = 1,
+        DXCORE_QUERYREGISTRY_STATUS_FAIL = 2,
+};
+
+struct dxcore_queryregistry_info {
+        enum dxcore_queryregistry_type        QueryType;
+        unsigned int                          QueryFlags;
+        wchar_t                               ValueName[DXCORE_MAX_PATH];
+        unsigned int                          ValueType;
+        unsigned int                          PhysicalAdapterIndex;
+        unsigned int                          OutputValueSize;
+        enum dxcore_queryregistry_status      Status;
+        union {
+                unsigned long long                    OutputQword;
+                wchar_t                               Output;
+        };
+};
+
+struct dxcore_queryAdapterInfo
+{
+        unsigned int                           hAdapter;
+        enum dxcore_kmtqueryAdapterInfoType    Type;
+        void                                   *pPrivateDriverData;
+        unsigned int                           PrivateDriverDataSize;
+};
+
+static int dxcore_query_adapter_info_helper(struct dxcore_lib* pLib,
+                                            unsigned int hAdapter,
+                                            enum dxcore_kmtqueryAdapterInfoType type,
+                                            void* pPrivateDriverDate,
+                                            unsigned int privateDriverDataSize)
+{
+        struct dxcore_queryAdapterInfo queryAdapterInfo = { 0 };
+
+        queryAdapterInfo.hAdapter = hAdapter;
+        queryAdapterInfo.Type = type;
+        queryAdapterInfo.pPrivateDriverData = pPrivateDriverDate;
+        queryAdapterInfo.PrivateDriverDataSize = privateDriverDataSize;
+
+        return pLib->pDxcoreQueryAdapterInfo(&queryAdapterInfo);
+}
+
+static int dxcore_query_adapter_wddm_version(struct dxcore_lib* pLib, unsigned int hAdapter, unsigned int* version)
+{
+        return dxcore_query_adapter_info_helper(pLib,
+                                                hAdapter,
+                                                DXCORE_QUERYDRIVERVERSION,
+                                                (void*)version,
+                                                sizeof(*version));
+}
+
+static int dxcore_query_adapter_driverstore(struct dxcore_lib* pLib, unsigned int hAdapter, char** ppDriverStorePath)
+{
+        struct dxcore_queryregistry_info params = {0};
+        struct dxcore_queryregistry_info* pValue = NULL;
+        wchar_t* pOutput;
+        size_t outputSizeInBytes;
+        size_t outputSize;
+
+        params.QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                             hAdapter,
+                                             DXCORE_QUERYREGISTRY,
+                                             (void*)&params,
+                                             sizeof(params)))
+        {
+                log_err("Failed to query driver store path size for the WDDM Adapter");
+                return (-1);
+        }
+
+        if (params.OutputValueSize > DXCORE_MAX_PATH * sizeof(wchar_t)) {
+                log_err("The driver store path size returned by dxcore is not valid");
+                return (-1);
+        }
+
+        outputSizeInBytes = (size_t)params.OutputValueSize;
+        outputSize = outputSizeInBytes / sizeof(wchar_t);
+
+        pValue = calloc(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes + sizeof(wchar_t), 1);
+        if (!pValue) {
+                log_err("Out of memory while allocating temp buffer to query adapter info");
+                return (-1);
+        }
+
+        pValue->QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+        pValue->OutputValueSize = (unsigned int)outputSizeInBytes;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                            hAdapter,
+                                            DXCORE_QUERYREGISTRY,
+                                            (void*)pValue,
+                                            (unsigned int)(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes)))
+        {
+                log_err("Failed to query driver store path data for the WDDM Adapter");
+                free(pValue);
+                return (-1);
+        }
+        pOutput = (wchar_t*)(&pValue->Output);
+
+        // Make sure no matter what happened the wchar_t string is null terminated
+        pOutput[outputSize] = L'\0';
+
+        // Convert the output into a regular c string
+        *ppDriverStorePath = (char*)calloc(outputSize + 1, sizeof(char));
+        if (!*ppDriverStorePath) {
+                log_err("Out of memory while allocating the buffer for the driver store path");
+                free(pValue);
+                return (-1);
+        }
+        wcstombs(*ppDriverStorePath, pOutput, outputSize);
+
+        free(pValue);
+
+    return 0;
+}
+
+static void dxcore_add_adapter(struct dxcore_context* pCtx, struct dxcore_lib* pLib, struct dxcore_adapterInfo *pAdapterInfo)
+{
+        unsigned int wddmVersion = 0;
+        char* driverStorePath = NULL;
+
+        log_infof("Creating a new WDDM Adapter for hAdapter:%x luid:%llx", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid));
+
+        if (dxcore_query_adapter_wddm_version(pLib, pAdapterInfo->hAdapter, &wddmVersion)) {
+                log_err("Failed to query the WDDM version for the specified adapter. Skipping it.");
+                return;
+        }
+
+        if (wddmVersion < 2700) {
+                log_err("Found a WDDM adapter running a driver with pre-WDDM 2.7 . Skipping it.");
+                return;
+        }
+
+        if (dxcore_query_adapter_driverstore(pLib, pAdapterInfo->hAdapter, &driverStorePath)) {
+                log_err("Failed to query driver store path for the WDDM Adapter . Skipping it.");
+                return;
+        }
+
+        // We got all the info we needed. Adding it to the tracking structure.
+        {
+                struct dxcore_adapter* newList;
+                newList = realloc(pCtx->adapterList, sizeof(struct dxcore_adapter) * (pCtx->adapterCount + 1));
+                if (!newList) {
+                        log_err("Out of memory when trying to add a new WDDM Adapter to the list of valid adapters");
+                        free(driverStorePath);
+                        return;
+                }
+
+                pCtx->adapterList = newList;
+
+                pCtx->adapterList[pCtx->adapterCount].hAdapter = pAdapterInfo->hAdapter;
+                pCtx->adapterList[pCtx->adapterCount].pDriverStorePath = driverStorePath;
+                pCtx->adapterList[pCtx->adapterCount].wddmVersion = wddmVersion;
+                pCtx->adapterCount++;
+        }
+
+        log_infof("Adding new adapter via dxcore hAdapter:%x luid:%llx wddm version:%d", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid), wddmVersion);
+}
+
+static void dxcore_enum_adapters(struct dxcore_context* pCtx, struct dxcore_lib* pLib)
+{
+        struct dxcore_enumAdapters2 params = {0};
+        unsigned int adapterIndex = 0;
+
+        params.NumAdapters = 0;
+        params.pAdapters = NULL;
+
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        params.pAdapters = malloc(sizeof(struct dxcore_adapterInfo) * params.NumAdapters);
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                free(params.pAdapters);
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        for (adapterIndex = 0; adapterIndex < params.NumAdapters; adapterIndex++) {
+                dxcore_add_adapter(pCtx, pLib, &params.pAdapters[adapterIndex]);
+        }
+
+        free(params.pAdapters);
+}
+
+int dxcore_init_context(struct dxcore_context* pCtx)
+{
+        struct dxcore_lib lib = {0};
+
+        pCtx->initialized = 0;
+        pCtx->adapterCount = 0;
+        pCtx->adapterList = NULL;
+
+        lib.hDxcoreLib = dlopen("libdxcore.so", RTLD_LAZY);
+        if (!lib.hDxcoreLib) {
+                goto error;
+        }
+
+        lib.pDxcoreEnumAdapters2 = (pfnDxcoreEnumAdapters2)dlsym(lib.hDxcoreLib, "D3DKMTEnumAdapters2");
+        if (!lib.pDxcoreEnumAdapters2) {
+                log_err("dxcore library is present but the symbol D3DKMTEnumAdapters2 is missing");
+                goto error;
+        }
+
+        lib.pDxcoreQueryAdapterInfo = (pfnDxcoreQueryAdapterInfo)dlsym(lib.hDxcoreLib, "D3DKMTQueryAdapterInfo");
+        if (!lib.pDxcoreQueryAdapterInfo) {
+                log_err("dxcore library is present but the symbol D3DKMTQueryAdapterInfo is missing");
+                goto error;
+        }
+
+        dxcore_enum_adapters(pCtx, &lib);
+
+        log_info("dxcore layer initialized successfully");
+        pCtx->initialized = 1;
+
+        dlclose(lib.hDxcoreLib);
+
+        return 0;
+
+error:
+        dxcore_deinit_context(pCtx);
+
+        if (lib.hDxcoreLib)
+                dlclose(lib.hDxcoreLib);
+
+        return (-1);
+}
+
+static void dxcore_deinit_adapter(struct dxcore_adapter* pAdapter)
+{
+        if (!pAdapter)
+            return;
+
+        free(pAdapter->pDriverStorePath);
+}
+
+void dxcore_deinit_context(struct dxcore_context* pCtx)
+{
+        unsigned int adapterIndex = 0;
+
+        if (!pCtx)
+                return;
+
+        for (adapterIndex = 0; adapterIndex < pCtx->adapterCount; adapterIndex++) {
+                dxcore_deinit_adapter(&pCtx->adapterList[adapterIndex]);
+        }
+
+        free(pCtx->adapterList);
+
+        pCtx->initialized = 0;
+}

internal/dxcore/dxcore.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+/*
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
+#include <dxcore.h>
+*/
+import "C"
+import (
+	"fmt"
+	"unsafe"
+)
+
+type context C.struct_dxcore_context
+type adapter C.struct_dxcore_adapter
+
+// initContext initializes the dxcore context and populates the list of adapters.
+func initContext() (*context, error) {
+	cContext := C.struct_dxcore_context{}
+	if C.dxcore_init_context(&cContext) != 0 {
+		return nil, fmt.Errorf("failed to initialize dxcore context")
+	}
+	c := (*context)(&cContext)
+	return c, nil
+}
+
+// deinitContext deinitializes the dxcore context and frees the list of adapters.
+func (c context) deinitContext() {
+	cContext := C.struct_dxcore_context(c)
+	C.dxcore_deinit_context(&cContext)
+}
+
+func (c context) getAdapterCount() int {
+	return int(c.adapterCount)
+}
+
+func (c context) getAdapter(index int) adapter {
+	arrayPointer := (*[1 << 30]C.struct_dxcore_adapter)(unsafe.Pointer(c.adapterList))
+	return adapter(arrayPointer[index])
+}
+
+func (a adapter) getDriverStorePath() string {
+	return C.GoString(a.pDriverStorePath)
+}

internal/dxcore/dxcore.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#ifndef HEADER_DXCORE_H_
+#define HEADER_DXCORE_H_
+
+#define MAX_DXCORE_DRIVERSTORE_LIBRAIRIES (16)
+
+struct dxcore_luid
+{
+        unsigned int lowPart;
+        int highPart;
+};
+
+struct dxcore_adapter
+{
+        unsigned int             hAdapter;
+        unsigned int             wddmVersion;
+        char*                    pDriverStorePath;
+        unsigned int             driverStoreComponentCount;
+        const char*              pDriverStoreComponents[MAX_DXCORE_DRIVERSTORE_LIBRAIRIES];
+        struct dxcore_context    *pContext;
+};
+
+struct dxcore_context
+{
+        unsigned int adapterCount;
+        struct dxcore_adapter *adapterList;
+
+        int initialized;
+};
+
+
+
+int dxcore_init_context(struct dxcore_context* pDxcore_context);
+void dxcore_deinit_context(struct dxcore_context* pDxcore_context);
+
+#endif // HEADER_DXCORE_H_

internal/edits/device.go

@@ -17,13 +17,9 @@
 package edits
 
 import (
-	"fmt"
-
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
-
-	"github.com/opencontainers/runc/libcontainer/devices"
 )
 
 type device discover.Device
@@ -46,21 +42,18 @@ func (d device) toEdits() (*cdi.ContainerEdits, error) {
 // toSpec converts a discovered Device to a CDI Spec Device. Note
 // that missing info is filled in when edits are applied by querying the Device node.
 func (d device) toSpec() (*specs.DeviceNode, error) {
-	// NOTE: This mirrors what cri-o does.
-	// https://github.com/cri-o/cri-o/blob/ca3bb80a3dda0440659fcf8da8ed6f23211de94e/internal/config/device/device.go#L93
-	// This can be removed once https://github.com/container-orchestrated-devices/container-device-interface/issues/72 is addressed
-	dev, err := devices.DeviceFromPath(d.HostPath, "rwm")
-	if err != nil {
-		return nil, fmt.Errorf("failed to query device node %v: %v", d.HostPath, err)
+	// The HostPath field was added in the v0.5.0 CDI specification.
+	// The cdi package uses strict unmarshalling when loading specs from file causing failures for
+	// unexpected fields.
+	// Since the behaviour for HostPath == "" and HostPath == Path are equivalent, we clear HostPath
+	// if it is equal to Path to ensure compatibility with the widest range of specs.
+	hostPath := d.HostPath
+	if hostPath == d.Path {
+		hostPath = ""
 	}
 	s := specs.DeviceNode{
+		HostPath: hostPath,
 		Path:     d.Path,
-		Type:     string(dev.Type),
-		Major:    dev.Major,
-		Minor:    dev.Minor,
-		FileMode: &dev.FileMode,
-		UID:      &dev.Uid,
-		GID:      &dev.Gid,
 	}
 
 	return &s, nil

internal/edits/device_test.go

@@ -0,0 +1,69 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeviceToSpec(t *testing.T) {
+	testCases := []struct {
+		device   discover.Device
+		expected *specs.DeviceNode
+	}{
+		{
+			device: discover.Device{
+				Path: "/foo",
+			},
+			expected: &specs.DeviceNode{
+				Path: "/foo",
+			},
+		},
+		{
+			device: discover.Device{
+				Path:     "/foo",
+				HostPath: "/foo",
+			},
+			expected: &specs.DeviceNode{
+				Path: "/foo",
+			},
+		},
+		{
+			device: discover.Device{
+				Path:     "/foo",
+				HostPath: "/not/foo",
+			},
+			expected: &specs.DeviceNode{
+				Path:     "/foo",
+				HostPath: "/not/foo",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			spec, err := device(tc.device).toSpec()
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expected, spec)
+		})
+	}
+}

internal/edits/edits.go

@@ -22,6 +22,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	ociSpecs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
@@ -34,6 +35,20 @@ type edits struct {
 // NewSpecEdits creates a SpecModifier that defines the required OCI spec edits (as CDI ContainerEdits) from the specified
 // discoverer.
 func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	c, err := FromDiscoverer(d)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing container edits: %v", err)
+	}
+	e := edits{
+		ContainerEdits: *c,
+		logger:         logger,
+	}
+
+	return &e, nil
+}
+
+// FromDiscoverer creates CDI container edits for the specified discoverer.
+func FromDiscoverer(d discover.Discover) (*cdi.ContainerEdits, error) {
 	devices, err := d.Devices()
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover devices: %v", err)
@@ -49,7 +64,7 @@ func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier,
 		return nil, fmt.Errorf("failed to discover hooks: %v", err)
 	}
 
-	c := cdi.ContainerEdits{}
+	c := NewContainerEdits()
 	for _, d := range devices {
 		edits, err := device(d).toEdits()
 		if err != nil {
@@ -66,12 +81,15 @@ func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier,
 		c.Append(hook(h).toEdits())
 	}
 
-	e := edits{
-		ContainerEdits: c,
-		logger:         logger,
-	}
+	return c, nil
+}
 
-	return &e, nil
+// NewContainerEdits is a utility function to create a CDI ContainerEdits struct.
+func NewContainerEdits() *cdi.ContainerEdits {
+	c := cdi.ContainerEdits{
+		ContainerEdits: &specs.ContainerEdits{},
+	}
+	return &c
 }
 
 // Modify applies the defined edits to the incoming OCI spec
@@ -90,7 +108,7 @@ func (e *edits) Modify(spec *ociSpecs.Spec) error {
 	}
 	e.logger.Infof("Hooks:")
 	for _, hook := range e.Hooks {
-		e.logger.Infof("Injecting %v", hook.Args)
+		e.logger.Infof("Injecting %v %v", hook.Path, hook.Args)
 	}
 
 	return e.Apply(spec)

internal/edits/edits_test.go

@@ -0,0 +1,31 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFromDiscovererAllowsMountsToIterate(t *testing.T) {
+	edits, err := FromDiscoverer(discover.None{})
+	require.NoError(t, err)
+
+	require.Empty(t, edits.Mounts)
+}

internal/edits/mount.go

@@ -40,12 +40,7 @@ func (d mount) toSpec() *specs.Mount {
 	s := specs.Mount{
 		HostPath:      d.HostPath,
 		ContainerPath: d.Path,
-		Options: []string{
-			"ro",
-			"nosuid",
-			"nodev",
-			"bind",
-		},
+		Options:       d.Options,
 	}
 
 	return &s

internal/info/proc/devices/devices.go

@@ -0,0 +1,125 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package devices
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// Device major numbers and device names for NVIDIA devices
+const (
+	NVIDIAUVMMinor      = 0
+	NVIDIAUVMToolsMinor = 1
+	NVIDIACTLMinor      = 255
+	NVIDIAModesetMinor  = 254
+
+	NVIDIAFrontend = Name("nvidia-frontend")
+	NVIDIAGPU      = NVIDIAFrontend
+	NVIDIACaps     = Name("nvidia-caps")
+	NVIDIAUVM      = Name("nvidia-uvm")
+
+	procDevicesPath    = "/proc/devices"
+	nvidiaDevicePrefix = "nvidia"
+)
+
+// Name represents the name of a device as specified under /proc/devices
+type Name string
+
+// Major represents a device major as specified under /proc/devices
+type Major int
+
+// Devices represents the set of devices under /proc/devices
+//
+//go:generate moq -stub -out devices_mock.go . Devices
+type Devices interface {
+	Exists(Name) bool
+	Get(Name) (Major, bool)
+}
+
+type devices map[Name]Major
+
+var _ Devices = devices(nil)
+
+// Exists checks if a Device with a given name exists or not
+func (d devices) Exists(name Name) bool {
+	_, exists := d[name]
+	return exists
+}
+
+// Get a Device from Devices
+func (d devices) Get(name Name) (Major, bool) {
+	device, exists := d[name]
+	return device, exists
+}
+
+// GetNVIDIADevices returns the set of NVIDIA Devices on the machine
+func GetNVIDIADevices() (Devices, error) {
+	devicesFile, err := os.Open(procDevicesPath)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error opening devices file: %v", err)
+	}
+	defer devicesFile.Close()
+
+	return nvidiaDeviceFrom(devicesFile), nil
+}
+
+func nvidiaDeviceFrom(reader io.Reader) devices {
+	allDevices := devicesFrom(reader)
+	nvidiaDevices := make(devices)
+	for n, d := range allDevices {
+		if !strings.HasPrefix(string(n), nvidiaDevicePrefix) {
+			continue
+		}
+		nvidiaDevices[n] = d
+	}
+
+	return nvidiaDevices
+}
+
+func devicesFrom(reader io.Reader) devices {
+	allDevices := make(devices)
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		device, major, err := processProcDeviceLine(scanner.Text())
+		if err != nil {
+			continue
+		}
+		allDevices[device] = major
+	}
+	return allDevices
+}
+
+func processProcDeviceLine(line string) (Name, Major, error) {
+	trimmed := strings.TrimSpace(line)
+
+	var name Name
+	var major Major
+
+	n, _ := fmt.Sscanf(trimmed, "%d %s", &major, &name)
+	if n == 2 {
+		return name, major, nil
+	}
+
+	return "", 0, fmt.Errorf("unparsable line: %v", line)
+}

internal/info/proc/devices/devices_mock.go

@@ -0,0 +1,125 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package devices
+
+import (
+	"sync"
+)
+
+// Ensure, that DevicesMock does implement Devices.
+// If this is not the case, regenerate this file with moq.
+var _ Devices = &DevicesMock{}
+
+// DevicesMock is a mock implementation of Devices.
+//
+//	func TestSomethingThatUsesDevices(t *testing.T) {
+//
+//		// make and configure a mocked Devices
+//		mockedDevices := &DevicesMock{
+//			ExistsFunc: func(name Name) bool {
+//				panic("mock out the Exists method")
+//			},
+//			GetFunc: func(name Name) (Major, bool) {
+//				panic("mock out the Get method")
+//			},
+//		}
+//
+//		// use mockedDevices in code that requires Devices
+//		// and then make assertions.
+//
+//	}
+type DevicesMock struct {
+	// ExistsFunc mocks the Exists method.
+	ExistsFunc func(name Name) bool
+
+	// GetFunc mocks the Get method.
+	GetFunc func(name Name) (Major, bool)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Exists holds details about calls to the Exists method.
+		Exists []struct {
+			// Name is the name argument value.
+			Name Name
+		}
+		// Get holds details about calls to the Get method.
+		Get []struct {
+			// Name is the name argument value.
+			Name Name
+		}
+	}
+	lockExists sync.RWMutex
+	lockGet    sync.RWMutex
+}
+
+// Exists calls ExistsFunc.
+func (mock *DevicesMock) Exists(name Name) bool {
+	callInfo := struct {
+		Name Name
+	}{
+		Name: name,
+	}
+	mock.lockExists.Lock()
+	mock.calls.Exists = append(mock.calls.Exists, callInfo)
+	mock.lockExists.Unlock()
+	if mock.ExistsFunc == nil {
+		var (
+			bOut bool
+		)
+		return bOut
+	}
+	return mock.ExistsFunc(name)
+}
+
+// ExistsCalls gets all the calls that were made to Exists.
+// Check the length with:
+//
+//	len(mockedDevices.ExistsCalls())
+func (mock *DevicesMock) ExistsCalls() []struct {
+	Name Name
+} {
+	var calls []struct {
+		Name Name
+	}
+	mock.lockExists.RLock()
+	calls = mock.calls.Exists
+	mock.lockExists.RUnlock()
+	return calls
+}
+
+// Get calls GetFunc.
+func (mock *DevicesMock) Get(name Name) (Major, bool) {
+	callInfo := struct {
+		Name Name
+	}{
+		Name: name,
+	}
+	mock.lockGet.Lock()
+	mock.calls.Get = append(mock.calls.Get, callInfo)
+	mock.lockGet.Unlock()
+	if mock.GetFunc == nil {
+		var (
+			majorOut Major
+			bOut     bool
+		)
+		return majorOut, bOut
+	}
+	return mock.GetFunc(name)
+}
+
+// GetCalls gets all the calls that were made to Get.
+// Check the length with:
+//
+//	len(mockedDevices.GetCalls())
+func (mock *DevicesMock) GetCalls() []struct {
+	Name Name
+} {
+	var calls []struct {
+		Name Name
+	}
+	mock.lockGet.RLock()
+	calls = mock.calls.Get
+	mock.lockGet.RUnlock()
+	return calls
+}

internal/info/proc/devices/devices_test.go

@@ -0,0 +1,102 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package devices
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNvidiaDevices(t *testing.T) {
+	devices := map[Name]Major{
+		"nvidia-frontend": 195,
+		"nvidia-nvlink":   234,
+		"nvidia-caps":     235,
+		"nvidia-uvm":      510,
+		"nvidia-nvswitch": 511,
+	}
+
+	nvidiaDevices := testDevices(devices)
+	for name, major := range devices {
+		device, exists := nvidiaDevices.Get(name)
+		require.True(t, exists, "Unexpected missing device")
+		require.Equal(t, device, major, "Unexpected device major")
+	}
+	_, exists := nvidiaDevices.Get("bogus")
+	require.False(t, exists, "Unexpected 'bogus' device found")
+}
+
+func TestProcessDeviceFile(t *testing.T) {
+	testCases := []struct {
+		lines    []string
+		expected devices
+	}{
+		{[]string{}, make(devices)},
+		{[]string{"Not a valid line:"}, make(devices)},
+		{[]string{"195 nvidia-frontend"}, devices{"nvidia-frontend": 195}},
+		{[]string{"195 nvidia-frontend", "235 nvidia-caps"}, devices{"nvidia-frontend": 195, "nvidia-caps": 235}},
+		{[]string{"  195 nvidia-frontend"}, devices{"nvidia-frontend": 195}},
+		{[]string{"Not a valid line:", "", "195 nvidia-frontend"}, devices{"nvidia-frontend": 195}},
+		{[]string{"195 not-nvidia-frontend"}, make(devices)},
+	}
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("testcase %d", i), func(t *testing.T) {
+			contents := strings.NewReader(strings.Join(tc.lines, "\n"))
+			d := nvidiaDeviceFrom(contents)
+
+			require.EqualValues(t, tc.expected, d)
+		})
+	}
+}
+
+func TestProcessDeviceFileLine(t *testing.T) {
+	testCases := []struct {
+		line  string
+		name  Name
+		major Major
+		err   bool
+	}{
+		{"", "", 0, true},
+		{"0", "", 0, true},
+		{"notint nvidia-frontend", "", 0, true},
+		{"195 nvidia-frontend", "nvidia-frontend", 195, false},
+		{"   195 nvidia-frontend", "nvidia-frontend", 195, false},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("testcase %d", i), func(t *testing.T) {
+			name, major, err := processProcDeviceLine(tc.line)
+
+			require.Equal(t, tc.name, name)
+			require.Equal(t, tc.major, major)
+			if tc.err {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+		})
+	}
+}
+
+// testDevices creates a set of test NVIDIA devices
+func testDevices(d map[Name]Major) Devices {
+	return devices(d)
+}

internal/ldcache/ldcache.go

@@ -22,8 +22,10 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"unsafe"
 
@@ -177,13 +179,15 @@ func (c *ldcache) parse() error {
 	return nil
 }
 
-// List creates a list of libraires in the ldcache.
-// The 32-bit and 64-bit libraries are returned separately.
-func (c *ldcache) List() ([]string, []string) {
-	paths := make(map[int][]string)
-
-	processed := make(map[string]bool)
+type entry struct {
+	libname string
+	bits    int
+	value   string
+}
 
+// getEntries returns the entires of the ldcache in a go-friendly struct.
+func (c *ldcache) getEntries(selected func(string) bool) []entry {
+	var entries []entry
 	for _, e := range c.entries {
 		bits := 0
 		if ((e.Flags & flagTypeMask) & flagTypeELF) == 0 {
@@ -204,89 +208,115 @@ func (c *ldcache) List() ([]string, []string) {
 		if e.Key > uint32(len(c.libs)) || e.Value > uint32(len(c.libs)) {
 			continue
 		}
-		value := c.libs[e.Value:]
-
-		n := bytes.IndexByte(value, 0)
-		if n < 0 {
-			break
+		lib := bytesToString(c.libs[e.Key:])
+		if lib == "" {
+			c.logger.Debugf("Skipping invalid lib")
+			continue
+		}
+		if !selected(lib) {
+			continue
+		}
+		value := bytesToString(c.libs[e.Value:])
+		if value == "" {
+			c.logger.Debugf("Skipping invalid value for lib %v", lib)
+			continue
+		}
+		e := entry{
+			libname: lib,
+			bits:    bits,
+			value:   value,
 		}
 
-		name := filepath.Join(c.root, strn(value, n))
-		c.logger.Debugf("checking %v", string(name))
+		entries = append(entries, e)
+	}
 
-		path, err := filepath.EvalSymlinks(name)
+	return entries
+}
+
+// List creates a list of libraires in the ldcache.
+// The 32-bit and 64-bit libraries are returned separately.
+func (c *ldcache) List() ([]string, []string) {
+	all := func(s string) bool { return true }
+
+	return c.resolveSelected(all)
+}
+
+// Lookup searches the ldcache for the specified prefixes.
+// The 32-bit and 64-bit libraries matching the prefixes are returned.
+func (c *ldcache) Lookup(libPrefixes ...string) ([]string, []string) {
+	c.logger.Debugf("Looking up %v in cache", libPrefixes)
+
+	// We define a functor to check whether a given library name matches any of the prefixes
+	matchesAnyPrefix := func(s string) bool {
+		for _, p := range libPrefixes {
+			if strings.HasPrefix(s, p) {
+				return true
+			}
+		}
+		return false
+	}
+
+	return c.resolveSelected(matchesAnyPrefix)
+}
+
+// resolveSelected process the entries in the LDCach based on the supplied filter and returns the resolved paths.
+// The paths are separated by bittage.
+func (c *ldcache) resolveSelected(selected func(string) bool) ([]string, []string) {
+	paths := make(map[int][]string)
+	processed := make(map[string]bool)
+
+	for _, e := range c.getEntries(selected) {
+		path, err := c.resolve(e.value)
 		if err != nil {
-			c.logger.Debugf("could not resolve symlink for %v", name)
-			break
+			c.logger.Debugf("Could not resolve entry: %v", err)
+			continue
 		}
 		if processed[path] {
 			continue
 		}
-		paths[bits] = append(paths[bits], path)
+		paths[e.bits] = append(paths[e.bits], path)
 		processed[path] = true
 	}
 
 	return paths[32], paths[64]
 }
 
-// Lookup searches the ldcache for the specified prefixes.
-// The 32-bit and 64-bit libraries matching the prefixes are returned.
-func (c *ldcache) Lookup(libs ...string) (paths32, paths64 []string) {
-	c.logger.Debugf("Looking up %v in cache", libs)
-	type void struct{}
-	var paths *[]string
+// resolve resolves the specified ldcache entry based on the value being processed.
+// The input is the name of the entry in the cache.
+func (c *ldcache) resolve(target string) (string, error) {
+	name := filepath.Join(c.root, target)
 
-	set := make(map[string]void)
-	prefix := make([][]byte, len(libs))
+	c.logger.Debugf("checking %v", string(name))
 
-	for i := range libs {
-		prefix[i] = []byte(libs[i])
+	info, err := os.Lstat(name)
+	if err != nil {
+		return "", fmt.Errorf("failed to get file info: %v", info)
 	}
-	for _, e := range c.entries {
-		if ((e.Flags & flagTypeMask) & flagTypeELF) == 0 {
-			continue
-		}
-		switch e.Flags & flagArchMask {
-		case flagArchX8664:
-			fallthrough
-		case flagArchPpc64le:
-			paths = &paths64
-		case flagArchX32:
-			fallthrough
-		case flagArchI386:
-			paths = &paths32
-		default:
-			continue
-		}
-		if e.Key > uint32(len(c.libs)) || e.Value > uint32(len(c.libs)) {
-			continue
-		}
-		lib := c.libs[e.Key:]
-		value := c.libs[e.Value:]
-
-		for _, p := range prefix {
-			if bytes.HasPrefix(lib, p) {
-				n := bytes.IndexByte(value, 0)
-				if n < 0 {
-					break
-				}
-
-				name := filepath.Join(c.root, strn(value, n))
-				c.logger.Debugf("checking %v", string(name))
-
-				path, err := filepath.EvalSymlinks(name)
-				if err != nil {
-					c.logger.Debugf("could not resolve symlink for %v", name)
-					break
-				}
-				if _, ok := set[path]; ok {
-					break
-				}
-				set[path] = void{}
-				*paths = append(*paths, path)
-				break
-			}
-		}
+	if info.Mode()&os.ModeSymlink == 0 {
+		c.logger.Debugf("Resolved regular file: %v", name)
+		return name, nil
 	}
-	return
+
+	link, err := os.Readlink(name)
+	if err != nil {
+		return "", fmt.Errorf("failed to resolve symlink: %v", err)
+	}
+
+	// We return absolute paths for all targets
+	if !filepath.IsAbs(link) || strings.HasPrefix(link, ".") {
+		link = filepath.Join(filepath.Dir(target), link)
+	}
+
+	return c.resolve(link)
+}
+
+// bytesToString converts a byte slice to a string.
+// This assumes that the byte slice is null-terminated
+func bytesToString(value []byte) string {
+	n := bytes.IndexByte(value, 0)
+	if n < 0 {
+		return ""
+	}
+
+	return strn(value, n)
 }

internal/lookup/cuda/cuda.go

@@ -0,0 +1,104 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cuda
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type cudaLocator struct {
+	logger     *logrus.Logger
+	driverRoot string
+}
+
+// Options is a function that configures a cudaLocator.
+type Options func(*cudaLocator)
+
+// WithLogger is an option that configures the logger used by the locator.
+func WithLogger(logger *logrus.Logger) Options {
+	return func(c *cudaLocator) {
+		c.logger = logger
+	}
+}
+
+// WithDriverRoot is an option that configures the driver root used by the locator.
+func WithDriverRoot(driverRoot string) Options {
+	return func(c *cudaLocator) {
+		c.driverRoot = driverRoot
+	}
+}
+
+// New creates a new CUDA library locator.
+func New(opts ...Options) lookup.Locator {
+	c := &cudaLocator{}
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	if c.logger == nil {
+		c.logger = logrus.StandardLogger()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+
+	return c
+}
+
+// Locate returns the path to the libcuda.so.RMVERSION file.
+// libcuda.so is prefixed to the specified pattern.
+func (l *cudaLocator) Locate(pattern string) ([]string, error) {
+	ldcacheLocator, err := lookup.NewLibraryLocator(
+		l.logger,
+		l.driverRoot,
+	)
+	if err != nil {
+		l.logger.Debugf("Failed to create LDCache locator: %v", err)
+	}
+
+	fullPattern := "libcuda.so" + pattern
+
+	candidates, err := ldcacheLocator.Locate("libcuda.so")
+	if err == nil {
+		for _, c := range candidates {
+			if match, err := filepath.Match(fullPattern, filepath.Base(c)); err != nil || !match {
+				l.logger.Debugf("Skipping non-matching candidate %v: %v", c, err)
+				continue
+			}
+			return []string{c}, nil
+		}
+	}
+	l.logger.Debugf("Could not locate %q in LDCache: Checking predefined library paths.", pattern)
+
+	pathLocator := lookup.NewFileLocator(
+		lookup.WithLogger(l.logger),
+		lookup.WithRoot(l.driverRoot),
+		lookup.WithSearchPaths(
+			"/usr/lib64",
+			"/usr/lib/x86_64-linux-gnu",
+			"/usr/lib/aarch64-linux-gnu",
+			"/usr/lib/x86_64-linux-gnu/nvidia/current",
+			"/usr/lib/aarch64-linux-gnu/nvidia/current",
+		),
+		lookup.WithCount(1),
+	)
+
+	return pathLocator.Locate(fullPattern)
+}

internal/lookup/device.go

@@ -19,9 +19,6 @@ package lookup
 import (
 	"fmt"
 	"os"
-	"path/filepath"
-
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -30,14 +27,14 @@ const (
 
 // NewCharDeviceLocator creates a Locator that can be used to find char devices at the specified root. A logger is
 // also specified.
-func NewCharDeviceLocator(logger *logrus.Logger, root string) Locator {
-	l := file{
-		logger:   logger,
-		prefixes: []string{root, filepath.Join(root, devRoot)},
-		filter:   assertCharDevice,
-	}
-
-	return &l
+func NewCharDeviceLocator(opts ...Option) Locator {
+	opts = append(opts,
+		WithSearchPaths("", devRoot),
+		WithFilter(assertCharDevice),
+	)
+	return NewFileLocator(
+		opts...,
+	)
 }
 
 // assertCharDevice checks whether the specified path is a char device and returns an error if this is not the case.

internal/lookup/device_test.go

@@ -0,0 +1,58 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCharDeviceLocator(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		root             string
+		expectedPrefixes []string
+	}{
+		{
+			root:             "",
+			expectedPrefixes: []string{"", "/dev"},
+		},
+		{
+			root:             "/",
+			expectedPrefixes: []string{"/", "/dev"},
+		},
+		{
+			root:             "/some/root",
+			expectedPrefixes: []string{"/some/root", "/some/root/dev"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			f := NewCharDeviceLocator(
+				WithLogger(logger),
+				WithRoot(tc.root),
+			).(*file)
+
+			require.EqualValues(t, tc.expectedPrefixes, f.prefixes)
+		})
+	}
+}

internal/lookup/dir.go

@@ -26,13 +26,11 @@ import (
 // NewDirectoryLocator creates a Locator that can be used to find directories at the specified root. A logger
 // is also specified.
 func NewDirectoryLocator(logger *log.Logger, root string) Locator {
-	l := file{
-		logger:   logger,
-		prefixes: []string{root},
-		filter:   assertDirectory,
-	}
-
-	return &l
+	return NewFileLocator(
+		WithLogger(logger),
+		WithRoot(root),
+		WithFilter(assertDirectory),
+	)
 }
 
 // assertDirectory checks wither the specified path is a directory.

internal/lookup/executable.go

@@ -19,7 +19,6 @@ package lookup
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -33,17 +32,22 @@ type executable struct {
 func NewExecutableLocator(logger *log.Logger, root string) Locator {
 	paths := GetPaths(root)
 
-	var prefixes []string
-	for _, dir := range paths {
-		prefixes = append(prefixes, filepath.Join(root, dir))
-	}
+	return newExecutableLocator(logger, root, paths...)
+}
+
+func newExecutableLocator(logger *log.Logger, root string, paths ...string) *executable {
+	f := newFileLocator(
+		WithLogger(logger),
+		WithRoot(root),
+		WithSearchPaths(paths...),
+		WithFilter(assertExecutable),
+		WithCount(1),
+	)
+
 	l := executable{
-		file: file{
-			logger:   logger,
-			prefixes: prefixes,
-			filter:   assertExecutable,
-		},
+		file: *f,
 	}
+
 	return &l
 }
 

internal/lookup/executable_test.go

@@ -0,0 +1,77 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecutableLocator(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		root             string
+		paths            []string
+		expectedPrefixes []string
+	}{
+		{
+			root:             "",
+			expectedPrefixes: []string{""},
+		},
+		{
+			root:             "",
+			paths:            []string{"/"},
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "",
+			paths:            []string{"/", "/bin"},
+			expectedPrefixes: []string{"/", "/bin"},
+		},
+		{
+			root:             "/",
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "/",
+			paths:            []string{"/"},
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "/",
+			paths:            []string{"/", "/bin"},
+			expectedPrefixes: []string{"/", "/bin"},
+		},
+		{
+			root:             "/some/path",
+			paths:            []string{"/", "/bin"},
+			expectedPrefixes: []string{"/some/path", "/some/path/bin"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			e := newExecutableLocator(logger, tc.root, tc.paths...)
+
+			require.EqualValues(t, tc.expectedPrefixes, e.prefixes)
+		})
+	}
+}

internal/lookup/file.go

@@ -27,33 +27,124 @@ import (
 // file can be used to locate file (or file-like elements) at a specified set of
 // prefixes. The validity of a file is determined by a filter function.
 type file struct {
-	logger   *log.Logger
-	prefixes []string
-	filter   func(string) error
+	logger     *log.Logger
+	root       string
+	prefixes   []string
+	filter     func(string) error
+	count      int
+	isOptional bool
 }
 
-// NewFileLocator creates a Locator that can be used to find files at the specified root. A logger
-// can also be specified.
-func NewFileLocator(logger *log.Logger, root string) Locator {
-	l := newFileLocator(logger, root)
+// Option defines a function for passing options to the NewFileLocator() call
+type Option func(*file)
 
-	return &l
-}
-
-func newFileLocator(logger *log.Logger, root string) file {
-	return file{
-		logger:   logger,
-		prefixes: []string{root},
-		filter:   assertFile,
+// WithRoot sets the root for the file locator
+func WithRoot(root string) Option {
+	return func(f *file) {
+		f.root = root
 	}
 }
 
+// WithLogger sets the logger for the file locator
+func WithLogger(logger *log.Logger) Option {
+	return func(f *file) {
+		f.logger = logger
+	}
+}
+
+// WithSearchPaths sets the search paths for the file locator.
+func WithSearchPaths(paths ...string) Option {
+	return func(f *file) {
+		f.prefixes = paths
+	}
+}
+
+// WithFilter sets the filter for the file locator
+// The filter is called for each candidate file and candidates that return nil are considered.
+func WithFilter(assert func(string) error) Option {
+	return func(f *file) {
+		f.filter = assert
+	}
+}
+
+// WithCount sets the maximum number of candidates to discover
+func WithCount(count int) Option {
+	return func(f *file) {
+		f.count = count
+	}
+}
+
+// WithOptional sets the optional flag for the file locator
+// If the optional flag is set, the locator will not return an error if the file is not found.
+func WithOptional(optional bool) Option {
+	return func(f *file) {
+		f.isOptional = optional
+	}
+}
+
+// NewFileLocator creates a Locator that can be used to find files with the specified options.
+func NewFileLocator(opts ...Option) Locator {
+	return newFileLocator(opts...)
+}
+
+func newFileLocator(opts ...Option) *file {
+	f := &file{}
+	for _, opt := range opts {
+		opt(f)
+	}
+	if f.logger == nil {
+		f.logger = log.StandardLogger()
+	}
+	if f.filter == nil {
+		f.filter = assertFile
+	}
+	// Since the `Locate` implementations rely on the root already being specified we update
+	// the prefixes to include the root.
+	f.prefixes = getSearchPrefixes(f.root, f.prefixes...)
+
+	return f
+}
+
+// getSearchPrefixes generates a list of unique paths to be searched by a file locator.
+//
+// For each of the unique prefixes <p> specified, the path <root><p> is searched, where <root> is the
+// specified root. If no prefixes are specified, <root> is returned as the only search prefix.
+//
+// Note that an empty root is equivalent to searching relative to the current working directory, and
+// if the root filesystem should be searched instead, root should be specified as "/" explicitly.
+//
+// Also, a prefix of "" forces the root to be included in returned set of paths. This means that if
+// the root in addition to another prefix must be searched the function should be called with:
+//
+//	getSearchPrefixes("/root", "", "another/path")
+//
+// and will result in the search paths []{"/root", "/root/another/path"} being returned.
+func getSearchPrefixes(root string, prefixes ...string) []string {
+	seen := make(map[string]bool)
+	var uniquePrefixes []string
+	for _, p := range prefixes {
+		if seen[p] {
+			continue
+		}
+		seen[p] = true
+		uniquePrefixes = append(uniquePrefixes, filepath.Join(root, p))
+	}
+
+	if len(uniquePrefixes) == 0 {
+		uniquePrefixes = append(uniquePrefixes, root)
+	}
+
+	return uniquePrefixes
+}
+
 var _ Locator = (*file)(nil)
 
 // Locate attempts to find files with names matching the specified pattern.
 // All prefixes are searched and any matching candidates are returned. If no matches are found, an error is returned.
 func (p file) Locate(pattern string) ([]string, error) {
 	var filenames []string
+
+visit:
 	for _, prefix := range p.prefixes {
 		pathPattern := filepath.Join(prefix, pattern)
 		candidates, err := filepath.Glob(pathPattern)
@@ -69,9 +160,14 @@ func (p file) Locate(pattern string) ([]string, error) {
 				continue
 			}
 			filenames = append(filenames, candidate)
+			if p.count > 0 && len(filenames) == p.count {
+				p.logger.Debugf("Found %d candidates; ignoring further candidates", len(filenames))
+				break visit
+			}
 		}
 	}
-	if len(filenames) == 0 {
+
+	if !p.isOptional && len(filenames) == 0 {
 		return nil, fmt.Errorf("pattern %v not found", pattern)
 	}
 	return filenames, nil

internal/lookup/file_test.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetSearchPrefixes(t *testing.T) {
+	testCases := []struct {
+		root             string
+		prefixes         []string
+		expectedPrefixes []string
+	}{
+		{
+			root:             "",
+			expectedPrefixes: []string{""},
+		},
+		{
+			root:             "/",
+			expectedPrefixes: []string{"/"},
+		},
+		{
+			root:             "/some/root",
+			expectedPrefixes: []string{"/some/root"},
+		},
+		{
+			root:             "",
+			prefixes:         []string{"foo", "bar"},
+			expectedPrefixes: []string{"foo", "bar"},
+		},
+		{
+			root:             "/",
+			prefixes:         []string{"foo", "bar"},
+			expectedPrefixes: []string{"/foo", "/bar"},
+		},
+		{
+			root:             "/",
+			prefixes:         []string{"/foo", "/bar"},
+			expectedPrefixes: []string{"/foo", "/bar"},
+		},
+		{
+			root:             "/some/root",
+			prefixes:         []string{"foo", "bar"},
+			expectedPrefixes: []string{"/some/root/foo", "/some/root/bar"},
+		},
+		{
+			root:             "",
+			prefixes:         []string{"foo", "bar", "bar", "foo"},
+			expectedPrefixes: []string{"foo", "bar"},
+		},
+		{
+			root:             "/some/root",
+			prefixes:         []string{"foo", "bar", "foo", "bar"},
+			expectedPrefixes: []string{"/some/root/foo", "/some/root/bar"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			prefixes := getSearchPrefixes(tc.root, tc.prefixes...)
+			require.EqualValues(t, tc.expectedPrefixes, prefixes)
+		})
+	}
+}

internal/lookup/library.go

@@ -40,6 +40,7 @@ func NewLibraryLocator(logger *log.Logger, root string) (Locator, error) {
 	}
 
 	l := library{
+		logger:  logger,
 		symlink: NewSymlinkLocator(logger, root),
 		cache:   cache,
 	}

internal/lookup/symlinks.go

@@ -35,8 +35,9 @@ type symlink struct {
 // NewSymlinkChainLocator creats a locator that can be used for locating files through symlinks.
 // A logger can also be specified.
 func NewSymlinkChainLocator(logger *logrus.Logger, root string) Locator {
+	f := newFileLocator(WithLogger(logger), WithRoot(root))
 	l := symlinkChain{
-		file: newFileLocator(logger, root),
+		file: *f,
 	}
 
 	return &l
@@ -45,8 +46,9 @@ func NewSymlinkChainLocator(logger *logrus.Logger, root string) Locator {
 // NewSymlinkLocator creats a locator that can be used for locating files through symlinks.
 // A logger can also be specified.
 func NewSymlinkLocator(logger *logrus.Logger, root string) Locator {
+	f := newFileLocator(WithLogger(logger), WithRoot(root))
 	l := symlink{
-		file: newFileLocator(logger, root),
+		file: *f,
 	}
 
 	return &l

internal/modifier/cdi.go

@@ -38,7 +38,7 @@ type cdiModifier struct {
 // CDI specifications available on the system. The NVIDIA_VISIBLE_DEVICES enviroment variable is
 // used to select the devices to include.
 func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	devices, err := getDevicesFromSpec(ociSpec)
+	devices, err := getDevicesFromSpec(logger, ociSpec, cfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get required devices from OCI specification: %v", err)
 	}
@@ -46,6 +46,7 @@ func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		logger.Debugf("No devices requested; no modification required.")
 		return nil, nil
 	}
+	logger.Debugf("Creating CDI modifier for devices: %v", devices)
 
 	specDirs := cdi.DefaultSpecDirs
 	if len(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs) > 0 {
@@ -61,38 +62,82 @@ func NewCDIModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 	return m, nil
 }
 
-func getDevicesFromSpec(ociSpec oci.Spec) ([]string, error) {
+func getDevicesFromSpec(logger *logrus.Logger, ociSpec oci.Spec, cfg *config.Config) ([]string, error) {
 	rawSpec, err := ociSpec.Load()
 	if err != nil {
 		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
 	}
 
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	envDevices := image.DevicesFromEnvvars(visibleDevicesEnvvar)
-
-	_, annotationDevices, err := cdi.ParseAnnotations(rawSpec.Annotations)
+	annotationDevices, err := getAnnotationDevices(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.AnnotationPrefixes, rawSpec.Annotations)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse container annotations: %v", err)
 	}
-
-	uniqueDevices := make(map[string]struct{})
-	for _, name := range append(envDevices.List(), annotationDevices...) {
-		if !cdi.IsQualifiedName(name) {
-			name = cdi.QualifiedName("nvidia.com", "gpu", name)
-		}
-		uniqueDevices[name] = struct{}{}
+	if len(annotationDevices) > 0 {
+		return annotationDevices, nil
 	}
 
+	container, err := image.NewCUDAImageFromSpec(rawSpec)
+	if err != nil {
+		return nil, err
+	}
+	envDevices := container.DevicesFromEnvvars(visibleDevicesEnvvar)
+
 	var devices []string
-	for name := range uniqueDevices {
+	seen := make(map[string]bool)
+	for _, name := range envDevices.List() {
+		if !cdi.IsQualifiedName(name) {
+			name = fmt.Sprintf("%s=%s", cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.DefaultKind, name)
+		}
+		if seen[name] {
+			logger.Debugf("Ignoring duplicate device %q", name)
+			continue
+		}
 		devices = append(devices, name)
 	}
 
-	return devices, nil
+	if len(devices) == 0 {
+		return nil, nil
+	}
+
+	if cfg.AcceptEnvvarUnprivileged || image.IsPrivileged(rawSpec) {
+		return devices, nil
+	}
+
+	logger.Warningf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES: %v", devices)
+
+	return nil, nil
+}
+
+// getAnnotationDevices returns a list of devices specified in the annotations.
+// Keys starting with the specified prefixes are considered and expected to contain a comma-separated list of
+// fully-qualified CDI devices names. If any device name is not fully-quality an error is returned.
+// The list of returned devices is deduplicated.
+func getAnnotationDevices(prefixes []string, annotations map[string]string) ([]string, error) {
+	devicesByKey := make(map[string][]string)
+	for key, value := range annotations {
+		for _, prefix := range prefixes {
+			if strings.HasPrefix(key, prefix) {
+				devicesByKey[key] = strings.Split(value, ",")
+			}
+		}
+	}
+
+	seen := make(map[string]bool)
+	var annotationDevices []string
+	for key, devices := range devicesByKey {
+		for _, device := range devices {
+			if !cdi.IsQualifiedName(device) {
+				return nil, fmt.Errorf("invalid device name %q in annotation %q", device, key)
+			}
+			if seen[device] {
+				continue
+			}
+			annotationDevices = append(annotationDevices, device)
+			seen[device] = true
+		}
+	}
+
+	return annotationDevices, nil
 }
 
 // Modify loads the CDI registry and injects the specified CDI devices into the OCI runtime specification.
@@ -105,21 +150,8 @@ func (m cdiModifier) Modify(spec *specs.Spec) error {
 		m.logger.Debugf("The following error was triggered when refreshing the CDI registry: %v", err)
 	}
 
-	devices := m.devices
-	for _, d := range devices {
-		if d == "nvidia.com/gpu=all" {
-			devices = []string{}
-			for _, candidate := range registry.DeviceDB().ListDevices() {
-				if strings.HasPrefix(candidate, "nvidia.com/gpu=") {
-					devices = append(devices, candidate)
-				}
-			}
-			break
-		}
-	}
-
-	m.logger.Debugf("Injecting devices using CDI: %v", devices)
-	_, err := registry.InjectDevices(spec, devices...)
+	m.logger.Debugf("Injecting devices using CDI: %v", m.devices)
+	_, err := registry.InjectDevices(spec, m.devices...)
 	if err != nil {
 		return fmt.Errorf("failed to inject CDI devices: %v", err)
 	}

internal/modifier/cdi_test.go

@@ -0,0 +1,92 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetAnnotationDevices(t *testing.T) {
+	testCases := []struct {
+		description     string
+		prefixes        []string
+		annotations     map[string]string
+		expectedDevices []string
+		expectedError   error
+	}{
+		{
+			description: "no annotations",
+		},
+		{
+			description: "no matching annotations",
+			prefixes:    []string{"not-prefix/"},
+			annotations: map[string]string{
+				"prefix/foo": "example.com/device=bar",
+			},
+		},
+		{
+			description: "single matching annotation",
+			prefixes:    []string{"prefix/"},
+			annotations: map[string]string{
+				"prefix/foo": "example.com/device=bar",
+			},
+			expectedDevices: []string{"example.com/device=bar"},
+		},
+		{
+			description: "multiple matching annotations",
+			prefixes:    []string{"prefix/", "another-prefix/"},
+			annotations: map[string]string{
+				"prefix/foo":         "example.com/device=bar",
+				"another-prefix/bar": "example.com/device=baz",
+			},
+			expectedDevices: []string{"example.com/device=bar", "example.com/device=baz"},
+		},
+		{
+			description: "multiple matching annotations with duplicate devices",
+			prefixes:    []string{"prefix/", "another-prefix/"},
+			annotations: map[string]string{
+				"prefix/foo":         "example.com/device=bar",
+				"another-prefix/bar": "example.com/device=bar",
+			},
+			expectedDevices: []string{"example.com/device=bar"},
+		},
+		{
+			description: "invalid devices",
+			prefixes:    []string{"prefix/"},
+			annotations: map[string]string{
+				"prefix/foo": "example.com/device",
+			},
+			expectedError: fmt.Errorf("invalid device %q", "example.com/device"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			devices, err := getAnnotationDevices(tc.prefixes, tc.annotations)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.ElementsMatch(t, tc.expectedDevices, devices)
+		})
+	}
+}

internal/modifier/csv.go

@@ -62,8 +62,8 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 	logger.Infof("Constructing modifier from config: %+v", *cfg)
 
 	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
+		DriverRoot:    cfg.NVIDIAContainerCLIConfig.Root,
+		NvidiaCTKPath: cfg.NVIDIACTKConfig.Path,
 	}
 
 	if err := checkRequirements(logger, image); err != nil {
@@ -79,7 +79,7 @@ func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec)
 		csvFiles = csv.BaseFilesOnly(csvFiles)
 	}
 
-	csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
+	csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.DriverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
 	}

internal/modifier/discover_test.go

@@ -37,8 +37,10 @@ func TestDiscoverModifier(t *testing.T) {
 		expectedSpec  *specs.Spec
 	}{
 		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
+			description:  "empty discoverer does not modify spec",
+			spec:         &specs.Spec{},
+			discover:     &discover.DiscoverMock{},
+			expectedSpec: &specs.Spec{},
 		},
 		{
 			description: "failed hooks discoverer returns error",

internal/modifier/graphics.go

@@ -45,8 +45,8 @@ func NewGraphicsModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.
 	}
 
 	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
+		DriverRoot:    cfg.NVIDIAContainerCLIConfig.Root,
+		NvidiaCTKPath: cfg.NVIDIACTKConfig.Path,
 	}
 	d, err := discover.NewGraphicsDiscoverer(
 		logger,

internal/modifier/tegra.go

@@ -33,7 +33,7 @@ func NewTegraPlatformFiles(logger *logrus.Logger) (oci.SpecModifier, error) {
 
 	tegraSystemMounts := discover.NewMounts(
 		logger,
-		lookup.NewFileLocator(logger, ""),
+		lookup.NewFileLocator(lookup.WithLogger(logger)),
 		"",
 		[]string{
 			"/etc/nv_tegra_release",

internal/oci/runtime_modifier.go

@@ -14,27 +14,26 @@
 # limitations under the License.
 */
 
-package runtime
+package oci
 
 import (
 	"fmt"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	log "github.com/sirupsen/logrus"
 )
 
 type modifyingRuntimeWrapper struct {
 	logger   *log.Logger
-	runtime  oci.Runtime
-	ociSpec  oci.Spec
-	modifier oci.SpecModifier
+	runtime  Runtime
+	ociSpec  Spec
+	modifier SpecModifier
 }
 
-var _ oci.Runtime = (*modifyingRuntimeWrapper)(nil)
+var _ Runtime = (*modifyingRuntimeWrapper)(nil)
 
 // NewModifyingRuntimeWrapper creates a runtime wrapper that applies the specified modifier to the OCI specification
 // before invoking the wrapped runtime. If the modifier is nil, the input runtime is returned.
-func NewModifyingRuntimeWrapper(logger *log.Logger, runtime oci.Runtime, spec oci.Spec, modifier oci.SpecModifier) oci.Runtime {
+func NewModifyingRuntimeWrapper(logger *log.Logger, runtime Runtime, spec Spec, modifier SpecModifier) Runtime {
 	if modifier == nil {
 		logger.Infof("Using low-level runtime with no modification")
 		return runtime
@@ -52,7 +51,7 @@ func NewModifyingRuntimeWrapper(logger *log.Logger, runtime oci.Runtime, spec oc
 // Exec checks whether a modification of the OCI specification is required and modifies it accordingly before exec-ing
 // into the wrapped runtime.
 func (r *modifyingRuntimeWrapper) Exec(args []string) error {
-	if oci.HasCreateSubcommand(args) {
+	if HasCreateSubcommand(args) {
 		err := r.modify()
 		if err != nil {
 			return fmt.Errorf("could not apply required modification to OCI specification: %v", err)

internal/oci/runtime_modifier_test.go

@@ -14,13 +14,12 @@
 # limitations under the License.
 */
 
-package runtime
+package oci
 
 import (
 	"fmt"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
@@ -38,7 +37,7 @@ func TestExec(t *testing.T) {
 		args          []string
 		modifyError   error
 		writeError    error
-		modifer       oci.SpecModifier
+		modifer       SpecModifier
 	}{
 		{
 			description:   "no args forwards",
@@ -92,9 +91,9 @@ func TestExec(t *testing.T) {
 		hook.Reset()
 
 		t.Run(tc.description, func(t *testing.T) {
-			runtimeMock := &oci.RuntimeMock{}
-			specMock := &oci.SpecMock{
-				ModifyFunc: func(specModifier oci.SpecModifier) error {
+			runtimeMock := &RuntimeMock{}
+			specMock := &SpecMock{
+				ModifyFunc: func(specModifier SpecModifier) error {
 					return tc.modifyError
 				},
 				FlushFunc: func() error {
@@ -144,8 +143,8 @@ func TestExec(t *testing.T) {
 func TestNilModiferReturnsRuntime(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
-	runtimeMock := &oci.RuntimeMock{}
-	specMock := &oci.SpecMock{}
+	runtimeMock := &RuntimeMock{}
+	specMock := &SpecMock{}
 
 	shim := NewModifyingRuntimeWrapper(
 		logger,

internal/runtime/api.go

@@ -0,0 +1,33 @@
+package runtime
+
+type rt struct {
+	logger       *Logger
+	modeOverride string
+}
+
+// Interface is the interface for the runtime library.
+type Interface interface {
+	Run([]string) error
+}
+
+// Option is a function that configures the runtime.
+type Option func(*rt)
+
+// New creates a runtime with the specified options.
+func New(opts ...Option) Interface {
+	r := rt{}
+	for _, opt := range opts {
+		opt(&r)
+	}
+	if r.logger == nil {
+		r.logger = NewLogger()
+	}
+	return &r
+}
+
+// WithModeOverride allows for overriding the mode specified in the config.
+func WithModeOverride(mode string) Option {
+	return func(r *rt) {
+		r.modeOverride = mode
+	}
+}

internal/runtime/logger.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 */
 
-package main
+package runtime
 
 import (
 	"fmt"
@@ -42,11 +42,17 @@ func NewLogger() *Logger {
 	}
 }
 
-// UpdateLogger constructs a Logger with a preddefined formatter
-func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, error) {
+// Update constructs a Logger with a preddefined formatter
+func (l *Logger) Update(filename string, logLevel string, argv []string) error {
+
 	configFromArgs := parseArgs(argv)
 
 	level, logLevelError := configFromArgs.getLevel(logLevel)
+	defer func() {
+		if logLevelError != nil {
+			l.Warn(logLevelError)
+		}
+	}()
 
 	var logFiles []*os.File
 	var argLogFileError error
@@ -55,7 +61,7 @@ func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, err
 	if !configFromArgs.version {
 		configLogFile, err := createLogFile(filename)
 		if err != nil {
-			return logger, fmt.Errorf("error opening debug log file: %v", err)
+			return fmt.Errorf("error opening debug log file: %v", err)
 		}
 		if configLogFile != nil {
 			logFiles = append(logFiles, configLogFile)
@@ -67,14 +73,15 @@ func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, err
 		}
 		argLogFileError = err
 	}
+	defer func() {
+		if argLogFileError != nil {
+			l.Warnf("Failed to open log file: %v", argLogFileError)
+		}
+	}()
 
-	l := &Logger{
-		Logger:         logrus.New(),
-		previousLogger: logger.Logger,
-		logFiles:       logFiles,
-	}
+	newLogger := logrus.New()
 
-	l.SetLevel(level)
+	newLogger.SetLevel(level)
 	if level == logrus.DebugLevel {
 		logrus.SetReportCaller(true)
 		// Shorten function and file names reported by the logger, by
@@ -92,30 +99,28 @@ func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, err
 	}
 
 	if configFromArgs.format == "json" {
-		l.SetFormatter(new(logrus.JSONFormatter))
+		newLogger.SetFormatter(new(logrus.JSONFormatter))
 	}
 
 	if len(logFiles) == 0 {
-		l.SetOutput(io.Discard)
+		newLogger.SetOutput(io.Discard)
 	} else if len(logFiles) == 1 {
-		l.SetOutput(logFiles[0])
+		newLogger.SetOutput(logFiles[0])
 	} else if len(logFiles) > 1 {
 		var writers []io.Writer
 		for _, f := range logFiles {
 			writers = append(writers, f)
 		}
-		l.SetOutput(io.MultiWriter(writers...))
+		newLogger.SetOutput(io.MultiWriter(writers...))
 	}
 
-	if logLevelError != nil {
-		l.Warn(logLevelError)
+	*l = Logger{
+		Logger:         newLogger,
+		previousLogger: l.Logger,
+		logFiles:       logFiles,
 	}
 
-	if argLogFileError != nil {
-		l.Warnf("Failed to open log file: %v", argLogFileError)
-	}
-
-	return l, nil
+	return nil
 }
 
 // Reset closes the log file (if any) and resets the logger output to what it
@@ -126,7 +131,9 @@ func (l *Logger) Reset() error {
 		if previous == nil {
 			previous = logrus.New()
 		}
-		logger = &Logger{Logger: previous}
+		l.Logger = previous
+		l.previousLogger = nil
+		l.logFiles = nil
 	}()
 
 	var errs []error

internal/runtime/logger_test.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLogger(t *testing.T) {
+	l := NewLogger()
+
+	l.Update("", "debug", nil)
+
+	require.Equal(t, logrus.DebugLevel, l.Logger.Level)
+	require.Equal(t, logrus.InfoLevel, l.previousLogger.Level)
+
+}