Merge branch 'fix-hook' into 'main'

Handle empty root in config

See merge request nvidia/container-toolkit/container-toolkit!454

.common-ci.yml

@@ -23,6 +23,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -34,91 +35,68 @@ stages:
   - scan
   - release
 
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
 .main-or-manual:
   rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+    - !reference [.pipeline-trigger-rules, rules]
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: manual
 
-# Define the distribution targets
-.dist-amazonlinux2:
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
   rules:
     - !reference [.main-or-manual, rules]
-  variables:
-    DIST: amazonlinux2
-    PACKAGE_REPO_TYPE: rpm
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
 
+# Define the distribution targets
 .dist-centos7:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: centos7
-    CVE_UPDATES: "cyrus-sasl-lib"
-    PACKAGE_REPO_TYPE: rpm
 
 .dist-centos8:
   variables:
     DIST: centos8
-    CVE_UPDATES: "cyrus-sasl-lib"
-    PACKAGE_REPO_TYPE: rpm
-
-.dist-debian10:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: debian10
-    PACKAGE_REPO_TYPE: debian
-
-.dist-debian9:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: debian9
-    PACKAGE_REPO_TYPE: debian
-
-.dist-fedora35:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: fedora35
-    PACKAGE_REPO_TYPE: rpm
-
-.dist-opensuse-leap15.1:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: opensuse-leap15.1
-    PACKAGE_REPO_TYPE: rpm
 
 .dist-ubi8:
   rules:
     - !reference [.main-or-manual, rules]
   variables:
     DIST: ubi8
-    CVE_UPDATES: "cyrus-sasl-lib"
-    PACKAGE_REPO_TYPE: rpm
-
-.dist-ubuntu16.04:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: ubuntu16.04
-    PACKAGE_REPO_TYPE: debian
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
-    PACKAGE_REPO_TYPE: debian
 
 .dist-ubuntu20.04:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubuntu20.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
-    PACKAGE_REPO_TYPE: debian
 
 .dist-packaging:
   variables:
@@ -236,6 +214,8 @@ test-packaging:
 .release:external:
   extends:
     - .release
+  variables:
+    FORCE_PUBLISH_IMAGES: "yes"
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -259,22 +239,15 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
 release:staging-ubuntu20.04:
   extends:
     - .release:staging
     - .dist-ubuntu20.04
   needs:
-    - image-ubuntu20.04
+    - test-toolkit-ubuntu20.04
+    - test-containerd-ubuntu20.04
+    - test-crio-ubuntu20.04
+    - test-docker-ubuntu20.04
 
 release:staging-packaging:
   extends:

.github/workflows/blossom-ci.yml

@@ -0,0 +1,113 @@
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to trigger ci on hybrid infra (github + self hosted runner)
+name: Blossom-CI
+on:
+  issue_comment:
+    types: [created]
+  workflow_dispatch:
+      inputs:
+          platform:
+            description: 'runs-on argument'     
+            required: false
+          args:
+            description: 'argument'     
+            required: false
+jobs:
+  Authorization:
+    name: Authorization
+    runs-on: blossom 
+    outputs:
+      args: ${{ env.args }}
+     
+    # This job only runs for pull request comments
+    if: |
+         contains( '\
+         anstockatnv,\
+         rorajani,\
+         cdesiniotis,\
+         shivamerla,\
+         ArangoGutierrez,\
+         elezar,\
+         klueska,\
+         zvonkok,\
+         ', format('{0},', github.actor)) && 
+         github.event.comment.body == '/blossom-ci'  
+    steps:
+      - name: Check if comment is issued by authorized person
+        run: blossom-ci
+        env:
+          OPERATION: 'AUTH'
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+      
+  Vulnerability-scan:
+    name: Vulnerability scan
+    needs: [Authorization]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
+          ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
+          lfs: 'true'
+      
+      # repo specific steps 
+      #- name: Setup java
+      #  uses: actions/setup-java@v1
+      #  with:
+      #    java-version: 1.8
+      
+      # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file
+      #- name: Setup blackduck properties
+      #  run: |
+      #       PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g')
+      #       echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties
+      #       echo detect.maven.included.scopes=compile >> application.properties
+          
+      - name: Run blossom action
+        uses: NVIDIA/blossom-action@main
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+        with:
+          args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }}
+          args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }}
+          args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }}
+          
+  Job-trigger:
+    name: Start ci job
+    needs: [Vulnerability-scan]
+    runs-on: blossom
+    steps:
+      - name: Start ci job
+        run: blossom-ci
+        env:
+          OPERATION: 'START-CI-JOB'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              
+  Upload-Log:
+    name: Upload log
+    runs-on: blossom
+    if : github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here)
+        run: blossom-ci
+        env:
+          OPERATION: 'POST-PROCESSING'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pre-sanity.yml

@@ -0,0 +1,22 @@
+name: Run pre sanity
+
+# run this workflow for each commit
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+             
+      - name: Build dev image
+        run: make .build-image
+      
+      - name: Build
+        run: make docker-build
+
+      - name: Tests
+        run: make docker-coverage
+
+      - name: Checks
+        run: make docker-check

.gitignore

@@ -1,9 +1,11 @@
 dist
+artifacts
 *.swp
 *.swo
 /coverage.out*
 /test/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
 /nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk

.gitlab-ci.yml

@@ -102,25 +102,35 @@ unit-tests:
     name: ${ARTIFACTS_NAME}
     paths:
       - ${ARTIFACTS_ROOT}
+  needs:
+    - job: package-meta-packages
+      artifacts: true
 
 # Define the package build targets
-package-amazonlinux2-aarch64:
+package-meta-packages:
   extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-aarch64
+    - .package-artifacts
+  stage: package-build
+  variables:
+    SKIP_LIBNVIDIA_CONTAINER: "yes"
+    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
+  parallel:
+    matrix:
+      - PACKAGING: [deb, rpm]
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+  script:
+    - ./scripts/build-packages.sh ${PACKAGING}
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
 
-package-amazonlinux2-x86_64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-package-centos7-ppc64le:
+package-centos7-aarch64:
   extends:
     - .package-build
     - .dist-centos7
-    - .arch-ppc64le
+    - .arch-aarch64
 
 package-centos7-x86_64:
   extends:
@@ -146,48 +156,6 @@ package-centos8-x86_64:
     - .dist-centos8
     - .arch-x86_64
 
-package-debian10-amd64:
-  extends:
-    - .package-build
-    - .dist-debian10
-    - .arch-amd64
-
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-fedora35-aarch64:
-  extends:
-    - .package-build
-    - .dist-fedora35
-    - .arch-aarch64
-
-package-fedora35-x86_64:
-  extends:
-    - .package-build
-    - .dist-fedora35
-    - .arch-x86_64
-
-package-opensuse-leap15.1-x86_64:
-  extends:
-    - .package-build
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -240,7 +208,6 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
-    - package-centos7-ppc64le
     - package-centos7-x86_64
 
 image-ubi8:
@@ -249,21 +216,9 @@ image-ubi8:
     - .package-artifacts
     - .dist-ubi8
   needs:
-    # Note: The ubi8 image uses the centos8 packages
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
+    # Note: The ubi8 image uses the centos7 packages
+    - package-centos7-aarch64
+    - package-centos7-x86_64
 
 image-ubuntu20.04:
   extends:
@@ -273,7 +228,8 @@ image-ubuntu20.04:
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -290,7 +246,7 @@ image-packaging:
       optional: true
     - job: package-amazonlinux2-x86_64
       optional: true
-    - job: package-centos7-ppc64le
+    - job: package-centos7-aarch64
       optional: true
     - job: package-centos7-x86_64
       optional: true
@@ -298,18 +254,8 @@ image-packaging:
       optional: true
     - job: package-debian10-amd64
       optional: true
-    - job: package-debian9-amd64
-      optional: true
-    - job: package-fedora35-aarch64
-      optional: true
-    - job: package-fedora35-x86_64
-      optional: true
     - job: package-opensuse-leap15.1-x86_64
       optional: true
-    - job: package-ubuntu16.04-amd64
-      optional: true
-    - job: package-ubuntu16.04-ppc64le
-      optional: true
     - job: package-ubuntu18.04-ppc64le
       optional: true
 
@@ -343,31 +289,31 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu18.04:
+test-toolkit-ubuntu20.04:
   extends:
     - .test:toolkit
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
-test-containerd-ubuntu18.04:
+test-containerd-ubuntu20.04:
   extends:
     - .test:containerd
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
-test-crio-ubuntu18.04:
+test-crio-ubuntu20.04:
   extends:
     - .test:crio
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
-test-docker-ubuntu18.04:
+test-docker-ubuntu20.04:
   extends:
     - .test:docker
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 

.gitmodules

@@ -2,11 +2,3 @@
 	path = third_party/libnvidia-container
 	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
 	branch = main
-[submodule "third_party/nvidia-container-runtime"]
-	path = third_party/nvidia-container-runtime
-	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
-	branch = main
-[submodule "third_party/nvidia-docker"]
-	path = third_party/nvidia-docker
-	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
-	branch = main

.nvidia-ci.yml

@@ -36,8 +36,8 @@ variables:
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
   ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
-  # TODO: We should set the kitmaker release folder here once we have the end-to-end workflow set up
-  KITMAKER_RELEASE_FOLDER: "testing"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
+  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
 .image-pull:
   stage: image-build
@@ -79,11 +79,6 @@ image-ubi8:
     - .dist-ubi8
     - .image-pull
 
-image-ubuntu18.04:
-  extends:
-    - .dist-ubuntu18.04
-    - .image-pull
-
 image-ubuntu20.04:
   extends:
     - .dist-ubuntu20.04
@@ -110,7 +105,7 @@ image-packaging:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
+    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
   rules:
     - if: $SKIP_SCANS != "yes"
     - when: manual
@@ -125,6 +120,7 @@ image-packaging:
     - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
   script:
     - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+    - rm -f "${IMAGE_ARCHIVE}"
   artifacts:
     when: always
     expire_in: 1 week
@@ -144,23 +140,6 @@ scan-centos7-amd64:
   needs:
     - image-centos7
 
-scan-centos7-arm64:
-  extends:
-    - .dist-centos7
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-centos7
-    - scan-centos7-amd64
-
-scan-ubuntu18.04-amd64:
-  extends:
-    - .dist-ubuntu18.04
-    - .platform-amd64
-    - .scan
-  needs:
-    - image-ubuntu18.04
-
 scan-ubuntu20.04-amd64:
   extends:
     - .dist-ubuntu20.04
@@ -195,6 +174,13 @@ scan-ubi8-arm64:
     - image-ubi8
     - scan-ubi8-amd64
 
+scan-packaging:
+  extends:
+    - .dist-packaging
+    - .scan
+  needs:
+    - image-packaging
+
 # Define external release helpers
 .release:ngc:
   extends:
@@ -217,25 +203,43 @@ scan-ubi8-arm64:
     PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
     KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+    ARTIFACTS_DIR: "${CI_PROJECT_DIR}/artifacts"
   script:
     - !reference [.regctl-setup, before_script]
     - apk add --no-cache bash git
     - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
     - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
-    # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
     - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+    - rm -rf ${ARTIFACTS_DIR}
 
 # Define the package release targets
 release:packages:kitmaker:
   extends:
     - .release:packages
 
-release:staging-ubuntu18.04:
+release:archive:
+  extends:
+    - .release:external
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}"
+  script:
+    - apk add --no-cache bash git
+    - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}"
+
+release:staging-ubuntu20.04:
   extends:
     - .release:staging
-    - .dist-ubuntu18.04
+    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu18.04
+    - image-ubuntu20.04
 
 # Define the external release targets
 # Release to NGC
@@ -244,11 +248,6 @@ release:ngc-centos7:
     - .dist-centos7
     - .release:ngc
 
-release:ngc-ubuntu18.04:
-  extends:
-    - .dist-ubuntu18.04
-    - .release:ngc
-
 release:ngc-ubuntu20.04:
   extends:
     - .dist-ubuntu20.04
@@ -258,3 +257,8 @@ release:ngc-ubi8:
   extends:
     - .dist-ubi8
     - .release:ngc
+
+release:ngc-packaging:
+  extends:
+    - .dist-packaging
+    - .release:ngc

CHANGELOG.md

@@ -1,10 +1,127 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.14.0-rc.2
+
+* Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
+* Remove dependency on coreutils when installing package on RPM-based systems.
+* Create ouput folders if required when running `nvidia-ctk runtime configure`
+* Generate default config as post-install step.
+* Added support for detecting GSP firmware at custom paths when generating CDI specifications.
+* Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.
+
+* [libnvidia-container] Include Shared Compiler Library (libnvidia-gpucomp.so) in the list of compute libaries.
+
+* [toolkit-container] Ensure that common envvars have higher priority when configuring the container engines.
+* [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
+
+## v1.14.0-rc.1
+
+* Add support for updating containerd configs to the `nvidia-ctk runtime configure` command.
+* Create file in `etc/ld.so.conf.d` with permissions `644` to support non-root containers.
+* Generate CDI specification files with `644` permissions to allow rootless applications (e.g. podman)
+* Add `nvidia-ctk cdi list` command to show the known CDI devices.
+* Add support for generating merged devices (e.g. `all` device) to the nvcdi API.
+* Use *.* pattern to locate libcuda.so when generating a CDI specification to support platforms where a patch version is not specified.
+* Update go-nvlib to skip devices that are not MIG capable when generating CDI specifications.
+* Add `nvidia-container-runtime-hook.path` config option to specify NVIDIA Container Runtime Hook path explicitly.
+* Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
+* Add option to load kernel modules when creating device nodes
+* Add option to create device nodes when creating `/dev/char` symlinks
+
+* [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
+
+* [toolkit-container] Allow same envars for all runtime configs
+
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
 ## v1.12.0-rc.4
 
 * Generate a minimum CDI spec version for improved compatibility.
 * Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
 * Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
 ## v1.12.0-rc.3
 
 * Don't fail if by-path symlinks for DRM devices do not exist

Makefile

@@ -61,7 +61,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-extldflags=-Wl,-z,lazy -s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...

build/container/Dockerfile.centos

@@ -17,19 +17,29 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN yum install -y \
+    wget make git gcc \
+     && \
+    rm -rf /var/cache/yum/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -87,11 +97,4 @@ LABEL description="See summary"
 
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        yum update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
-
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.packaging

@@ -18,19 +18,23 @@ ARG GOLANG_VERSION=x.x.x
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
-ARG VERSION="N/A"
-
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
+# build-args are added to the manifest.txt file below.
+ARG BASE_DIST
+ARG PACKAGE_DIST
+ARG PACKAGE_VERSION
 ARG GIT_BRANCH
 ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
 ARG SOURCE_DATE_EPOCH
+ARG VERSION
 
 # Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
-RUN echo "IMAGE_EPOCH=$(date '+%s')" | sed 's/^/#/g' > /artifacts/manifest.txt && \
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
     env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
     find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
 

build/container/Dockerfile.ubuntu

@@ -17,19 +17,28 @@ ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
 ARG VERSION="N/A"
 
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} as build
 
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
+RUN apt-get update && \
+    apt-get install -y wget make git gcc \
+    && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -95,11 +104,4 @@ LABEL description="See summary"
 
 RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
-
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -14,6 +14,7 @@
 
 BUILD_MULTI_ARCH_IMAGES ?= false
 DOCKER ?= docker
+REGCTL ?= regctl
 
 BUILDX =
 ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
@@ -44,13 +45,13 @@ OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
 DEFAULT_PUSH_TARGET := ubuntu20.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7
+DISTRIBUTIONS := ubuntu20.04 ubi8 centos7
 
 META_TARGETS := packaging
 
 BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
 PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
 
 .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -74,6 +75,14 @@ endif
 push-%: DIST = $(*)
 push-short: DIST = $(DEFAULT_PUSH_TARGET)
 
+# Define the push targets
+$(PUSH_TARGETS): push-%:
+	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+
+push-short:
+	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+
+
 build-%: DIST = $(*)
 build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
 
@@ -95,9 +104,9 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
 		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
 		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
 		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
-		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
 
@@ -105,24 +114,20 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
 build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
 
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
-build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-ubi8: PACKAGE_DIST = centos7
 
 build-centos7: BASE_DIST = $(*)
 build-centos7: DOCKERFILE_SUFFIX := centos
 build-centos7: PACKAGE_DIST = $(BASE_DIST)
-build-centos7: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
@@ -138,18 +143,11 @@ $(TEST_TARGETS): test-%:
 test-packaging: DIST = packaging
 test-packaging:
 	@echo "Testing package image contents"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/aarch64" || echo "Missing centos7/aarch64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/multi-arch.mk

@@ -16,17 +16,6 @@ PUSH_ON_BUILD ?= false
 DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD)
 DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64
 
-REGCTL ?= regctl
-$(PUSH_TARGETS): push-%:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE)
-
-push-short:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)
-
 # We only have x86_64 packages for centos7
 build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
 

build/container/native-only.mk

@@ -13,11 +13,3 @@
 # limitations under the License.
 
 DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
-
-$(PUSH_TARGETS): push-%:
-	$(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)"
-	$(DOCKER) push "$(OUT_IMAGE)"
-
-push-short:
-	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"
-	$(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"

cmd/nvidia-container-runtime-hook/container_config.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
 )
 
@@ -37,14 +38,16 @@ type nvidiaConfig struct {
 	MigConfigDevices   string
 	MigMonitorDevices  string
 	DriverCapabilities string
-	Requirements       []string
-	DisableRequire     bool
+	// Requirements defines the requirements DSL for the container to run.
+	// This is empty if no specific requirements are needed, or if requirements are
+	// explicitly disabled.
+	Requirements []string
 }
 
 type containerConfig struct {
 	Pid    int
 	Rootfs string
-	Env    map[string]string
+	Image  image.CUDA
 	Nvidia *nvidiaConfig
 }
 
@@ -130,7 +133,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -139,28 +142,31 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		for _, c := range caps {
+			if c == capSysAdmin {
+				return true
+			}
 		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
+		return false
 	}
 
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	process := specs.Process{
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
 func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
@@ -323,15 +329,12 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 		log.Panicln("failed to get requirements", err)
 	}
 
-	disableRequire := image.HasDisableRequire()
-
 	return &nvidiaConfig{
 		Devices:            devices,
 		MigConfigDevices:   migConfigDevices,
 		MigMonitorDevices:  migMonitorDevices,
 		DriverCapabilities: driverCapabilities,
 		Requirements:       requirements,
-		DisableRequire:     disableRequire,
 	}
 }
 
@@ -349,7 +352,10 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	image, err := image.New(
+		image.WithEnv(s.Process.Env),
+		image.WithDisableRequire(hook.DisableRequire),
+	)
 	if err != nil {
 		log.Panicln(err)
 	}
@@ -358,7 +364,7 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    image,
+		Image:  image,
 		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -40,7 +40,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -54,7 +53,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -86,7 +84,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -100,7 +97,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -115,7 +111,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -130,7 +125,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -145,7 +139,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -162,7 +155,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -179,8 +171,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -211,7 +202,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -243,7 +233,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -257,7 +246,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -272,7 +260,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -287,7 +274,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -302,7 +288,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -319,7 +304,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -336,8 +320,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
 				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
+				Requirements:       []string{},
 			},
 		},
 		{
@@ -351,7 +334,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				Devices:            "all",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -367,7 +349,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				MigConfigDevices:   "mig0,mig1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -393,7 +374,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				MigMonitorDevices:  "mig0,mig1",
 				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
 			},
 		},
 		{
@@ -496,7 +476,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			getConfig := func() {
 				hookConfig := tc.hookConfig
 				if hookConfig == nil {
-					defaultConfig := getDefaultHookConfig()
+					defaultConfig, _ := getDefaultHookConfig()
 					hookConfig = &defaultConfig
 				}
 				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
@@ -525,7 +505,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
 
 			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
-			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
 		})
 	}
 }
@@ -708,7 +687,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				env := map[string]string{
 					envNVVisibleDevices: tc.envvarDevices,
 				}
-				hookConfig := getDefaultHookConfig()
+				hookConfig, _ := getDefaultHookConfig()
 				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
 				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"os"
 	"path"
@@ -43,12 +44,23 @@ type HookConfig struct {
 	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
 	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
+	NvidiaContainerCLI         CLIConfig                `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime     config.RuntimeConfig     `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHook config.RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
 }
 
-func getDefaultHookConfig() HookConfig {
-	return HookConfig{
+func getDefaultHookConfig() (HookConfig, error) {
+	rtConfig, err := config.GetDefaultRuntimeConfig()
+	if err != nil {
+		return HookConfig{}, err
+	}
+
+	rtHookConfig, err := config.GetDefaultRuntimeHookConfig()
+	if err != nil {
+		return HookConfig{}, err
+	}
+
+	c := HookConfig{
 		DisableRequire:                 false,
 		SwarmResource:                  nil,
 		AcceptEnvvarUnprivileged:       true,
@@ -66,27 +78,37 @@ func getDefaultHookConfig() HookConfig {
 			User:        nil,
 			Ldconfig:    nil,
 		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntime:     *rtConfig,
+		NVIDIAContainerRuntimeHook: *rtHookConfig,
 	}
+
+	return c, nil
 }
 
-func getHookConfig() (config HookConfig) {
+func getHookConfig() (*HookConfig, error) {
 	var err error
+	var config HookConfig
 
 	if len(*configflag) > 0 {
-		config = getDefaultHookConfig()
+		config, err = getDefaultHookConfig()
+		if err != nil {
+			return nil, fmt.Errorf("couldn't get default configuration: %v", err)
+		}
 		_, err = toml.DecodeFile(*configflag, &config)
 		if err != nil {
-			log.Panicln("couldn't open configuration file:", err)
+			return nil, fmt.Errorf("couldn't open configuration file: %v", err)
 		}
 	} else {
 		for _, p := range defaultPaths {
-			config = getDefaultHookConfig()
+			config, err = getDefaultHookConfig()
+			if err != nil {
+				return nil, fmt.Errorf("couldn't get default configuration: %v", err)
+			}
 			_, err = toml.DecodeFile(p, &config)
 			if err == nil {
 				break
 			} else if !os.IsNotExist(err) {
-				log.Panicln("couldn't open default configuration file:", err)
+				return nil, fmt.Errorf("couldn't open default configuration file: %v", err)
 			}
 		}
 	}
@@ -100,7 +122,7 @@ func getHookConfig() (config HookConfig) {
 		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
 	}
 
-	return config
+	return &config, nil
 }
 
 // getConfigOption returns the toml config option associated with the

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -89,7 +89,8 @@ func TestGetHookConfig(t *testing.T) {
 
 			var config HookConfig
 			getHookConfig := func() {
-				config = getHookConfig()
+				c, _ := getHookConfig()
+				config = *c
 			}
 
 			if tc.expectedPanic {

cmd/nvidia-container-runtime-hook/main.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
@@ -71,24 +72,27 @@ func doPrestart() {
 	defer exit()
 	log.SetFlags(0)
 
-	hook := getHookConfig()
+	hook, err := getHookConfig()
+	if err != nil || hook == nil {
+		log.Panicln("error getting hook config:", err)
+	}
 	cli := hook.NvidiaContainerCLI
 
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
-	}
-
-	container := getContainerConfig(hook)
+	container := getContainerConfig(*hook)
 	nvidia := container.Nvidia
 	if nvidia == nil {
 		// Not a GPU container, nothing to do.
 		return
 	}
 
+	if !hook.NVIDIAContainerRuntimeHook.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode, container.Image) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
+	}
+
 	rootfs := getRootfsPath(container)
 
 	args := []string{getCLIPath(cli)}
-	if cli.Root != nil {
+	if cli.Root != nil && *cli.Root != "" {
 		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
 	}
 	if cli.LoadKmods {
@@ -133,10 +137,8 @@ func doPrestart() {
 		args = append(args, capabilityToCLI(cap))
 	}
 
-	if !hook.DisableRequire && !nvidia.DisableRequire {
-		for _, req := range nvidia.Requirements {
-			args = append(args, fmt.Sprintf("--require=%s", req))
-		}
+	for _, req := range nvidia.Requirements {
+		args = append(args, fmt.Sprintf("--require=%s", req))
 	}
 
 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
@@ -185,11 +187,11 @@ func main() {
 	}
 }
 
-// logInterceptor implements the info.Logger interface to allow for logging from this function.
-type logInterceptor struct{}
+// logInterceptor implements the logger.Interface to allow for logging from executable.
+type logInterceptor struct {
+	logger.NullLogger
+}
 
 func (l *logInterceptor) Infof(format string, args ...interface{}) {
 	log.Printf(format, args...)
 }
-
-func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/main.go

@@ -1,89 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer func() {
-		if rerr != nil {
-			logger.Errorf("%v", rerr)
-		}
-		logger.Reset()
-	}()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,6 +14,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,7 +43,7 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		log.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
@@ -53,11 +55,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,7 +79,7 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -91,9 +93,10 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -170,7 +173,8 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	logger, _ := testlog.NewNullLogger()
+	m := modifier.NewStableRuntimeModifier(logger, nvidiaHook)
 	return m.Modify(spec)
 }
 

cmd/nvidia-ctk/cdi/cdi.go

@@ -18,16 +18,18 @@ package cdi
 
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/list"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs an info command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -44,6 +46,8 @@ func (m command) build() *cli.Command {
 
 	hook.Subcommands = []*cli.Command{
 		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
+		list.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/cdi/generate/driver.go

@@ -1,156 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package generate
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-)
-
-// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
-// The supplied NVML Library is used to query the expected driver version.
-func NewDriverDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
-	version, r := nvmllib.SystemGetDriverVersion()
-	if r != nvml.SUCCESS {
-		return nil, fmt.Errorf("failed to determine driver version: %v", r)
-	}
-
-	libraries, err := NewDriverLibraryDiscoverer(logger, root, nvidiaCTKPath, version)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
-	}
-
-	firmwares := NewDriverFirmwareDiscoverer(logger, root, version)
-
-	binaries := NewDriverBinariesDiscoverer(logger, root)
-
-	d := discover.Merge(
-		libraries,
-		firmwares,
-		binaries,
-	)
-
-	return d, nil
-}
-
-// NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version.
-func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, version string) (discover.Discover, error) {
-	libraryPaths, err := getVersionLibs(logger, root, version)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get libraries for driver version: %v", err)
-	}
-
-	libraries := discover.NewMounts(
-		logger,
-		lookup.NewFileLocator(
-			lookup.WithLogger(logger),
-			lookup.WithRoot(root),
-		),
-		root,
-		libraryPaths,
-	)
-
-	cfg := &discover.Config{
-		Root:          root,
-		NvidiaCTKPath: nvidiaCTKPath,
-	}
-	hooks, _ := discover.NewLDCacheUpdateHook(logger, libraries, cfg)
-
-	d := discover.Merge(
-		libraries,
-		hooks,
-	)
-
-	return d, nil
-}
-
-// NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version.
-func NewDriverFirmwareDiscoverer(logger *logrus.Logger, root string, version string) discover.Discover {
-	gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp.bin")
-	return discover.NewMounts(
-		logger,
-		lookup.NewFileLocator(
-			lookup.WithLogger(logger),
-			lookup.WithRoot(root),
-		),
-		root,
-		[]string{gspFirmwarePath},
-	)
-}
-
-// NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver.
-func NewDriverBinariesDiscoverer(logger *logrus.Logger, root string) discover.Discover {
-	return discover.NewMounts(
-		logger,
-		lookup.NewExecutableLocator(logger, root),
-		root,
-		[]string{
-			"nvidia-smi",              /* System management interface */
-			"nvidia-debugdump",        /* GPU coredump utility */
-			"nvidia-persistenced",     /* Persistence mode utility */
-			"nvidia-cuda-mps-control", /* Multi process service CLI */
-			"nvidia-cuda-mps-server",  /* Multi process service server */
-		},
-	)
-}
-
-// getVersionLibs checks the LDCache for libraries ending in the specified driver version.
-// Although the ldcache at the specified root is queried, the paths are returned relative to this root.
-// This allows the standard mount location logic to be used for resolving the mounts.
-func getVersionLibs(logger *logrus.Logger, root string, version string) ([]string, error) {
-	logger.Infof("Using driver version %v", version)
-
-	cache, err := ldcache.New(logger, root)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load ldcache: %v", err)
-	}
-
-	libs32, libs64 := cache.List()
-
-	var libs []string
-	for _, l := range libs64 {
-		if strings.HasSuffix(l, version) {
-			logger.Infof("found 64-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
-	}
-
-	for _, l := range libs32 {
-		if strings.HasSuffix(l, version) {
-			logger.Infof("found 32-bit driver lib: %v", l)
-			libs = append(libs, l)
-		}
-	}
-
-	if root == "/" || root == "" {
-		return libs, nil
-	}
-
-	var relative []string
-	for _, l := range libs {
-		relative = append(relative, strings.TrimPrefix(l, root))
-	}
-
-	return relative, nil
-}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -18,41 +18,45 @@ package generate
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-	"sigs.k8s.io/yaml"
 )
 
 const (
-	formatJSON = "json"
-	formatYAML = "yaml"
+	allDeviceName = "all"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
-type config struct {
+type options struct {
 	output             string
 	format             string
 	deviceNameStrategy string
-	root               string
+	driverRoot         string
 	nvidiaCTKPath      string
+	mode               string
+	vendor             string
+	class              string
+
+	csv struct {
+		files cli.StringSlice
+	}
 }
 
 // NewCommand constructs a generate-cdi command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -61,17 +65,17 @@ func NewCommand(logger *logrus.Logger) *cli.Command {
 
 // build creates the CLI command
 func (m command) build() *cli.Command {
-	cfg := config{}
+	opts := options{}
 
 	// Create the 'generate-cdi' command
 	c := cli.Command{
 		Name:  "generate",
 		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
 		Before: func(c *cli.Context) error {
-			return m.validateFlags(c, &cfg)
+			return m.validateFlags(c, &opts)
 		},
 		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
+			return m.run(c, &opts)
 		},
 	}
 
@@ -79,305 +83,175 @@ func (m command) build() *cli.Command {
 		&cli.StringFlag{
 			Name:        "output",
 			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
-			Destination: &cfg.output,
+			Destination: &opts.output,
 		},
 		&cli.StringFlag{
 			Name:        "format",
 			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
-			Value:       formatYAML,
-			Destination: &cfg.format,
+			Value:       spec.FormatYAML,
+			Destination: &opts.format,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Aliases:     []string{"discovery-mode"},
+			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       nvcdi.ModeAuto,
+			Destination: &opts.mode,
 		},
 		&cli.StringFlag{
 			Name:        "device-name-strategy",
 			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
-			Value:       deviceNameStrategyIndex,
-			Destination: &cfg.deviceNameStrategy,
+			Value:       nvcdi.DeviceNameStrategyIndex,
+			Destination: &opts.deviceNameStrategy,
 		},
 		&cli.StringFlag{
-			Name:        "root",
-			Usage:       "Specify the root to use when discovering the entities that should be included in the CDI specification.",
-			Destination: &cfg.root,
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.driverRoot,
 		},
 		&cli.StringFlag{
 			Name:        "nvidia-ctk-path",
 			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
-			Destination: &cfg.nvidiaCTKPath,
+			Destination: &opts.nvidiaCTKPath,
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &opts.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &opts.class,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.file",
+			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CSV mode.",
+			Value:       cli.NewStringSlice(csv.DefaultFileList()...),
+			Destination: &opts.csv.files,
 		},
 	}
 
 	return &c
 }
 
-func (m command) validateFlags(r *cli.Context, cfg *config) error {
-	cfg.format = strings.ToLower(cfg.format)
-	switch cfg.format {
-	case formatJSON:
-	case formatYAML:
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	opts.format = strings.ToLower(opts.format)
+	switch opts.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
 	default:
-		return fmt.Errorf("invalid output format: %v", cfg.format)
+		return fmt.Errorf("invalid output format: %v", opts.format)
 	}
 
-	_, err := newDeviceNamer(cfg.deviceNameStrategy)
+	opts.mode = strings.ToLower(opts.mode)
+	switch opts.mode {
+	case nvcdi.ModeAuto:
+	case nvcdi.ModeCSV:
+	case nvcdi.ModeNvml:
+	case nvcdi.ModeWsl:
+	case nvcdi.ModeManagement:
+	default:
+		return fmt.Errorf("invalid discovery mode: %v", opts.mode)
+	}
+
+	_, err := nvcdi.NewDeviceNamer(opts.deviceNameStrategy)
 	if err != nil {
 		return err
 	}
 
+	opts.nvidiaCTKPath = config.ResolveNVIDIACTKPath(m.logger, opts.nvidiaCTKPath)
+
+	if outputFileFormat := formatFromFilename(opts.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			opts.format = outputFileFormat
+		} else if outputFileFormat != opts.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", opts.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(opts.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(opts.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
 	return nil
 }
 
-func (m command) run(c *cli.Context, cfg *config) error {
-	deviceNamer, err := newDeviceNamer(cfg.deviceNameStrategy)
-	if err != nil {
-		return fmt.Errorf("failed to create device namer: %v", err)
-	}
-
-	spec, err := m.generateSpec(
-		cfg.root,
-		discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath),
-		deviceNamer,
-	)
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := m.generateSpec(opts)
 	if err != nil {
 		return fmt.Errorf("failed to generate CDI spec: %v", err)
 	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
 
-	var outputTo io.Writer
-	if cfg.output == "" {
-		outputTo = os.Stdout
-	} else {
-		err := createParentDirsIfRequired(cfg.output)
+	if opts.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
 		if err != nil {
-			return fmt.Errorf("failed to create parent folders for output file: %v", err)
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
 		}
-		outputFile, err := os.Create(cfg.output)
-		if err != nil {
-			return fmt.Errorf("failed to create output file: %v", err)
-		}
-		defer outputFile.Close()
-		outputTo = outputFile
+		return nil
 	}
 
-	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
-		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
-		if !c.IsSet("format") {
-			cfg.format = outputFileFormat
-		} else if outputFileFormat != cfg.format {
-			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", cfg.format, outputFileFormat)
-		}
-	}
-
-	data, err := yaml.Marshal(spec)
-	if err != nil {
-		return fmt.Errorf("failed to marshal CDI spec: %v", err)
-	}
-
-	if strings.ToLower(cfg.format) == formatJSON {
-		data, err = yaml.YAMLToJSONStrict(data)
-		if err != nil {
-			return fmt.Errorf("failed to convert CDI spec from YAML to JSON: %v", err)
-		}
-	}
-
-	err = writeToOutput(cfg.format, data, outputTo)
-	if err != nil {
-		return fmt.Errorf("failed to write output: %v", err)
-	}
-
-	return nil
+	return spec.Save(opts.output)
 }
 
 func formatFromFilename(filename string) string {
 	ext := filepath.Ext(filename)
 	switch strings.ToLower(ext) {
 	case ".json":
-		return formatJSON
-	case ".yaml":
-		return formatYAML
-	case ".yml":
-		return formatYAML
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
 	}
 
 	return ""
 }
 
-func writeToOutput(format string, data []byte, output io.Writer) error {
-	if format == formatYAML {
-		_, err := output.Write([]byte("---\n"))
-		if err != nil {
-			return fmt.Errorf("failed to write YAML separator: %v", err)
-		}
-	}
-	_, err := output.Write(data)
+func (m command) generateSpec(opts *options) (spec.Interface, error) {
+	deviceNamer, err := nvcdi.NewDeviceNamer(opts.deviceNameStrategy)
 	if err != nil {
-		return fmt.Errorf("failed to write data: %v", err)
+		return nil, fmt.Errorf("failed to create device namer: %v", err)
 	}
 
-	return nil
-}
-
-func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNamer) (*specs.Spec, error) {
-	nvmllib := nvml.New()
-	if r := nvmllib.Init(); r != nvml.SUCCESS {
-		return nil, r
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(opts.driverRoot),
+		nvcdi.WithNVIDIACTKPath(opts.nvidiaCTKPath),
+		nvcdi.WithDeviceNamer(deviceNamer),
+		nvcdi.WithMode(string(opts.mode)),
+		nvcdi.WithCSVFiles(opts.csv.files.Value()),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
 	}
-	defer nvmllib.Shutdown()
 
-	devicelib := device.New(device.WithNvml(nvmllib))
-
-	deviceSpecs, err := m.generateDeviceSpecs(devicelib, root, nvidiaCTKPath, namer)
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
 	}
 
-	allDevice := createAllDevice(deviceSpecs)
-
-	deviceSpecs = append(deviceSpecs, allDevice)
-
-	allEdits := edits.NewContainerEdits()
-
-	ipcs, err := NewIPCDiscoverer(m.logger, root)
+	commonEdits, err := cdilib.GetCommonEdits()
 	if err != nil {
-		return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
 	}
 
-	ipcEdits, err := edits.FromDiscoverer(ipcs)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create container edits for IPC sockets: %v", err)
-	}
-	// TODO: We should not have to update this after the fact
-	for _, s := range ipcEdits.Mounts {
-		s.Options = append(s.Options, "noexec")
-	}
-
-	allEdits.Append(ipcEdits)
-
-	common, err := NewCommonDiscoverer(m.logger, root, nvidiaCTKPath, nvmllib)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
-	}
-
-	deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, root, nvidiaCTKPath, deviceSpecs)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
-	}
-
-	commonEdits, err := edits.FromDiscoverer(discover.Merge(common, deviceFolderPermissionHooks))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create container edits for common entities: %v", err)
-	}
-
-	allEdits.Append(commonEdits)
-
-	// We construct the spec and determine the minimum required version based on the specification.
-	spec := specs.Spec{
-		Version:        "NOT_SET",
-		Kind:           "nvidia.com/gpu",
-		Devices:        deviceSpecs,
-		ContainerEdits: *allEdits.ContainerEdits,
-	}
-
-	minVersion, err := cdi.MinimumRequiredVersion(&spec)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get minumum required CDI spec version: %v", err)
-	}
-	m.logger.Infof("Using minimum required CDI spec version: %s", minVersion)
-
-	spec.Version = minVersion
-
-	return &spec, nil
-}
-
-func (m command) generateDeviceSpecs(devicelib device.Interface, root string, nvidiaCTKPath string, namer deviceNamer) ([]specs.Device, error) {
-	var deviceSpecs []specs.Device
-
-	err := devicelib.VisitDevices(func(i int, d device.Device) error {
-		isMigEnabled, err := d.IsMigEnabled()
-		if err != nil {
-			return fmt.Errorf("failed to check whether device is MIG device: %v", err)
-		}
-		if isMigEnabled {
-			return nil
-		}
-		device, err := NewFullGPUDiscoverer(m.logger, root, nvidiaCTKPath, d)
-		if err != nil {
-			return fmt.Errorf("failed to create device: %v", err)
-		}
-
-		deviceEdits, err := edits.FromDiscoverer(device)
-		if err != nil {
-			return fmt.Errorf("failed to create container edits for device: %v", err)
-		}
-
-		deviceName, err := namer.GetDeviceName(i, d)
-		if err != nil {
-			return fmt.Errorf("failed to get device name: %v", err)
-		}
-		deviceSpec := specs.Device{
-			Name:           deviceName,
-			ContainerEdits: *deviceEdits.ContainerEdits,
-		}
-
-		deviceSpecs = append(deviceSpecs, deviceSpec)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate CDI spec for GPU devices: %v", err)
-	}
-
-	err = devicelib.VisitMigDevices(func(i int, d device.Device, j int, mig device.MigDevice) error {
-		device, err := NewMigDeviceDiscoverer(m.logger, "", d, mig)
-		if err != nil {
-			return fmt.Errorf("failed to create MIG device: %v", err)
-		}
-
-		deviceEdits, err := edits.FromDiscoverer(device)
-		if err != nil {
-			return fmt.Errorf("failed to create container edits for MIG device: %v", err)
-		}
-
-		deviceName, err := namer.GetMigDeviceName(i, j, mig)
-		if err != nil {
-			return fmt.Errorf("failed to get device name: %v", err)
-		}
-		deviceSpec := specs.Device{
-			Name:           deviceName,
-			ContainerEdits: *deviceEdits.ContainerEdits,
-		}
-
-		deviceSpecs = append(deviceSpecs, deviceSpec)
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("falied to generate CDI spec for MIG devices: %v", err)
-	}
-
-	return deviceSpecs, nil
-}
-
-// createAllDevice creates an 'all' device which combines the edits from the previous devices
-func createAllDevice(deviceSpecs []specs.Device) specs.Device {
-	edits := edits.NewContainerEdits()
-
-	for _, d := range deviceSpecs {
-		edit := cdi.ContainerEdits{
-			ContainerEdits: &d.ContainerEdits,
-		}
-		edits.Append(&edit)
-	}
-
-	all := specs.Device{
-		Name:           "all",
-		ContainerEdits: *edits.ContainerEdits,
-	}
-	return all
-}
-
-// createParentDirsIfRequired creates the parent folders of the specified path if requried.
-// Note that MkdirAll does not specifically check whether the specified path is non-empty and raises an error if it is.
-// The path will be empty if filename in the current folder is specified, for example
-func createParentDirsIfRequired(filename string) error {
-	dir := filepath.Dir(filename)
-	if dir == "" {
-		return nil
-	}
-	return os.MkdirAll(dir, 0755)
+	return spec.New(
+		spec.WithVendor(opts.vendor),
+		spec.WithClass(opts.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(opts.format),
+		spec.WithMergedDeviceOptions(
+			transform.WithName(allDeviceName),
+			transform.WithSkipIfExists(true),
+		),
+		spec.WithPermissions(0644),
+	)
 }

cmd/nvidia-ctk/cdi/generate/mig-device.go

@@ -1,75 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package generate
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
-	"github.com/sirupsen/logrus"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-)
-
-// NewMigDeviceDiscoverer creates a discoverer for the specified mig device and its parent.
-func NewMigDeviceDiscoverer(logger *logrus.Logger, root string, parent device.Device, d device.MigDevice) (discover.Discover, error) {
-	minor, ret := parent.GetMinorNumber()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
-	}
-	parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
-
-	migCaps, err := nvcaps.NewMigCaps()
-	if err != nil {
-		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
-	}
-
-	gi, ret := d.GetGpuInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
-	}
-
-	ci, ret := d.GetComputeInstanceId()
-	if ret != nvml.SUCCESS {
-		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
-	}
-
-	giCap := nvcaps.NewGPUInstanceCap(minor, gi)
-	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
-	}
-
-	ciCap := nvcaps.NewComputeInstanceCap(minor, gi, ci)
-	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
-	}
-
-	deviceNodes := discover.NewCharDeviceDiscoverer(
-		logger,
-		[]string{
-			parentPath,
-			giCapDevicePath,
-			ciCapDevicePath,
-		},
-		root,
-	)
-
-	return deviceNodes, nil
-}

cmd/nvidia-ctk/cdi/generate/namer.go

@@ -1,84 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package generate
-
-import (
-	"fmt"
-
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
-)
-
-type deviceNamer interface {
-	GetDeviceName(int, device.Device) (string, error)
-	GetMigDeviceName(int, int, device.MigDevice) (string, error)
-}
-
-const (
-	deviceNameStrategyIndex     = "index"
-	deviceNameStrategyTypeIndex = "type-index"
-	deviceNameStrategyUUID      = "uuid"
-)
-
-type deviceNameIndex struct {
-	gpuPrefix string
-	migPrefix string
-}
-type deviceNameUUID struct{}
-
-// newDeviceNamer creates a Device Namer based on the supplied strategy.
-// This namer can be used to construct the names for MIG and GPU devices when generating the CDI spec.
-func newDeviceNamer(strategy string) (deviceNamer, error) {
-	switch strategy {
-	case deviceNameStrategyIndex:
-		return deviceNameIndex{}, nil
-	case deviceNameStrategyTypeIndex:
-		return deviceNameIndex{gpuPrefix: "gpu", migPrefix: "mig"}, nil
-	case deviceNameStrategyUUID:
-		return deviceNameUUID{}, nil
-	}
-
-	return nil, fmt.Errorf("invalid device name strategy: %v", strategy)
-}
-
-// GetDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameIndex) GetDeviceName(i int, d device.Device) (string, error) {
-	return fmt.Sprintf("%s%d", s.gpuPrefix, i), nil
-}
-
-// GetMigDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameIndex) GetMigDeviceName(i int, j int, d device.MigDevice) (string, error) {
-	return fmt.Sprintf("%s%d:%d", s.migPrefix, i, j), nil
-}
-
-// GetDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameUUID) GetDeviceName(i int, d device.Device) (string, error) {
-	uuid, ret := d.GetUUID()
-	if ret != nvml.SUCCESS {
-		return "", fmt.Errorf("failed to get device UUID: %v", ret)
-	}
-	return uuid, nil
-}
-
-// GetMigDeviceName returns the name for the specified device based on the naming strategy
-func (s deviceNameUUID) GetMigDeviceName(i int, j int, d device.MigDevice) (string, error) {
-	uuid, ret := d.GetUUID()
-	if ret != nvml.SUCCESS {
-		return "", fmt.Errorf("failed to get device UUID: %v", ret)
-	}
-	return uuid, nil
-}

cmd/nvidia-ctk/cdi/list/list.go

@@ -0,0 +1,87 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package list
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct{}
+
+// NewCommand constructs a cdi list command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the command
+	c := cli.Command{
+		Name:  "list",
+		Usage: "List the available CDI devices",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	registry, err := cdi.NewCache(
+		cdi.WithAutoRefresh(false),
+		cdi.WithSpecDirs(cdi.DefaultSpecDirs...),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI cache: %v", err)
+	}
+
+	devices := registry.ListDevices()
+	if len(devices) == 0 {
+		m.logger.Info("No CDI devices found")
+		return nil
+	}
+
+	m.logger.Infof("Found %d CDI devices", len(devices))
+	for _, device := range devices {
+		fmt.Printf("%s\n", device)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,159 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/urfave/cli/v2"
+)
+
+type loadSaver interface {
+	Load() (spec.Interface, error)
+	Save(spec.Interface) error
+}
+
+type command struct {
+	logger logger.Interface
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from string
+	to   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transform.NewRootTransformer(
+		opts.from,
+		opts.to,
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/config/config.go

@@ -0,0 +1,50 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	createdefault "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/create-default"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an config command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'config' command
+	c := cli.Command{
+		Name:  "config",
+		Usage: "Interact with the NVIDIA Container Toolkit configuration",
+	}
+
+	c.Subcommands = []*cli.Command{
+		createdefault.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/config/create-default/create-default.go

@@ -0,0 +1,183 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package defaultsubcommand
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// options stores the subcommand options
+type options struct {
+	config  string
+	output  string
+	inPlace bool
+}
+
+// NewCommand constructs a default command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'default' command
+	c := cli.Command{
+		Name:    "generate-default",
+		Aliases: []string{"default"},
+		Usage:   "Generate the default NVIDIA Container Toolkit configuration file",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "Specify the config file to process; The contents of this file overrides the default config",
+			Destination: &opts.config,
+		},
+		&cli.BoolFlag{
+			Name:        "in-place",
+			Aliases:     []string{"i"},
+			Usage:       "Modify the config file in-place",
+			Destination: &opts.inPlace,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.output,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	if opts.inPlace {
+		if opts.output != "" {
+			return fmt.Errorf("cannot specify both --in-place and --output")
+		}
+		opts.output = opts.config
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	if err := opts.ensureOutputFolder(); err != nil {
+		return fmt.Errorf("unable to create output directory: %v", err)
+	}
+
+	contents, err := opts.getFormattedConfig()
+	if err != nil {
+		return fmt.Errorf("unable to fix comments: %v", err)
+	}
+
+	if _, err := opts.Write(contents); err != nil {
+		return fmt.Errorf("unable to write to output: %v", err)
+	}
+
+	return nil
+}
+
+// getFormattedConfig returns the default config formatted as required from the specified config file.
+// The config is then formatted as required.
+// No indentation is used and comments are modified so that there is no space
+// after the '#' character.
+func (opts options) getFormattedConfig() ([]byte, error) {
+	cfg, err := config.Load(opts.config)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load or create config: %v", err)
+	}
+
+	buffer := bytes.NewBuffer(nil)
+
+	if _, err := cfg.Save(buffer); err != nil {
+		return nil, fmt.Errorf("unable to save config: %v", err)
+	}
+	return fixComments(buffer.Bytes())
+}
+
+func fixComments(contents []byte) ([]byte, error) {
+	r, err := regexp.Compile(`(\n*)\s*?#\s*(\S.*)`)
+	if err != nil {
+		return nil, fmt.Errorf("unable to compile regexp: %v", err)
+	}
+	replaced := r.ReplaceAll(contents, []byte("$1#$2"))
+
+	return replaced, nil
+}
+
+func (opts options) outputExists() (bool, error) {
+	if opts.output == "" {
+		return false, nil
+	}
+	_, err := os.Stat(opts.output)
+	if err == nil {
+		return true, nil
+	} else if !os.IsNotExist(err) {
+		return false, fmt.Errorf("unable to stat output file: %v", err)
+	}
+	return false, nil
+}
+
+func (opts options) ensureOutputFolder() error {
+	if opts.output == "" {
+		return nil
+	}
+	if dir := filepath.Dir(opts.output); dir != "" {
+		return os.MkdirAll(dir, 0755)
+	}
+	return nil
+}
+
+// Write writes the contents to the output file specified in the options.
+func (opts options) Write(contents []byte) (int, error) {
+	var output io.Writer
+	if opts.output == "" {
+		output = os.Stdout
+	} else {
+		outputFile, err := os.Create(opts.output)
+		if err != nil {
+			return 0, fmt.Errorf("unable to create output file: %v", err)
+		}
+		defer outputFile.Close()
+		output = outputFile
+	}
+
+	return output.Write(contents)
+}

cmd/nvidia-ctk/config/create-default/create-default_test.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package defaultsubcommand
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFixComment(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "# comment",
+			expected: "#comment",
+		},
+		{
+			input:    " #comment",
+			expected: "#comment",
+		},
+		{
+			input:    " # comment",
+			expected: "#comment",
+		},
+		{
+			input: strings.Join([]string{
+				"some",
+				"# comment",
+				" # comment",
+				" #comment",
+				"other"}, "\n"),
+			expected: strings.Join([]string{
+				"some",
+				"#comment",
+				"#comment",
+				"#comment",
+				"other"}, "\n"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			actual, _ := fixComments([]byte(tc.input))
+			require.Equal(t, tc.expected, string(actual))
+		})
+	}
+}
+
+func TestGetFormattedConfig(t *testing.T) {
+	expectedLines := []string{
+		"#no-cgroups = false",
+		"#debug = \"/var/log/nvidia-container-toolkit.log\"",
+		"#debug = \"/var/log/nvidia-container-runtime.log\"",
+	}
+
+	opts := &options{}
+	contents, err := opts.getFormattedConfig()
+	require.NoError(t, err)
+	lines := strings.Split(string(contents), "\n")
+
+	for _, line := range expectedLines {
+		require.Contains(t, lines, line)
+	}
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -18,18 +18,19 @@ package chmod
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -39,7 +40,7 @@ type config struct {
 }
 
 // NewCommand constructs a chmod command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -133,7 +134,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 func (m command) getPaths(root string, paths []string) []string {
 	var pathsInRoot []string
 	for _, f := range paths {
-		pathsInRoot = append(pathsInRoot, filepath.Join(root, f))
+		path := filepath.Join(root, f)
+		if _, err := os.Stat(path); err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
 	}
 
 	return pathsInRoot

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -23,14 +23,15 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -41,7 +42,7 @@ type config struct {
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -100,7 +101,10 @@ func (m command) run(c *cli.Context, cfg *config) error {
 
 	csvFiles := cfg.filenames.Value()
 
-	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
+	chainLocator := lookup.NewSymlinkChainLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(cfg.hostRoot),
+	)
 
 	var candidates []string
 	for _, file := range csvFiles {
@@ -116,7 +120,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 			}
 			targets, err := chainLocator.Locate(ms.Path)
 			if err != nil {
-				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
+				m.logger.Warningf("Failed to locate symlink %v", ms.Path)
 			}
 			candidates = append(candidates, targets...)
 		}
@@ -125,21 +129,18 @@ func (m command) run(c *cli.Context, cfg *config) error {
 	created := make(map[string]bool)
 	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
 	for _, candidate := range candidates {
-		targets, err := m.Locate(candidate)
+		target, err := symlinks.Resolve(candidate)
 		if err != nil {
 			m.logger.Debugf("Skipping invalid link: %v", err)
 			continue
-		} else if len(targets) != 1 {
-			m.logger.Debugf("Unexepected number of targets: %v", targets)
-			continue
-		} else if targets[0] == candidate {
+		} else if target == candidate {
 			m.logger.Debugf("%v is not a symlink", candidate)
 			continue
 		}
 
-		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
+		err = m.createLink(created, cfg.hostRoot, containerRoot, target, candidate)
 		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
+			m.logger.Warningf("Failed to create link %v: %v", []string{target, candidate}, err)
 		}
 	}
 
@@ -147,13 +148,13 @@ func (m command) run(c *cli.Context, cfg *config) error {
 	for _, l := range links {
 		parts := strings.Split(l, "::")
 		if len(parts) != 2 {
-			m.logger.Warnf("Invalid link specification %v", l)
+			m.logger.Warningf("Invalid link specification %v", l)
 			continue
 		}
 
 		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
 		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", parts, err)
+			m.logger.Warningf("Failed to create link %v: %v", parts, err)
 		}
 	}
 
@@ -164,7 +165,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
 	linkPath, err := changeRoot(hostRoot, containerRoot, link)
 	if err != nil {
-		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
+		m.logger.Warningf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
 	}
 	if created[linkPath] {
 		m.logger.Debugf("Link %v already created", linkPath)
@@ -173,7 +174,7 @@ func (m command) createLink(created map[string]bool, hostRoot string, containerR
 
 	targetPath, err := changeRoot(hostRoot, "/", target)
 	if err != nil {
-		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
+		m.logger.Warningf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
 	}
 
 	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)

cmd/nvidia-ctk/hook/hook.go

@@ -18,19 +18,19 @@ package hook
 
 import (
 	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type hookCommand struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := hookCommand{
 		logger: logger,
 	}

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -22,13 +22,13 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
@@ -37,7 +37,7 @@ type config struct {
 }
 
 // NewCommand constructs an update-ldcache command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -84,6 +84,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
+	_, err = os.Stat(filepath.Join(containerRoot, "/etc/ld.so.cache"))
+	if err != nil && os.IsNotExist(err) {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		return nil
+	}
+
 	err = m.createConfig(containerRoot, cfg.folders.Value())
 	if err != nil {
 		return fmt.Errorf("failed to update ld.so.conf: %v", err)
@@ -105,6 +111,10 @@ func (m command) createConfig(root string, folders []string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
 	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
 	if err != nil {
 		return fmt.Errorf("failed to create config file: %v", err)
@@ -125,5 +135,10 @@ func (m command) createConfig(root string, folders []string) error {
 		configured[folder] = true
 	}
 
+	// The created file needs to be world readable for the cases where the container is run as a non-root user.
+	if err := os.Chmod(configFile.Name(), 0644); err != nil {
+		return fmt.Errorf("failed to chmod config file: %v", err)
+	}
+
 	return nil
 }

cmd/nvidia-ctk/info/info.go

@@ -17,16 +17,16 @@
 package info
 
 import (
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs an info command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -35,13 +35,13 @@ func NewCommand(logger *logrus.Logger) *cli.Command {
 
 // build
 func (m command) build() *cli.Command {
-	// Create the 'hook' command
-	hook := cli.Command{
+	// Create the 'info' command
+	info := cli.Command{
 		Name:  "info",
 		Usage: "Provide information about the system",
 	}
 
-	hook.Subcommands = []*cli.Command{}
+	info.Subcommands = []*cli.Command{}
 
-	return &hook
+	return &info
 }

cmd/nvidia-ctk/main.go

@@ -20,28 +20,31 @@ import (
 	"os"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
 	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/sirupsen/logrus"
 
-	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 )
 
-var logger = log.New()
-
-// config defines the options that can be set for the CLI through config files,
+// options defines the options that can be set for the CLI through config files,
 // environment variables, or command line flags
-type config struct {
+type options struct {
 	// Debug indicates whether the CLI is started in "debug" mode
 	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
 }
 
 func main() {
-	// Create a config struct to hold the parsed environment variables or command line flags
-	config := config{}
+	logger := logrus.New()
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
 
 	// Create the top-level CLI
 	c := cli.NewApp()
@@ -57,16 +60,25 @@ func main() {
 			Name:        "debug",
 			Aliases:     []string{"d"},
 			Usage:       "Enable debug-level logging",
-			Destination: &config.Debug,
+			Destination: &opts.Debug,
 			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CTK_QUIET"},
+		},
 	}
 
 	// Set log-level for all subcommands
 	c.Before = func(c *cli.Context) error {
-		logLevel := log.InfoLevel
-		if config.Debug {
-			logLevel = log.DebugLevel
+		logLevel := logrus.InfoLevel
+		if opts.Debug {
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
 		}
 		logger.SetLevel(logLevel)
 		return nil
@@ -79,12 +91,13 @@ func main() {
 		infoCLI.NewCommand(logger),
 		cdi.NewCommand(logger),
 		system.NewCommand(logger),
+		config.NewCommand(logger),
 	}
 
 	// Run the CLI
 	err := c.Run(os.Args)
 	if err != nil {
-		log.Errorf("%v", err)
-		log.Exit(1)
+		logger.Errorf("%v", err)
+		os.Exit(1)
 	}
 }

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -17,31 +17,37 @@
 package configure
 
 import (
-	"encoding/json"
 	"fmt"
-	"os"
+	"path/filepath"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
-	"github.com/pelletier/go-toml"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
 	"github.com/urfave/cli/v2"
 )
 
 const (
 	defaultRuntime = "docker"
 
-	defaultDockerConfigFilePath = "/etc/docker/daemon.json"
-	defaultCrioConfigFilePath   = "/etc/crio/crio.conf"
+	// defaultNVIDIARuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	defaultNVIDIARuntimeName = "nvidia"
+	// defaultNVIDIARuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	defaultNVIDIARuntimeExecutable      = "nvidia-container-runtime"
+	defailtNVIDIARuntimeExpecutablePath = "/usr/bin/nvidia-container-runtime"
+
+	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
+	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
+	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs an configure command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -54,7 +60,12 @@ type config struct {
 	dryRun         bool
 	runtime        string
 	configFilePath string
-	nvidiaOptions  nvidia.Options
+
+	nvidiaRuntime struct {
+		name         string
+		path         string
+		setAsDefault bool
+	}
 }
 
 func (m command) build() *cli.Command {
@@ -65,6 +76,9 @@ func (m command) build() *cli.Command {
 	configure := cli.Command{
 		Name:  "configure",
 		Usage: "Add a runtime to the specified container engine",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &config)
+		},
 		Action: func(c *cli.Context) error {
 			return m.configureWrapper(c, &config)
 		},
@@ -78,7 +92,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringFlag{
 			Name:        "runtime",
-			Usage:       "the target runtime engine. One of [crio, docker]",
+			Usage:       "the target runtime engine; one of [containerd, crio, docker]",
 			Value:       defaultRuntime,
 			Destination: &config.runtime,
 		},
@@ -90,114 +104,124 @@ func (m command) build() *cli.Command {
 		&cli.StringFlag{
 			Name:        "nvidia-runtime-name",
 			Usage:       "specify the name of the NVIDIA runtime that will be added",
-			Value:       nvidia.RuntimeName,
-			Destination: &config.nvidiaOptions.RuntimeName,
+			Value:       defaultNVIDIARuntimeName,
+			Destination: &config.nvidiaRuntime.name,
 		},
 		&cli.StringFlag{
-			Name:        "runtime-path",
+			Name:        "nvidia-runtime-path",
+			Aliases:     []string{"runtime-path"},
 			Usage:       "specify the path to the NVIDIA runtime executable",
-			Value:       nvidia.RuntimeExecutable,
-			Destination: &config.nvidiaOptions.RuntimePath,
+			Value:       defaultNVIDIARuntimeExecutable,
+			Destination: &config.nvidiaRuntime.path,
 		},
 		&cli.BoolFlag{
-			Name:        "set-as-default",
-			Usage:       "set the specified runtime as the default runtime",
-			Destination: &config.nvidiaOptions.SetAsDefault,
+			Name:        "nvidia-set-as-default",
+			Aliases:     []string{"set-as-default"},
+			Usage:       "set the NVIDIA runtime as the default runtime",
+			Destination: &config.nvidiaRuntime.setAsDefault,
 		},
 	}
 
 	return &configure
 }
 
-func (m command) configureWrapper(c *cli.Context, config *config) error {
+func validateFlags(c *cli.Context, config *config) error {
 	switch config.runtime {
+	case "containerd", "crio", "docker":
+		break
+	default:
+		return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+
+	switch config.runtime {
+	case "containerd", "crio":
+		if config.nvidiaRuntime.path == defaultNVIDIARuntimeExecutable {
+			config.nvidiaRuntime.path = defailtNVIDIARuntimeExpecutablePath
+		}
+		if !filepath.IsAbs(config.nvidiaRuntime.path) {
+			return fmt.Errorf("the NVIDIA runtime path %q is not an absolute path", config.nvidiaRuntime.path)
+		}
+	}
+
+	return nil
+}
+
+// configureWrapper updates the specified container engine config to enable the NVIDIA runtime
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	configFilePath := config.resolveConfigFilePath()
+
+	var cfg engine.Interface
+	var err error
+	switch config.runtime {
+	case "containerd":
+		cfg, err = containerd.New(
+			containerd.WithLogger(m.logger),
+			containerd.WithPath(configFilePath),
+		)
 	case "crio":
-		return m.configureCrio(c, config)
+		cfg, err = crio.New(
+			crio.WithLogger(m.logger),
+			crio.WithPath(configFilePath),
+		)
 	case "docker":
-		return m.configureDocker(c, config)
+		cfg, err = docker.New(
+			docker.WithLogger(m.logger),
+			docker.WithPath(configFilePath),
+		)
+	default:
+		err = fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+	if err != nil || cfg == nil {
+		return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
 	}
 
-	return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
-}
-
-// configureDocker updates the docker config to enable the NVIDIA Container Runtime
-func (m command) configureDocker(c *cli.Context, config *config) error {
-	configFilePath := config.configFilePath
-	if configFilePath == "" {
-		configFilePath = defaultDockerConfigFilePath
-	}
-
-	cfg, err := docker.LoadConfig(configFilePath)
-	if err != nil {
-		return fmt.Errorf("unable to load config: %v", err)
-	}
-
-	err = docker.UpdateConfig(
-		cfg,
-		config.nvidiaOptions.RuntimeName,
-		config.nvidiaOptions.RuntimePath,
-		config.nvidiaOptions.SetAsDefault,
+	err = cfg.AddRuntime(
+		config.nvidiaRuntime.name,
+		config.nvidiaRuntime.path,
+		config.nvidiaRuntime.setAsDefault,
 	)
 	if err != nil {
 		return fmt.Errorf("unable to update config: %v", err)
 	}
 
-	if config.dryRun {
-		output, err := json.MarshalIndent(cfg, "", "    ")
-		if err != nil {
-			return fmt.Errorf("unable to convert to JSON: %v", err)
-		}
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return nil
-	}
-	err = docker.FlushConfig(cfg, configFilePath)
+	outputPath := config.getOuputConfigPath()
+	n, err := cfg.Save(outputPath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)
 	}
 
-	m.logger.Infof("Wrote updated config to %v", configFilePath)
-	m.logger.Infof("It is recommended that the docker daemon be restarted.")
-
-	return nil
-}
-
-// configureCrio updates the crio config to enable the NVIDIA Container Runtime
-func (m command) configureCrio(c *cli.Context, config *config) error {
-	configFilePath := config.configFilePath
-	if configFilePath == "" {
-		configFilePath = defaultCrioConfigFilePath
-	}
-
-	cfg, err := crio.LoadConfig(configFilePath)
-	if err != nil {
-		return fmt.Errorf("unable to load config: %v", err)
-	}
-
-	err = crio.UpdateConfig(
-		cfg,
-		config.nvidiaOptions.RuntimeName,
-		config.nvidiaOptions.RuntimePath,
-		config.nvidiaOptions.SetAsDefault,
-	)
-	if err != nil {
-		return fmt.Errorf("unable to update config: %v", err)
-	}
-
-	if config.dryRun {
-		output, err := toml.Marshal(cfg)
-		if err != nil {
-			return fmt.Errorf("unable to convert to TOML: %v", err)
+	if outputPath != "" {
+		if n == 0 {
+			m.logger.Infof("Removed empty config from %v", outputPath)
+		} else {
+			m.logger.Infof("Wrote updated config to %v", outputPath)
 		}
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return nil
+		m.logger.Infof("It is recommended that %v daemon be restarted.", config.runtime)
 	}
-	err = crio.FlushConfig(configFilePath, cfg)
-	if err != nil {
-		return fmt.Errorf("unable to flush config: %v", err)
-	}
-
-	m.logger.Infof("Wrote updated config to %v", configFilePath)
-	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
 
 	return nil
 }
+
+// resolveConfigFilePath returns the default config file path for the configured container engine
+func (c *config) resolveConfigFilePath() string {
+	if c.configFilePath != "" {
+		return c.configFilePath
+	}
+	switch c.runtime {
+	case "containerd":
+		return defaultContainerdConfigFilePath
+	case "crio":
+		return defaultCrioConfigFilePath
+	case "docker":
+		return defaultDockerConfigFilePath
+	}
+	return ""
+}
+
+// getOuputConfigPath returns the configured config path or "" if dry-run is enabled
+func (c *config) getOuputConfigPath() string {
+	if c.dryRun {
+		return ""
+	}
+	return c.resolveConfigFilePath()
+}

cmd/nvidia-ctk/runtime/nvidia/nvidia.go

@@ -1,75 +0,0 @@
-/*
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package nvidia
-
-const (
-	// RuntimeName is the default name to use in configs for the NVIDIA Container Runtime
-	RuntimeName = "nvidia"
-	// RuntimeExecutable is the default NVIDIA Container Runtime executable file name
-	RuntimeExecutable = "nvidia-container-runtime"
-)
-
-// Options specifies the options for the NVIDIA Container Runtime w.r.t a container engine such as docker.
-type Options struct {
-	SetAsDefault bool
-	RuntimeName  string
-	RuntimePath  string
-}
-
-// Runtime defines an NVIDIA runtime with a name and a executable
-type Runtime struct {
-	Name string
-	Path string
-}
-
-// DefaultRuntime returns the default runtime for the configured options.
-// If the configuration is invalid or the default runtimes should not be set
-// the empty string is returned.
-func (o Options) DefaultRuntime() string {
-	if !o.SetAsDefault {
-		return ""
-	}
-
-	return o.RuntimeName
-}
-
-// Runtime creates a runtime struct based on the options.
-func (o Options) Runtime() Runtime {
-	path := o.RuntimePath
-
-	if o.RuntimePath == "" {
-		path = RuntimeExecutable
-	}
-
-	r := Runtime{
-		Name: o.RuntimeName,
-		Path: path,
-	}
-
-	return r
-}
-
-// DockerRuntimesConfig generatest the expected docker config for the specified runtime
-func (r Runtime) DockerRuntimesConfig() map[string]interface{} {
-	runtimes := make(map[string]interface{})
-	runtimes[r.Name] = map[string]interface{}{
-		"path": r.Path,
-		"args": []string{},
-	}
-
-	return runtimes
-}

cmd/nvidia-ctk/runtime/runtime.go

@@ -18,16 +18,16 @@ package runtime
 
 import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/urfave/cli/v2"
 )
 
 type runtimeCommand struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs a runtime command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := runtimeCommand{
 		logger: logger,
 	}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -21,36 +21,47 @@ import (
 	"path/filepath"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
-	"github.com/sirupsen/logrus"
 	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
 )
 
 type allPossible struct {
-	logger       *logrus.Logger
-	driverRoot   string
+	logger       logger.Interface
+	devRoot      string
 	deviceMajors devices.Devices
 	migCaps      nvcaps.MigCaps
 }
 
 // newAllPossible returns a new allPossible device node lister.
 // This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
-func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error) {
+func newAllPossible(logger logger.Interface, devRoot string) (nodeLister, error) {
 	deviceMajors, err := devices.GetNVIDIADevices()
 	if err != nil {
 		return nil, fmt.Errorf("failed reading device majors: %v", err)
 	}
+
+	var requiredMajors []devices.Name
 	migCaps, err := nvcaps.NewMigCaps()
 	if err != nil {
 		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
 	}
 	if migCaps == nil {
 		migCaps = make(nvcaps.MigCaps)
+	} else {
+		requiredMajors = append(requiredMajors, devices.NVIDIACaps)
+	}
+
+	requiredMajors = append(requiredMajors, devices.NVIDIAGPU, devices.NVIDIAUVM)
+	for _, name := range requiredMajors {
+		if !deviceMajors.Exists(name) {
+			return nil, fmt.Errorf("missing required device major %s", name)
+		}
 	}
 
 	l := allPossible{
 		logger:       logger,
-		driverRoot:   driverRoot,
+		devRoot:      devRoot,
 		deviceMajors: deviceMajors,
 		migCaps:      migCaps,
 	}
@@ -60,8 +71,8 @@ func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error
 
 // DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
 func (m allPossible) DeviceNodes() ([]deviceNode, error) {
-	gpus, err := nvpci.NewFrom(
-		filepath.Join(m.driverRoot, nvpci.PCIDevicesRoot),
+	gpus, err := nvpci.New(
+		nvpci.WithPCIDevicesRoot(filepath.Join(m.devRoot, nvpci.PCIDevicesRoot)),
 	).GetGPUs()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get GPU information: %v", err)
@@ -69,7 +80,7 @@ func (m allPossible) DeviceNodes() ([]deviceNode, error) {
 
 	count := len(gpus)
 	if count == 0 {
-		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
 		return nil, nil
 	}
 
@@ -168,7 +179,7 @@ func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor i
 	major, _ := m.deviceMajors.Get(deviceName)
 
 	return deviceNode{
-		path:  filepath.Join(m.driverRoot, path),
+		path:  filepath.Join(m.devRoot, path),
 		major: uint32(major),
 		minor: uint32(minor),
 	}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -24,8 +24,10 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
 	"github.com/fsnotify/fsnotify"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
@@ -34,19 +36,21 @@ const (
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 type config struct {
-	devCharPath string
-	driverRoot  string
-	dryRun      bool
-	watch       bool
-	createAll   bool
+	devCharPath       string
+	driverRoot        string
+	dryRun            bool
+	watch             bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
 }
 
 // NewCommand constructs a command sub-command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -97,6 +101,18 @@ func (m command) build() *cli.Command {
 			Destination: &cfg.createAll,
 			EnvVars:     []string{"CREATE_ALL"},
 		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "Load the NVIDIA kernel modules before creating symlinks. This is only applicable when --create-all is set.",
+			Destination: &cfg.loadKernelModules,
+			EnvVars:     []string{"LOAD_KERNEL_MODULES"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-device-nodes",
+			Usage:       "Create the NVIDIA control device nodes in the driver root if they do not exist. This is only applicable when --create-all is set",
+			Destination: &cfg.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
 		&cli.BoolFlag{
 			Name:        "dry-run",
 			Usage:       "If set, the command will not create any symlinks.",
@@ -114,6 +130,16 @@ func (m command) validateFlags(r *cli.Context, cfg *config) error {
 		return fmt.Errorf("create-all and watch are mutually exclusive")
 	}
 
+	if cfg.loadKernelModules && !cfg.createAll {
+		m.logger.Warning("load-kernel-modules is only applicable when create-all is set; ignoring")
+		cfg.loadKernelModules = false
+	}
+
+	if cfg.createDeviceNodes && !cfg.createAll {
+		m.logger.Warning("create-device-nodes is only applicable when create-all is set; ignoring")
+		cfg.createDeviceNodes = false
+	}
+
 	return nil
 }
 
@@ -137,6 +163,8 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		WithDriverRoot(cfg.driverRoot),
 		WithDryRun(cfg.dryRun),
 		WithCreateAll(cfg.createAll),
+		WithLoadKernelModules(cfg.loadKernelModules),
+		WithCreateDeviceNodes(cfg.createDeviceNodes),
 	)
 	if err != nil {
 		return fmt.Errorf("failed to create symlink creator: %v", err)
@@ -186,12 +214,15 @@ create:
 }
 
 type linkCreator struct {
-	logger      *logrus.Logger
-	lister      nodeLister
-	driverRoot  string
-	devCharPath string
-	dryRun      bool
-	createAll   bool
+	logger            logger.Interface
+	lister            nodeLister
+	driverRoot        string
+	devRoot           string
+	devCharPath       string
+	dryRun            bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
 }
 
 // Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
@@ -209,34 +240,81 @@ func NewSymlinkCreator(opts ...Option) (Creator, error) {
 		opt(&c)
 	}
 	if c.logger == nil {
-		c.logger = logrus.StandardLogger()
+		c.logger = logger.New()
 	}
 	if c.driverRoot == "" {
 		c.driverRoot = "/"
 	}
+	if c.devRoot == "" {
+		c.devRoot = "/"
+	}
 	if c.devCharPath == "" {
 		c.devCharPath = defaultDevCharPath
 	}
 
+	if err := c.setup(); err != nil {
+		return nil, err
+	}
+
 	if c.createAll {
-		lister, err := newAllPossible(c.logger, c.driverRoot)
+		lister, err := newAllPossible(c.logger, c.devRoot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
 		}
 		c.lister = lister
 	} else {
-		c.lister = existing{c.logger, c.driverRoot}
+		c.lister = existing{c.logger, c.devRoot}
 	}
 	return c, nil
 }
 
+func (m linkCreator) setup() error {
+	if !m.loadKernelModules && !m.createDeviceNodes {
+		return nil
+	}
+
+	if m.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(m.dryRun),
+			nvmodules.WithRoot(m.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if m.createDeviceNodes {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(m.dryRun),
+			nvdevices.WithDevRoot(m.devRoot),
+		)
+		if err != nil {
+			return err
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA device nodes: %v", err)
+		}
+	}
+	return nil
+}
+
 // WithDriverRoot sets the driver root path.
+// This is the path in which kernel modules must be loaded.
 func WithDriverRoot(root string) Option {
 	return func(c *linkCreator) {
 		c.driverRoot = root
 	}
 }
 
+// WithDevRoot sets the root path for the /dev directory.
+func WithDevRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.devRoot = root
+	}
+}
+
 // WithDevCharPath sets the path at which the symlinks will be created.
 func WithDevCharPath(path string) Option {
 	return func(c *linkCreator) {
@@ -252,7 +330,7 @@ func WithDryRun(dryRun bool) Option {
 }
 
 // WithLogger sets the logger.
-func WithLogger(logger *logrus.Logger) Option {
+func WithLogger(logger logger.Interface) Option {
 	return func(c *linkCreator) {
 		c.logger = logger
 	}
@@ -265,6 +343,20 @@ func WithCreateAll(createAll bool) Option {
 	}
 }
 
+// WithLoadKernelModules sets the loadKernelModules flag for the linkCreator.
+func WithLoadKernelModules(loadKernelModules bool) Option {
+	return func(lc *linkCreator) {
+		lc.loadKernelModules = loadKernelModules
+	}
+}
+
+// WithCreateDeviceNodes sets the createDeviceNodes flag for the linkCreator.
+func WithCreateDeviceNodes(createDeviceNodes bool) Option {
+	return func(lc *linkCreator) {
+		lc.createDeviceNodes = createDeviceNodes
+	}
+}
+
 // CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
 func (m linkCreator) CreateLinks() error {
 	deviceNodes, err := m.lister.DeviceNodes()
@@ -290,7 +382,7 @@ func (m linkCreator) CreateLinks() error {
 
 		err = os.Symlink(target, linkPath)
 		if err != nil {
-			m.logger.Warnf("Could not create symlink: %v", err)
+			m.logger.Warningf("Could not create symlink: %v", err)
 		}
 	}
 

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -20,8 +20,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
@@ -30,8 +30,8 @@ type nodeLister interface {
 }
 
 type existing struct {
-	logger     *logrus.Logger
-	driverRoot string
+	logger  logger.Interface
+	devRoot string
 }
 
 // DeviceNodes returns a list of NVIDIA device nodes in the specified root.
@@ -39,22 +39,22 @@ type existing struct {
 func (m existing) DeviceNodes() ([]deviceNode, error) {
 	locator := lookup.NewCharDeviceLocator(
 		lookup.WithLogger(m.logger),
-		lookup.WithRoot(m.driverRoot),
+		lookup.WithRoot(m.devRoot),
 		lookup.WithOptional(true),
 	)
 
 	devices, err := locator.Locate("/dev/nvidia*")
 	if err != nil {
-		m.logger.Warnf("Error while locating device: %v", err)
+		m.logger.Warningf("Error while locating device: %v", err)
 	}
 
 	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
 	if err != nil {
-		m.logger.Warnf("Error while locating caps device: %v", err)
+		m.logger.Warningf("Error while locating caps device: %v", err)
 	}
 
 	if len(devices) == 0 && len(capDevices) == 0 {
-		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
 		return nil, nil
 	}
 
@@ -67,7 +67,7 @@ func (m existing) DeviceNodes() ([]deviceNode, error) {
 		var stat unix.Stat_t
 		err := unix.Stat(d, &stat)
 		if err != nil {
-			m.logger.Warnf("Could not stat device: %v", err)
+			m.logger.Warningf("Could not stat device: %v", err)
 			continue
 		}
 		deviceNode := deviceNode{

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,126 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverRoot string
+
+	dryRun bool
+
+	control bool
+
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device nodes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "load the NVIDIA Kernel Modules before creating devices nodes",
+			Destination: &opts.loadKernelModules,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	if opts.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(opts.dryRun),
+			nvmodules.WithRoot(opts.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if opts.control {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(opts.dryRun),
+			nvdevices.WithDevRoot(opts.driverRoot),
+		)
+		if err != nil {
+			return err
+		}
+		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/print-ldcache/print-ldcache.go

@@ -0,0 +1,101 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverRoot string
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "print-ldcache",
+		Usage: "A utility to print the contents of the ldcache",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	cache, err := ldcache.New(m.logger, opts.driverRoot)
+	if err != nil {
+		return fmt.Errorf("failed to create ldcache: %v", err)
+	}
+
+	lib32, lib64 := cache.List()
+
+	if len(lib32) == 0 {
+		m.logger.Info("No 32-bit libraries found")
+	} else {
+		m.logger.Infof("%d 32-bit libraries found", len(lib32))
+		for _, lib := range lib32 {
+			m.logger.Infof("%v", lib)
+		}
+	}
+	if len(lib64) == 0 {
+		m.logger.Info("No 64-bit libraries found")
+	} else {
+		m.logger.Infof("%d 64-bit libraries found", len(lib64))
+		for _, lib := range lib64 {
+			m.logger.Infof("%v", lib)
+		}
+	}
+
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -18,16 +18,18 @@ package system
 
 import (
 	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
-	"github.com/sirupsen/logrus"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/print-ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/urfave/cli/v2"
 )
 
 type command struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs a runtime command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := command{
 		logger: logger,
 	}
@@ -43,6 +45,8 @@ func (m command) build() *cli.Command {
 
 	system.Subcommands = []*cli.Command{
 		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
+		ldcache.NewCommand(m.logger),
 	}
 
 	return &system

config/config.toml.debian

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.opensuse-leap

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.rpm-yum

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig.real"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

docker/Dockerfile.debian

@@ -53,15 +53,6 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Debian Jessie still had ldconfig.real
-RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
-       sed -i 's;"@/sbin/ldconfig";"@/sbin/ldconfig.real";' $DIST_DIR/config.toml; \
-    fi
-
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 

docker/Dockerfile.opensuse-leap

@@ -50,10 +50,6 @@ COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
 # Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
 COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 

docker/Dockerfile.rpm-yum

@@ -62,10 +62,6 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
 

docker/Dockerfile.ubuntu

@@ -51,10 +51,6 @@ ARG GIT_COMMIT
 ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 

docker/docker.mk

@@ -14,10 +14,10 @@
 
 # Supported OSs by architecture
 AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
-X86_64_TARGETS := fedora35 centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
+X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := fedora35 centos8 rhel8 amazonlinux2
+AARCH64_TARGETS := centos7 centos8 rhel8 amazonlinux2
 
 # Define top-level build targets
 docker%: SHELL:=/bin/bash
@@ -88,63 +88,35 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
 LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
 
+LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
---ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---ubuntu%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
---ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
---debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---debian%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
---debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
---centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
 --centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---centos%: CONFIG_TOML_SUFFIX := rpm-yum
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
-# private fedora target
---fedora%: OS := fedora
---fedora%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
---fedora%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---fedora%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---fedora%: CONFIG_TOML_SUFFIX := rpm-yum
-# The fedora(35) base image has very slow performance when building aarch64 packages.
-# Since our primary concern here is glibc versions, we use the older glibc version available in centos8.
---fedora35%: BASEIMAGE = quay.io/centos/centos:stream8
-
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
 --rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---rhel%: CONFIG_TOML_SUFFIX := rpm-yum
 --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 
-# We allow the CONFIG_TOML_SUFFIX to be overridden.
-CONFIG_TOML_SUFFIX ?= $(OS)
-
 docker-build-%:
 	@echo "Building for $(TARGET_PLATFORM)"
 	docker pull --platform=linux/$(ARCH) $(BASEIMAGE)
@@ -155,10 +127,9 @@ docker-build-%:
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 	    --build-arg PKG_NAME="$(LIB_NAME)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
+	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
 	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
-	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
 	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .

go.mod

@@ -1,31 +1,30 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.18
+go 1.20
 
 require (
-	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82
-	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
+	github.com/BurntSushi/toml v1.2.1
+	github.com/NVIDIA/go-nvml v0.12.0-1
+	github.com/container-orchestrated-devices/container-device-interface v0.6.0
 	github.com/fsnotify/fsnotify v1.5.4
-	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
+	github.com/opencontainers/runtime-spec v1.1.0-rc.2
 	github.com/pelletier/go-toml v1.9.4
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
-	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230613182322-7663cf900f0a
 	golang.org/x/mod v0.5.0
-	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
-	sigs.k8s.io/yaml v1.3.0
+	golang.org/x/sys v0.7.0
 )
 
 require (
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/opencontainers/runc v1.1.6 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
-	github.com/opencontainers/selinux v1.10.1 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
@@ -33,4 +32,5 @@ require (
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -1,84 +1,75 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
-github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82 h1:x751Xx1tdxkiA/sdkv2J769n21UbYKzVOpe9S/h1M3k=
-github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/NVIDIA/go-nvml v0.12.0-1 h1:6mdjtlFo+17dWL7VFPfuRMtf0061TF4DKls9pkSw6uM=
+github.com/NVIDIA/go-nvml v0.12.0-1/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
-github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
-github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a h1:sP3PcgyIkRlHqfF3Jfpe/7G8kf/qpzG4C8r94y9hLbE=
-github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a/go.mod h1:xMRa4fJgXzSDFUCURSimOUgoSc+odohvO3uXT9xjqH0=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0 h1:aWwcz/Ep0Fd7ZuBjQGjU/jdPloM7ydhMW13h85jZNvk=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0/go.mod h1:OQlgtJtDrOxSQ1BWODC8OZK1tzi9W69wek+Jy17ndzo=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
-github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
-github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg=
-github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
-github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb h1:1xSVPOd7/UA+39/hXEGnBJ13p6JFB0E1EvQFlrRDOXI=
+github.com/opencontainers/runc v1.1.6 h1:XbhB8IfG/EsnhNvZtNdLB0GBw92GYEFvKlhaJk9jUgA=
+github.com/opencontainers/runc v1.1.6/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50=
 github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.1.0-rc.2 h1:ucBtEms2tamYYW/SvGpvq9yUN0NEVL6oyLEwDcTSrk8=
+github.com/opencontainers/runtime-spec v1.1.0-rc.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
 github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w=
-github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=
+github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -86,33 +77,19 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342 h1:083n9fJt2dWOpJd/X/q9Xgl5XtQLL22uSFYbzVqJssg=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230613182322-7663cf900f0a h1:lceJVurLqiWFdxK6KMDw+SIwrAsFW/af44XrNlbGw78=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230613182322-7663cf900f0a/go.mod h1:KYZksBgh18o+uzgnpDazzG4LVYtnfB96VXHMXypEtik=
 golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6 h1:cy1ko5847T/lJ45eyg/7uLprIE/amW5IXxGtEnQdYMI=
-golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

internal/config/cli.go

@@ -16,33 +16,10 @@
 
 package config
 
-import (
-	"github.com/pelletier/go-toml"
-)
-
 // ContainerCLIConfig stores the options for the nvidia-container-cli
 type ContainerCLIConfig struct {
-	Root string
-}
-
-// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
-	cfg := getDefaultContainerCLIConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
-
-	return cfg
-}
-
-// getDefaultContainerCLIConfig defines the default values for the config
-func getDefaultContainerCLIConfig() *ContainerCLIConfig {
-	c := ContainerCLIConfig{
-		Root: "",
-	}
-
-	return &c
+	Root        string   `toml:"root"`
+	LoadKmods   bool     `toml:"load-kmods"`
+	Ldconfig    string   `toml:"ldconfig"`
+	Environment []string `toml:"environment"`
 }

internal/config/config.go

@@ -17,17 +17,29 @@
 package config
 
 import (
+	"bufio"
 	"fmt"
 	"io"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/pelletier/go-toml"
 )
 
 const (
 	configOverride = "XDG_CONFIG_HOME"
 	configFilePath = "nvidia-container-runtime/config.toml"
+
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+
+	nvidiaContainerRuntimeHookExecutable  = "nvidia-container-runtime-hook"
+	nvidiaContainerRuntimeHookDefaultPath = "/usr/bin/nvidia-container-runtime-hook"
 )
 
 var (
@@ -45,9 +57,13 @@ var (
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
-	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+	DisableRequire           bool `toml:"disable-require"`
+	AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+
+	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file
@@ -59,13 +75,22 @@ func GetConfig() (*Config, error) {
 
 	configFilePath := path.Join(configDir, configFilePath)
 
+	return Load(configFilePath)
+}
+
+// Load loads the config from the specified file path.
+func Load(configFilePath string) (*Config, error) {
+	if configFilePath == "" {
+		return getDefault()
+	}
+
 	tomlFile, err := os.Open(configFilePath)
 	if err != nil {
-		return getDefaultConfig(), nil
+		return getDefault()
 	}
 	defer tomlFile.Close()
 
-	cfg, err := loadConfigFrom(tomlFile)
+	cfg, err := LoadFrom(tomlFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read config values: %v", err)
 	}
@@ -73,42 +98,224 @@ func GetConfig() (*Config, error) {
 	return cfg, nil
 }
 
-// loadRuntimeConfigFrom reads the config from the specified Reader
-func loadConfigFrom(reader io.Reader) (*Config, error) {
-	toml, err := toml.LoadReader(reader)
+// LoadFrom reads the config from the specified Reader
+func LoadFrom(reader io.Reader) (*Config, error) {
+	var tree *toml.Tree
+	if reader != nil {
+		toml, err := toml.LoadReader(reader)
+		if err != nil {
+			return nil, err
+		}
+		tree = toml
+	}
+
+	return getFromTree(tree)
+}
+
+// getFromTree reads the nvidia container runtime config from the specified toml Tree.
+func getFromTree(toml *toml.Tree) (*Config, error) {
+	cfg, err := getDefault()
 	if err != nil {
 		return nil, err
 	}
-
-	return getConfigFrom(toml)
-}
-
-// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getConfigFrom(toml *toml.Tree) (*Config, error) {
-	cfg := getDefaultConfig()
-
 	if toml == nil {
 		return cfg, nil
 	}
 
-	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
-	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
-	runtimeConfig, err := getRuntimeConfigFrom(toml)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
+	if err := toml.Unmarshal(cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal config: %v", err)
 	}
-	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
 
 	return cfg, nil
 }
 
-// getDefaultConfig defines the default values for the config
-func getDefaultConfig() *Config {
-	c := Config{
-		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
-		NVIDIACTKConfig:              *getDefaultCTKConfig(),
-		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
+// getDefault defines the default values for the config
+func getDefault() (*Config, error) {
+	d := Config{
+		AcceptEnvvarUnprivileged: true,
+		NVIDIAContainerCLIConfig: ContainerCLIConfig{
+			LoadKmods: true,
+			Ldconfig:  getLdConfigPath(),
+		},
+		NVIDIACTKConfig: CTKConfig{
+			Path: nvidiaCTKExecutable,
+		},
+		NVIDIAContainerRuntimeConfig: RuntimeConfig{
+			DebugFilePath: "/dev/null",
+			LogLevel:      "info",
+			Runtimes:      []string{"docker-runc", "runc"},
+			Mode:          "auto",
+			Modes: modesConfig{
+				CSV: csvModeConfig{
+					MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+				},
+				CDI: cdiModeConfig{
+					DefaultKind:        "nvidia.com/gpu",
+					AnnotationPrefixes: []string{cdi.AnnotationPrefix},
+					SpecDirs:           cdi.DefaultSpecDirs,
+				},
+			},
+		},
+		NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+			Path: NVIDIAContainerRuntimeHookExecutable,
+		},
+	}
+	return &d, nil
+}
+
+func getLdConfigPath() string {
+	if _, err := os.Stat("/sbin/ldconfig.real"); err == nil {
+		return "@/sbin/ldconfig.real"
+	}
+	return "@/sbin/ldconfig"
+}
+
+// getCommentedUserGroup returns whether the nvidia-container-cli user and group config option should be commented.
+func getCommentedUserGroup() bool {
+	uncommentIf := map[string]bool{
+		"suse":     true,
+		"opensuse": true,
 	}
 
-	return &c
+	idsLike := getDistIDLike()
+	for _, id := range idsLike {
+		if uncommentIf[id] {
+			return false
+		}
+	}
+	return true
+}
+
+// getDistIDLike returns the ID_LIKE field from /etc/os-release.
+func getDistIDLike() []string {
+	releaseFile, err := os.Open("/etc/os-release")
+	if err != nil {
+		return nil
+	}
+	defer releaseFile.Close()
+
+	scanner := bufio.NewScanner(releaseFile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "ID_LIKE=") {
+			value := strings.Trim(strings.TrimPrefix(line, "ID_LIKE="), "\"")
+			return strings.Split(value, " ")
+		}
+	}
+	return nil
+}
+
+// ResolveNVIDIACTKPath resolves the path to the nvidia-ctk binary.
+// This executable is used in hooks and needs to be an absolute path.
+// If the path is specified as an absolute path, it is used directly
+// without checking for existence of an executable at that path.
+func ResolveNVIDIACTKPath(logger logger.Interface, nvidiaCTKPath string) string {
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Toolkit CLI",
+		nvidiaCTKPath,
+		nvidiaCTKDefaultFilePath,
+	)
+}
+
+// ResolveNVIDIAContainerRuntimeHookPath resolves the path the nvidia-container-runtime-hook binary.
+func ResolveNVIDIAContainerRuntimeHookPath(logger logger.Interface, nvidiaContainerRuntimeHookPath string) string {
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Runtime Hook",
+		nvidiaContainerRuntimeHookPath,
+		nvidiaContainerRuntimeHookDefaultPath,
+	)
+}
+
+// resolveWithDefault resolves the path to the specified binary.
+// If an absolute path is specified, it is used directly without searching for the binary.
+// If the binary cannot be found in the path, the specified default is used instead.
+func resolveWithDefault(logger logger.Interface, label string, path string, defaultPath string) string {
+	if filepath.IsAbs(path) {
+		logger.Debugf("Using specified %v path %v", label, path)
+		return path
+	}
+
+	if path == "" {
+		path = filepath.Base(defaultPath)
+	}
+	logger.Debugf("Locating %v as %v", label, path)
+	lookup := lookup.NewExecutableLocator(logger, "")
+
+	resolvedPath := defaultPath
+	targets, err := lookup.Locate(path)
+	if err != nil {
+		logger.Warningf("Failed to locate %v: %v", path, err)
+	} else {
+		logger.Debugf("Found %v candidates: %v", path, targets)
+		resolvedPath = targets[0]
+	}
+	logger.Debugf("Using %v path %v", label, path)
+
+	return resolvedPath
+}
+
+func (c Config) asCommentedToml() (*toml.Tree, error) {
+	contents, err := toml.Marshal(c)
+	if err != nil {
+		return nil, err
+	}
+	asToml, err := toml.LoadBytes(contents)
+	if err != nil {
+		return nil, err
+	}
+
+	commentedDefaults := map[string]interface{}{
+		"swarm-resource": "DOCKER_RESOURCE_GPU",
+		"accept-nvidia-visible-devices-envvar-when-unprivileged": true,
+		"accept-nvidia-visible-devices-as-volume-mounts":         false,
+		"nvidia-container-cli.root":                              "/run/nvidia/driver",
+		"nvidia-container-cli.path":                              "/usr/bin/nvidia-container-cli",
+		"nvidia-container-cli.debug":                             "/var/log/nvidia-container-toolkit.log",
+		"nvidia-container-cli.ldcache":                           "/etc/ld.so.cache",
+		"nvidia-container-cli.no-cgroups":                        false,
+		"nvidia-container-cli.user":                              "root:video",
+		"nvidia-container-runtime.debug":                         "/var/log/nvidia-container-runtime.log",
+	}
+	for k, v := range commentedDefaults {
+		set := asToml.Get(k)
+		if !shouldComment(k, v, set) {
+			continue
+		}
+		asToml.SetWithComment(k, "", true, v)
+	}
+
+	return asToml, nil
+}
+
+func shouldComment(key string, defaultValue interface{}, setTo interface{}) bool {
+	if key == "nvidia-container-cli.root" && setTo == "" {
+		return true
+	}
+	if key == "nvidia-container-cli.user" && !getCommentedUserGroup() {
+		return false
+	}
+	if key == "nvidia-container-runtime.debug" && setTo == "/dev/null" {
+		return true
+	}
+	if setTo == nil || defaultValue == setTo {
+		return true
+	}
+
+	return false
+}
+
+// Save writes the config to the specified writer.
+func (c Config) Save(w io.Writer) (int64, error) {
+	asToml, err := c.asCommentedToml()
+	if err != nil {
+		return 0, err
+	}
+
+	enc := toml.NewEncoder(w).Indentation("")
+	if err := enc.Encode(asToml); err != nil {
+		return 0, fmt.Errorf("invalid config: %v", err)
+	}
+	return 0, nil
 }

internal/config/config_test.go

@@ -17,6 +17,7 @@
 package config
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -49,16 +50,21 @@ func TestGetConfigWithCustomConfig(t *testing.T) {
 
 func TestGetConfig(t *testing.T) {
 	testCases := []struct {
-		description    string
-		contents       []string
-		expectedError  error
-		expectedConfig *Config
+		description     string
+		contents        []string
+		expectedError   error
+		inspectLdconfig bool
+		expectedConfig  *Config
 	}{
 		{
-			description: "empty config is default",
+			description:     "empty config is default",
+			inspectLdconfig: true,
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: true,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "",
+					Root:      "",
+					LoadKmods: true,
+					Ldconfig:  "WAS_CHECKED",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/dev/null",
@@ -69,8 +75,16 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+							SpecDirs:           []string{"/etc/cdi", "/var/run/cdi"},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "nvidia-ctk",
 				},
@@ -79,19 +93,28 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set inline",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"nvidia-container-cli.root = \"/bar/baz\"",
+				"nvidia-container-cli.load-kmods = false",
+				"nvidia-container-cli.ldconfig = \"/foo/bar/ldconfig\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
-				"nvidia-container-runtime.experimental = true",
 				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
+				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"nvidia-container-runtime.modes.cdi.spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"nvidia-container-runtime-hook.path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -102,8 +125,22 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "/foo/bar/nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
 				},
@@ -112,23 +149,34 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set in section",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
+				"load-kmods = false",
+				"ldconfig = \"/foo/bar/ldconfig\"",
 				"[nvidia-container-runtime]",
 				"debug = \"/foo/bar\"",
-				"experimental = true",
 				"discover-mode = \"not-legacy\"",
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
 				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.cdi]",
+				"default-kind = \"example.vendor.com/device\"",
+				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
+				"spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"[nvidia-container-runtime-hook]",
+				"path = \"/foo/bar/nvidia-container-runtime-hook\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
+					Root:      "/bar/baz",
+					LoadKmods: false,
+					Ldconfig:  "/foo/bar/ldconfig",
 				},
 				NVIDIAContainerRuntimeConfig: RuntimeConfig{
 					DebugFilePath: "/foo/bar",
@@ -139,8 +187,22 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+							SpecDirs: []string{
+								"/except/etc/cdi",
+								"/not/var/run/cdi",
+							},
+						},
 					},
 				},
+				NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
+					Path: "/foo/bar/nvidia-container-runtime-hook",
+				},
 				NVIDIACTKConfig: CTKConfig{
 					Path: "/foo/bar/nvidia-ctk",
 				},
@@ -152,14 +214,68 @@ func TestGetConfig(t *testing.T) {
 		t.Run(tc.description, func(t *testing.T) {
 			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
 
-			cfg, err := loadConfigFrom(reader)
+			cfg, err := LoadFrom(reader)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
 			}
 
+			// We first handle the ldconfig path since this is currently system-dependent.
+			if tc.inspectLdconfig {
+				ldconfig := cfg.NVIDIAContainerCLIConfig.Ldconfig
+				require.True(t, strings.HasPrefix(ldconfig, "@/sbin/ldconfig"))
+				remaining := strings.TrimPrefix(ldconfig, "@/sbin/ldconfig")
+				require.True(t, remaining == ".real" || remaining == "")
+
+				cfg.NVIDIAContainerCLIConfig.Ldconfig = "WAS_CHECKED"
+			}
+
 			require.EqualValues(t, tc.expectedConfig, cfg)
 		})
 	}
 }
+
+func TestConfigDefault(t *testing.T) {
+	config, err := getDefault()
+	require.NoError(t, err)
+
+	buffer := new(bytes.Buffer)
+	_, err = config.Save(buffer)
+	require.NoError(t, err)
+
+	var lines []string
+	for _, l := range strings.Split(buffer.String(), "\n") {
+		l = strings.TrimSpace(l)
+		if strings.HasPrefix(l, "# ") {
+			l = "#" + strings.TrimPrefix(l, "# ")
+		}
+		lines = append(lines, l)
+	}
+
+	// We take the lines from the config that was included in previous packages.
+	expectedLines := []string{
+		"disable-require = false",
+		"#swarm-resource = \"DOCKER_RESOURCE_GPU\"",
+		"#accept-nvidia-visible-devices-envvar-when-unprivileged = true",
+		"#accept-nvidia-visible-devices-as-volume-mounts = false",
+
+		"#root = \"/run/nvidia/driver\"",
+		"#path = \"/usr/bin/nvidia-container-cli\"",
+		"environment = []",
+		"#debug = \"/var/log/nvidia-container-toolkit.log\"",
+		"#ldcache = \"/etc/ld.so.cache\"",
+		"load-kmods = true",
+		"#no-cgroups = false",
+		"#user = \"root:video\"",
+
+		"[nvidia-container-runtime]",
+		"#debug = \"/var/log/nvidia-container-runtime.log\"",
+		"log-level = \"info\"",
+		"mode = \"auto\"",
+
+		"mount-spec-path = \"/etc/nvidia-container-runtime/host-files-for-container.d\"",
+	}
+
+	require.Subset(t, lines, expectedLines)
+}

internal/config/crio/crio.go

@@ -1,125 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package crio
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pelletier/go-toml"
-	log "github.com/sirupsen/logrus"
-)
-
-// LoadConfig loads the cri-o config from disk
-func LoadConfig(config string) (*toml.Tree, error) {
-	log.Infof("Loading config: %v", config)
-
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	configFile := config
-	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		log.Infof("Config file does not exist, creating new one")
-	}
-
-	cfg, err := toml.LoadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-
-	return cfg, nil
-}
-
-// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
-func UpdateConfig(config *toml.Tree, runtimeClass string, runtimePath string, setAsDefault bool) error {
-	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
-	case *toml.Tree:
-		runc, _ = toml.Load(runc.String())
-		config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass}, runc)
-	}
-
-	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_path"}, runtimePath)
-	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_type"}, "oci")
-
-	if setAsDefault {
-		config.SetPath([]string{"crio", "runtime", "default_runtime"}, runtimeClass)
-	}
-
-	return nil
-}
-
-// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
-func RevertConfig(config *toml.Tree, runtimeClass string) error {
-	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
-		if runtimeClass == runtime {
-			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
-		}
-	}
-
-	runtimeClassPath := []string{"crio", "runtime", "runtimes", runtimeClass}
-	config.DeletePath(runtimeClassPath)
-	for i := 0; i < len(runtimeClassPath); i++ {
-		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
-		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
-			if len(entry.Keys()) != 0 {
-				break
-			}
-			config.DeletePath(remainingPath)
-		}
-	}
-
-	return nil
-}
-
-// FlushConfig flushes the updated/reverted config out to disk
-func FlushConfig(config string, cfg *toml.Tree) error {
-	log.Infof("Flushing config")
-
-	output, err := cfg.ToTomlString()
-	if err != nil {
-		return fmt.Errorf("unable to convert to TOML: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(config)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(config)
-		if err != nil {
-			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(output)
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
-}

internal/config/docker/docker.go

@@ -1,117 +0,0 @@
-/**
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package docker
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// LoadConfig loads the docker config from disk
-func LoadConfig(configFilePath string) (map[string]interface{}, error) {
-	log.Infof("Loading docker config from %v", configFilePath)
-
-	info, err := os.Stat(configFilePath)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	cfg := make(map[string]interface{})
-
-	if os.IsNotExist(err) {
-		log.Infof("Config file does not exist, creating new one")
-		return cfg, nil
-	}
-
-	readBytes, err := ioutil.ReadFile(configFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("unable to read config: %v", err)
-	}
-
-	reader := bytes.NewReader(readBytes)
-	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-	return cfg, nil
-}
-
-// UpdateConfig updates the docker config to include the nvidia runtimes
-func UpdateConfig(config map[string]interface{}, runtimeName string, runtimePath string, setAsDefault bool) error {
-	// Read the existing runtimes
-	runtimes := make(map[string]interface{})
-	if _, exists := config["runtimes"]; exists {
-		runtimes = config["runtimes"].(map[string]interface{})
-	}
-
-	// Add / update the runtime definitions
-	runtimes[runtimeName] = map[string]interface{}{
-		"path": runtimePath,
-		"args": []string{},
-	}
-
-	// Update the runtimes definition
-	if len(runtimes) > 0 {
-		config["runtimes"] = runtimes
-	}
-
-	if setAsDefault {
-		config["default-runtime"] = runtimeName
-	}
-
-	return nil
-}
-
-// FlushConfig flushes the updated/reverted config out to disk
-func FlushConfig(cfg map[string]interface{}, configFilePath string) error {
-	log.Infof("Flushing docker config to %v", configFilePath)
-
-	output, err := json.MarshalIndent(cfg, "", "    ")
-	if err != nil {
-		return fmt.Errorf("unable to convert to JSON: %v", err)
-	}
-
-	switch len(output) {
-	case 0:
-		err := os.Remove(configFilePath)
-		if err != nil {
-			return fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		log.Infof("Config empty, removing file")
-	default:
-		f, err := os.Create(configFilePath)
-		if err != nil {
-			return fmt.Errorf("unable to open %v for writing: %v", configFilePath, err)
-		}
-		defer f.Close()
-
-		_, err = f.WriteString(string(output))
-		if err != nil {
-			return fmt.Errorf("unable to write output: %v", err)
-		}
-	}
-
-	log.Infof("Successfully flushed config")
-
-	return nil
-}

internal/config/hook.go

@@ -0,0 +1,36 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeHookConfig struct {
+	// Path specifies the path to the NVIDIA Container Runtime hook binary.
+	// If an executable name is specified, this will be resolved in the path.
+	Path string `toml:"path"`
+	// SkipModeDetection disables the mode check for the runtime hook.
+	SkipModeDetection bool `toml:"skip-mode-detection"`
+}
+
+// GetDefaultRuntimeHookConfig defines the default values for the config
+func GetDefaultRuntimeHookConfig() (*RuntimeHookConfig, error) {
+	cfg, err := getDefault()
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg.NVIDIAContainerRuntimeHookConfig, nil
+}

internal/config/image/builder.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"strings"
+)
+
+type builder struct {
+	env            []string
+	disableRequire bool
+}
+
+// New creates a new CUDA image from the input options.
+func New(opt ...Option) (CUDA, error) {
+	b := &builder{}
+	for _, o := range opt {
+		o(b)
+	}
+
+	return b.build()
+}
+
+// build creates a CUDA image from the builder.
+func (b builder) build() (CUDA, error) {
+	c := make(CUDA)
+
+	for _, e := range b.env {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid environment variable: %v", e)
+		}
+		c[parts[0]] = parts[1]
+	}
+
+	if b.disableRequire {
+		c[envNVDisableRequire] = "true"
+	}
+
+	return c, nil
+}
+
+// Option is a functional option for creating a CUDA image.
+type Option func(*builder)
+
+// WithDisableRequire sets the disable require option.
+func WithDisableRequire(disableRequire bool) Option {
+	return func(b *builder) {
+		b.disableRequire = disableRequire
+	}
+}
+
+// WithEnv sets the environment variables to use when creating the CUDA image.
+func WithEnv(env []string) Option {
+	return func(b *builder) {
+		b.env = env
+	}
+}

internal/config/image/cuda_image.go

@@ -42,27 +42,18 @@ type CUDA map[string]string
 // NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
 // The process environment is read (if present) to construc the CUDA Image.
 func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
-	if spec == nil || spec.Process == nil {
-		return NewCUDAImageFromEnv(nil)
+	var env []string
+	if spec != nil && spec.Process != nil {
+		env = spec.Process.Env
 	}
 
-	return NewCUDAImageFromEnv(spec.Process.Env)
+	return New(WithEnv(env))
 }
 
 // NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
 // is a list of strings of the form ENVAR=VALUE.
 func NewCUDAImageFromEnv(env []string) (CUDA, error) {
-	c := make(CUDA)
-
-	for _, e := range env {
-		parts := strings.SplitN(e, "=", 2)
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("invalid environment variable: %v", e)
-		}
-		c[parts[0]] = parts[1]
-	}
-
-	return c, nil
+	return New(WithEnv(env))
 }
 
 // IsLegacy returns whether the associated CUDA image is a "legacy" image. An
@@ -77,11 +68,9 @@ func (i CUDA) IsLegacy() bool {
 // GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
 // variables.
 func (i CUDA) GetRequirements() ([]string, error) {
-	// TODO: We need not process this if disable require is set, but this will be done
-	// in a single follow-up to ensure that the behavioural change is accurately captured.
-	// if i.HasDisableRequire() {
-	// 	return nil, nil
-	// }
+	if i.HasDisableRequire() {
+		return nil, nil
+	}
 
 	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
 	var requirements []string
@@ -159,9 +148,10 @@ func (i CUDA) GetDriverCapabilities() DriverCapabilities {
 }
 
 func (i CUDA) legacyVersion() (string, error) {
-	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
+	cudaVersion := i[envCUDAVersion]
+	majorMinor, err := parseMajorMinorVersion(cudaVersion)
 	if err != nil {
-		return "", fmt.Errorf("invalid CUDA version: %v", err)
+		return "", fmt.Errorf("invalid CUDA version %v: %v", cudaVersion, err)
 	}
 
 	return majorMinor, nil

internal/config/image/cuda_image_test.go

@@ -106,6 +106,16 @@ func TestGetRequirements(t *testing.T) {
 			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
 			requirements: []string{"cuda>=11.6", "brand=tesla"},
 		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores requirements",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
+		{
+			description:  "NVIDIA_DISABLE_REQUIRE ignores legacy image requirements",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla", "NVIDIA_DISABLE_REQUIRE=true"},
+			requirements: []string{},
+		},
 	}
 
 	for _, tc := range testCases {

internal/config/image/privileged.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged NVIDIA capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

@@ -16,20 +16,6 @@
 
 package config
 
-import (
-	"fmt"
-
-	"github.com/pelletier/go-toml"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	dockerRuncExecutableName = "docker-runc"
-	runcExecutableName       = "runc"
-
-	auto = "auto"
-)
-
 // RuntimeConfig stores the config options for the NVIDIA Container Runtime
 type RuntimeConfig struct {
 	DebugFilePath string `toml:"debug"`
@@ -50,52 +36,22 @@ type modesConfig struct {
 type cdiModeConfig struct {
 	// SpecDirs allows for the default spec dirs for CDI to be overridden
 	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
 }
 
 type csvModeConfig struct {
 	MountSpecPath string `toml:"mount-spec-path"`
 }
 
-// dummy allows us to unmarshal only a RuntimeConfig from a *toml.Tree
-type dummy struct {
-	Runtime RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-// getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getRuntimeConfigFrom(toml *toml.Tree) (*RuntimeConfig, error) {
-	cfg := GetDefaultRuntimeConfig()
-
-	if toml == nil {
-		return cfg, nil
-	}
-
-	d := dummy{
-		Runtime: *cfg,
-	}
-
-	if err := toml.Unmarshal(&d); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
-	}
-
-	return &d.Runtime, nil
-}
-
 // GetDefaultRuntimeConfig defines the default values for the config
-func GetDefaultRuntimeConfig() *RuntimeConfig {
-	c := RuntimeConfig{
-		DebugFilePath: "/dev/null",
-		LogLevel:      logrus.InfoLevel.String(),
-		Runtimes: []string{
-			dockerRuncExecutableName,
-			runcExecutableName,
-		},
-		Mode: auto,
-		Modes: modesConfig{
-			CSV: csvModeConfig{
-				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
-			},
-		},
+func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
+	cfg, err := getDefault()
+	if err != nil {
+		return nil, err
 	}
 
-	return &c
+	return &cfg.NVIDIAContainerRuntimeConfig, nil
 }

internal/config/toolkit-cli.go

@@ -16,31 +16,7 @@
 
 package config
 
-import "github.com/pelletier/go-toml"
-
 // CTKConfig stores the config options for the NVIDIA Container Toolkit CLI (nvidia-ctk)
 type CTKConfig struct {
 	Path string `toml:"path"`
 }
-
-// getCTKConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getCTKConfigFrom(toml *toml.Tree) *CTKConfig {
-	cfg := getDefaultCTKConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Path = toml.GetDefault("nvidia-ctk.path", cfg.Path).(string)
-
-	return cfg
-}
-
-// getDefaultCTKConfig defines the default values for the config
-func getDefaultCTKConfig() *CTKConfig {
-	c := CTKConfig{
-		Path: "nvidia-ctk",
-	}
-
-	return &c
-}

internal/discover/char_devices.go

@@ -17,8 +17,8 @@
 package discover
 
 import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
 )
 
 // charDevices is a discover for a list of character devices
@@ -27,7 +27,7 @@ type charDevices mounts
 var _ Discover = (*charDevices)(nil)
 
 // NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
-func NewCharDeviceDiscoverer(logger *logrus.Logger, devices []string, root string) Discover {
+func NewCharDeviceDiscoverer(logger logger.Interface, devices []string, root string) Discover {
 	locator := lookup.NewCharDeviceLocator(
 		lookup.WithLogger(logger),
 		lookup.WithRoot(root),
@@ -37,7 +37,7 @@ func NewCharDeviceDiscoverer(logger *logrus.Logger, devices []string, root strin
 }
 
 // NewDeviceDiscoverer creates a discoverer which locates the specified set of device nodes using the specified locator.
-func NewDeviceDiscoverer(logger *logrus.Logger, locator lookup.Locator, root string, devices []string) Discover {
+func NewDeviceDiscoverer(logger logger.Interface, locator lookup.Locator, root string, devices []string) Discover {
 	m := NewMounts(logger, locator, root, devices).(*mounts)
 
 	return (*charDevices)(m)
@@ -58,7 +58,11 @@ func (d *charDevices) Devices() ([]Device, error) {
 	}
 	var devices []Device
 	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
 	}
 
 	return devices, nil

internal/discover/csv.go

@@ -20,23 +20,26 @@ import (
 	"fmt"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
 )
 
 // NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
 // The constructed discoverer is comprised of a list, with each element in the list being associated with a
 // single CSV files.
-func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
+func NewFromCSVFiles(logger logger.Interface, files []string, driverRoot string) (Discover, error) {
 	if len(files) == 0 {
-		logger.Warnf("No CSV files specified")
+		logger.Warningf("No CSV files specified")
 		return None{}, nil
 	}
 
-	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
+	symlinkLocator := lookup.NewSymlinkLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(driverRoot),
+	)
 	locators := map[csv.MountSpecType]lookup.Locator{
-		csv.MountSpecDev: lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(root)),
-		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
+		csv.MountSpecDev: lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
+		csv.MountSpecDir: lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
 		// Libraries and symlinks are handled in the same way
 		csv.MountSpecLib: symlinkLocator,
 		csv.MountSpecSym: symlinkLocator,
@@ -46,17 +49,17 @@ func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discov
 	for _, filename := range files {
 		targets, err := loadCSVFile(logger, filename)
 		if err != nil {
-			logger.Warnf("Skipping CSV file %v: %v", filename, err)
+			logger.Warningf("Skipping CSV file %v: %v", filename, err)
 			continue
 		}
 		mountSpecs = append(mountSpecs, targets...)
 	}
 
-	return newFromMountSpecs(logger, locators, root, mountSpecs)
+	return newFromMountSpecs(logger, locators, driverRoot, mountSpecs)
 }
 
 // loadCSVFile loads the specified CSV file and returns the list of mount specs
-func loadCSVFile(logger *logrus.Logger, filename string) ([]*csv.MountSpec, error) {
+func loadCSVFile(logger logger.Interface, filename string) ([]*csv.MountSpec, error) {
 	// Create a discoverer for each file-kind combination
 	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
 	if err != nil {
@@ -71,7 +74,7 @@ func loadCSVFile(logger *logrus.Logger, filename string) ([]*csv.MountSpec, erro
 
 // newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
 // A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
-func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, root string, targets []*csv.MountSpec) (Discover, error) {
+func newFromMountSpecs(logger logger.Interface, locators map[csv.MountSpecType]lookup.Locator, driverRoot string, targets []*csv.MountSpec) (Discover, error) {
 	if len(targets) == 0 {
 		return &None{}, nil
 	}
@@ -95,9 +98,9 @@ func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]loo
 		var m Discover
 		switch t {
 		case csv.MountSpecDev:
-			m = NewDeviceDiscoverer(logger, locator, root, candidatesByType[t])
+			m = NewDeviceDiscoverer(logger, locator, driverRoot, candidatesByType[t])
 		default:
-			m = NewMounts(logger, locator, root, candidatesByType[t])
+			m = NewMounts(logger, locator, driverRoot, candidatesByType[t])
 		}
 		discoverers = append(discoverers, m)
 

internal/discover/csv/csv.go

@@ -25,7 +25,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
 const (
@@ -33,6 +33,22 @@ const (
 	DefaultMountSpecPath = "/etc/nvidia-container-runtime/host-files-for-container.d"
 )
 
+// DefaultFileList returns the list of CSV files that are used by default.
+func DefaultFileList() []string {
+	files := []string{
+		"devices.csv",
+		"drivers.csv",
+		"l4t.csv",
+	}
+
+	var paths []string
+	for _, file := range files {
+		paths = append(paths, filepath.Join(DefaultMountSpecPath, file))
+	}
+
+	return paths
+}
+
 // GetFileList returns the (non-recursive) list of CSV files in the specified
 // folder
 func GetFileList(root string) ([]string, error) {
@@ -87,12 +103,12 @@ type Parser interface {
 }
 
 type csv struct {
-	logger   *logrus.Logger
+	logger   logger.Interface
 	filename string
 }
 
 // NewCSVFileParser creates a new parser for reading MountSpecs from the specified CSV file
-func NewCSVFileParser(logger *logrus.Logger, filename string) Parser {
+func NewCSVFileParser(logger logger.Interface, filename string) Parser {
 	p := csv{
 		logger:   logger,
 		filename: filename,

internal/discover/discover.go

@@ -16,12 +16,6 @@
 
 package discover
 
-// Config represents the configuration options for discovery
-type Config struct {
-	Root          string
-	NvidiaCTKPath string
-}
-
 // Device represents a discovered character device.
 type Device struct {
 	HostPath string
@@ -32,6 +26,7 @@ type Device struct {
 type Mount struct {
 	HostPath string
 	Path     string
+	Options  []string
 }
 
 // Hook represents a discovered hook.

internal/discover/discover_mock.go

@@ -13,25 +13,25 @@ var _ Discover = &DiscoverMock{}
 
 // DiscoverMock is a mock implementation of Discover.
 //
-// 	func TestSomethingThatUsesDiscover(t *testing.T) {
+//	func TestSomethingThatUsesDiscover(t *testing.T) {
 //
-// 		// make and configure a mocked Discover
-// 		mockedDiscover := &DiscoverMock{
-// 			DevicesFunc: func() ([]Device, error) {
-// 				panic("mock out the Devices method")
-// 			},
-// 			HooksFunc: func() ([]Hook, error) {
-// 				panic("mock out the Hooks method")
-// 			},
-// 			MountsFunc: func() ([]Mount, error) {
-// 				panic("mock out the Mounts method")
-// 			},
-// 		}
+//		// make and configure a mocked Discover
+//		mockedDiscover := &DiscoverMock{
+//			DevicesFunc: func() ([]Device, error) {
+//				panic("mock out the Devices method")
+//			},
+//			HooksFunc: func() ([]Hook, error) {
+//				panic("mock out the Hooks method")
+//			},
+//			MountsFunc: func() ([]Mount, error) {
+//				panic("mock out the Mounts method")
+//			},
+//		}
 //
-// 		// use mockedDiscover in code that requires Discover
-// 		// and then make assertions.
+//		// use mockedDiscover in code that requires Discover
+//		// and then make assertions.
 //
-// 	}
+//	}
 type DiscoverMock struct {
 	// DevicesFunc mocks the Devices method.
 	DevicesFunc func() ([]Device, error)
@@ -78,7 +78,8 @@ func (mock *DiscoverMock) Devices() ([]Device, error) {
 
 // DevicesCalls gets all the calls that were made to Devices.
 // Check the length with:
-//     len(mockedDiscover.DevicesCalls())
+//
+//	len(mockedDiscover.DevicesCalls())
 func (mock *DiscoverMock) DevicesCalls() []struct {
 } {
 	var calls []struct {
@@ -108,7 +109,8 @@ func (mock *DiscoverMock) Hooks() ([]Hook, error) {
 
 // HooksCalls gets all the calls that were made to Hooks.
 // Check the length with:
-//     len(mockedDiscover.HooksCalls())
+//
+//	len(mockedDiscover.HooksCalls())
 func (mock *DiscoverMock) HooksCalls() []struct {
 } {
 	var calls []struct {
@@ -138,7 +140,8 @@ func (mock *DiscoverMock) Mounts() ([]Mount, error) {
 
 // MountsCalls gets all the calls that were made to Mounts.
 // Check the length with:
-//     len(mockedDiscover.MountsCalls())
+//
+//	len(mockedDiscover.MountsCalls())
 func (mock *DiscoverMock) MountsCalls() []struct {
 } {
 	var calls []struct {

internal/discover/filter.go

@@ -16,7 +16,7 @@
 
 package discover
 
-import "github.com/sirupsen/logrus"
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 
 // Filter defines an interface for filtering discovered entities
 type Filter interface {
@@ -26,12 +26,12 @@ type Filter interface {
 // filtered represents a filtered discoverer
 type filtered struct {
 	Discover
-	logger *logrus.Logger
+	logger logger.Interface
 	filter Filter
 }
 
 // newFilteredDisoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
-func newFilteredDisoverer(logger *logrus.Logger, applyTo Discover, filter Filter) Discover {
+func newFilteredDisoverer(logger logger.Interface, applyTo Discover, filter Filter) Discover {
 	return filtered{
 		Discover: applyTo,
 		logger:   logger,

internal/discover/gds.go

@@ -17,19 +17,19 @@
 package discover
 
 import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
 )
 
 type gdsDeviceDiscoverer struct {
 	None
-	logger  *logrus.Logger
+	logger  logger.Interface
 	devices Discover
 	mounts  Discover
 }
 
 // NewGDSDiscoverer creates a discoverer for GPUDirect Storage devices and mounts.
-func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+func NewGDSDiscoverer(logger logger.Interface, root string) (Discover, error) {
 	devices := NewCharDeviceDiscoverer(
 		logger,
 		[]string{"/dev/nvidia-fs*"},
@@ -38,7 +38,7 @@ func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
 
 	udev := NewMounts(
 		logger,
-		lookup.NewDirectoryLocator(logger, root),
+		lookup.NewDirectoryLocator(lookup.WithLogger(logger), lookup.WithRoot(root)),
 		root,
 		[]string{"/run/udev"},
 	)

internal/discover/graphics.go

@@ -20,29 +20,29 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
 )
 
 // NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
-func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, cfg *Config) (Discover, error) {
-	root := cfg.Root
-
-	mounts, err := NewGraphicsMountsDiscoverer(logger, root)
+func NewGraphicsDiscoverer(logger logger.Interface, devices image.VisibleDevices, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	mounts, err := NewGraphicsMountsDiscoverer(logger, driverRoot, nvidiaCTKPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
 	}
 
-	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, root)
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
 	}
 
-	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, driverRoot, nvidiaCTKPath)
 
 	discover := Merge(
 		Merge(drmDeviceNodes, drmByPathSymlinks),
@@ -53,15 +53,15 @@ func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices,
 }
 
 // NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
-func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
-	locator, err := lookup.NewLibraryLocator(logger, root)
+func NewGraphicsMountsDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct library locator: %v", err)
 	}
 	libraries := NewMounts(
 		logger,
 		locator,
-		root,
+		driverRoot,
 		[]string{
 			"libnvidia-egl-gbm.so",
 		},
@@ -71,10 +71,10 @@ func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover,
 		logger,
 		lookup.NewFileLocator(
 			lookup.WithLogger(logger),
-			lookup.WithRoot(root),
+			lookup.WithRoot(driverRoot),
 			lookup.WithSearchPaths("/etc", "/usr/share"),
 		),
-		root,
+		driverRoot,
 		[]string{
 			"glvnd/egl_vendor.d/10_nvidia.json",
 			"vulkan/icd.d/nvidia_icd.json",
@@ -84,9 +84,12 @@ func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover,
 		},
 	)
 
+	xorg := optionalXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+
 	discover := Merge(
 		libraries,
 		jsonMounts,
+		xorg,
 	)
 
 	return discover, nil
@@ -94,18 +97,18 @@ func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover,
 
 type drmDevicesByPath struct {
 	None
-	logger        *logrus.Logger
+	logger        logger.Interface
 	nvidiaCTKPath string
-	root          string
+	driverRoot    string
 	devicesFrom   Discover
 }
 
 // newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
-func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover {
+func newCreateDRMByPathSymlinks(logger logger.Interface, devices Discover, driverRoot string, nvidiaCTKPath string) Discover {
 	d := drmDevicesByPath{
 		logger:        logger,
-		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
-		root:          cfg.Root,
+		nvidiaCTKPath: nvidiaCTKPath,
+		driverRoot:    driverRoot,
 		devicesFrom:   devices,
 	}
 
@@ -152,7 +155,7 @@ func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error
 
 	linkLocator := lookup.NewFileLocator(
 		lookup.WithLogger(d.logger),
-		lookup.WithRoot(d.root),
+		lookup.WithRoot(d.driverRoot),
 	)
 	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
 	if err != nil {
@@ -178,21 +181,21 @@ func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error
 }
 
 // newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
-func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, root string) (Discover, error) {
+func newDRMDeviceDiscoverer(logger logger.Interface, devices image.VisibleDevices, driverRoot string) (Discover, error) {
 	allDevices := NewDeviceDiscoverer(
 		logger,
 		lookup.NewCharDeviceLocator(
 			lookup.WithLogger(logger),
-			lookup.WithRoot(root),
+			lookup.WithRoot(driverRoot),
 		),
-		root,
+		driverRoot,
 		[]string{
 			"/dev/dri/card*",
 			"/dev/dri/renderD*",
 		},
 	)
 
-	filter, err := newDRMDeviceFilter(logger, devices, root)
+	filter, err := newDRMDeviceFilter(logger, devices, driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
 	}
@@ -208,8 +211,8 @@ func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices,
 }
 
 // newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
-func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, root string) (Filter, error) {
-	gpuInformationPaths, err := proc.GetInformationFilePaths(root)
+func newDRMDeviceFilter(logger logger.Interface, devices image.VisibleDevices, driverRoot string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(driverRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read GPU information: %v", err)
 	}
@@ -243,6 +246,123 @@ func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, roo
 	return filter, nil
 }
 
+type xorgHooks struct {
+	libraries     Discover
+	driverVersion string
+	nvidiaCTKPath string
+}
+
+var _ Discover = (*xorgHooks)(nil)
+
+// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
+// If the creation of the discoverer fails, a None discoverer is returned.
+func optionalXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		logger.Warningf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
+		return None{}
+	}
+	return xorg
+}
+
+func newXorgDiscoverer(logger logger.Interface, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate(".*.*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+	libcudaPath := libCudaPaths[0]
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	libRoot := filepath.Dir(libcudaPath)
+	xorgLibs := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"nvidia/xorg/nvidia_drv.so",
+			fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
+		},
+	)
+	xorgHooks := xorgHooks{
+		libraries:     xorgLibs,
+		driverVersion: version,
+		nvidiaCTKPath: nvidiaCTKPath,
+	}
+
+	xorgConfg := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/usr/share"),
+		),
+		driverRoot,
+		[]string{"X11/xorg.conf.d/10-nvidia.conf"},
+	)
+
+	d := Merge(
+		xorgLibs,
+		xorgConfg,
+		xorgHooks,
+	)
+
+	return d, nil
+}
+
+// Devices returns no devices for Xorg
+func (m xorgHooks) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a hook to create symlinks for Xorg libraries
+func (m xorgHooks) Hooks() ([]Hook, error) {
+	mounts, err := m.libraries.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get mounts: %v", err)
+	}
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+
+	var target string
+	for _, mount := range mounts {
+		filename := filepath.Base(mount.HostPath)
+		if filename == "libglxserver_nvidia.so."+m.driverVersion {
+			target = mount.Path
+		}
+	}
+
+	if target == "" {
+		return nil, nil
+	}
+
+	link := strings.TrimSuffix(target, "."+m.driverVersion)
+	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
+	symlinkHook := CreateCreateSymlinkHook(
+		m.nvidiaCTKPath,
+		links,
+	)
+
+	return symlinkHook.Hooks()
+}
+
+// Mounts returns the libraries required for Xorg
+func (m xorgHooks) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
 // selectDeviceByPath is a filter that allows devices to be selected by the path
 type selectDeviceByPath map[string]bool
 

internal/discover/hooks.go

@@ -19,45 +19,49 @@ package discover
 import (
 	"path/filepath"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	"github.com/sirupsen/logrus"
 )
 
-const (
-	nvidiaCTKExecutable      = "nvidia-ctk"
-	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
-)
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+// CreateCreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateCreateSymlinkHook(nvidiaCTKPath string, links []string) Discover {
+	if len(links) == 0 {
+		return None{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateNvidiaCTKHook(
+		nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+}
 
 // CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
-func CreateNvidiaCTKHook(executable string, hookName string, additionalArgs ...string) Hook {
+func CreateNvidiaCTKHook(nvidiaCTKPath string, hookName string, additionalArgs ...string) Hook {
 	return Hook{
 		Lifecycle: cdi.CreateContainerHook,
-		Path:      executable,
-		Args:      append([]string{filepath.Base(executable), "hook", hookName}, additionalArgs...),
+		Path:      nvidiaCTKPath,
+		Args:      append([]string{filepath.Base(nvidiaCTKPath), "hook", hookName}, additionalArgs...),
 	}
 }
-
-// FindNvidiaCTK locates the nvidia-ctk executable to be used in hooks.
-// If an override is specified, this is used instead.
-func FindNvidiaCTK(logger *logrus.Logger, override string) string {
-	if override != "" {
-		logger.Debugf("Using specified NVIDIA Container Toolkit CLI path %v", override)
-		return override
-	}
-
-	lookup := lookup.NewExecutableLocator(logger, "")
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := lookup.Locate(nvidiaCTKExecutable)
-	if err != nil {
-		logger.Warnf("Failed to locate %v: %v", nvidiaCTKExecutable, err)
-	} else if len(targets) == 0 {
-		logger.Warnf("%v not found", nvidiaCTKExecutable)
-	} else {
-		logger.Debugf("Found %v candidates: %v", nvidiaCTKExecutable, targets)
-		hookPath = targets[0]
-	}
-	logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	return hookPath
-}

internal/discover/icp_test.go

@@ -0,0 +1,61 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIPCMounts(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	l := ipcMounts(
+		mounts{
+			logger: logger,
+			lookup: &lookup.LocatorMock{
+				LocateFunc: func(path string) ([]string, error) {
+					return []string{"/host/path"}, nil
+				},
+			},
+			required: []string{"target"},
+		},
+	)
+
+	mounts, err := l.Mounts()
+	require.NoError(t, err)
+
+	require.EqualValues(
+		t,
+		[]Mount{
+			{
+				HostPath: "/host/path",
+				Path:     "/host/path",
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+					"noexec",
+				},
+			},
+		},
+		mounts,
+	)
+}

internal/discover/ipc.go

@@ -0,0 +1,78 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type ipcMounts mounts
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger logger.Interface, driverRoot string) (Discover, error) {
+	sockets := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/run", "/var/run"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/nvidia-persistenced/socket",
+			"/nvidia-fabricmanager/socket",
+		},
+	)
+
+	mps := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	d := Merge(
+		(*ipcMounts)(sockets),
+		(*ipcMounts)(mps),
+	)
+	return d, nil
+}
+
+// Mounts returns the discovered mounts with "noexec" added to the mount options.
+func (d *ipcMounts) Mounts() ([]Mount, error) {
+	mounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+
+	var modifiedMounts []Mount
+	for _, m := range mounts {
+		mount := m
+		mount.Options = append(m.Options, "noexec")
+		modifiedMounts = append(modifiedMounts, mount)
+	}
+
+	return modifiedMounts, nil
+}

internal/discover/ldconfig.go

@@ -21,17 +21,15 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
 // NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
-func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
+func NewLDCacheUpdateHook(logger logger.Interface, mounts Discover, nvidiaCTKPath string) (Discover, error) {
 	d := ldconfig{
 		logger:        logger,
+		nvidiaCTKPath: nvidiaCTKPath,
 		mountsFrom:    mounts,
-		lookup:        lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKPath: cfg.NvidiaCTKPath,
 	}
 
 	return &d, nil
@@ -39,10 +37,9 @@ func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (
 
 type ldconfig struct {
 	None
-	logger        *logrus.Logger
-	mountsFrom    Discover
-	lookup        lookup.Locator
+	logger        logger.Interface
 	nvidiaCTKPath string
+	mountsFrom    Discover
 }
 
 // Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.

internal/discover/ldconfig_test.go

@@ -31,11 +31,6 @@ const (
 func TestLDCacheUpdateHook(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
-	cfg := Config{
-		Root:          "/",
-		NvidiaCTKPath: testNvidiaCTKPath,
-	}
-
 	testCases := []struct {
 		description   string
 		mounts        []Mount
@@ -95,7 +90,7 @@ func TestLDCacheUpdateHook(t *testing.T) {
 				Lifecycle: "createContainer",
 			}
 
-			d, err := NewLDCacheUpdateHook(logger, mountMock, &cfg)
+			d, err := NewLDCacheUpdateHook(logger, mountMock, testNvidiaCTKPath)
 			require.NoError(t, err)
 
 			hooks, err := d.Hooks()

internal/discover/mofed.go

@@ -16,12 +16,10 @@
 
 package discover
 
-import (
-	"github.com/sirupsen/logrus"
-)
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 
 // NewMOFEDDiscoverer creates a discoverer for MOFED devices.
-func NewMOFEDDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+func NewMOFEDDiscoverer(logger logger.Interface, root string) (Discover, error) {
 	devices := NewCharDeviceDiscoverer(
 		logger,
 		[]string{

internal/discover/mounts.go

@@ -22,8 +22,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
 )
 
 // mounts is a generic discoverer for Mounts. It is customized by specifying the
@@ -31,7 +31,7 @@ import (
 // based on the entry in the list.
 type mounts struct {
 	None
-	logger   *logrus.Logger
+	logger   logger.Interface
 	lookup   lookup.Locator
 	root     string
 	required []string
@@ -42,7 +42,12 @@ type mounts struct {
 var _ Discover = (*mounts)(nil)
 
 // NewMounts creates a discoverer for the required mounts using the specified locator.
-func NewMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) Discover {
+func NewMounts(logger logger.Interface, lookup lookup.Locator, root string, required []string) Discover {
+	return newMounts(logger, lookup, root, required)
+}
+
+// newMounts creates a discoverer for the required mounts using the specified locator.
+func newMounts(logger logger.Interface, lookup lookup.Locator, root string, required []string) *mounts {
 	return &mounts{
 		logger:   logger,
 		lookup:   lookup,
@@ -70,11 +75,11 @@ func (d *mounts) Mounts() ([]Mount, error) {
 		d.logger.Debugf("Locating %v", candidate)
 		located, err := d.lookup.Locate(candidate)
 		if err != nil {
-			d.logger.Warnf("Could not locate %v: %v", candidate, err)
+			d.logger.Warningf("Could not locate %v: %v", candidate, err)
 			continue
 		}
 		if len(located) == 0 {
-			d.logger.Warnf("Missing %v", candidate)
+			d.logger.Warningf("Missing %v", candidate)
 			continue
 		}
 		d.logger.Debugf("Located %v as %v", candidate, located)
@@ -93,6 +98,12 @@ func (d *mounts) Mounts() ([]Mount, error) {
 			uniqueMounts[p] = Mount{
 				HostPath: p,
 				Path:     r,
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+				},
 			}
 		}
 	}

internal/discover/mounts_test.go

@@ -35,6 +35,14 @@ func TestMountsReturnsEmptyDevices(t *testing.T) {
 }
 
 func TestMounts(t *testing.T) {
+
+	mountOptions := []string{
+		"ro",
+		"nosuid",
+		"nodev",
+		"bind",
+	}
+
 	logger, logHook := testlog.NewNullLogger()
 
 	testCases := []struct {
@@ -70,7 +78,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required"},
 			},
-			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description:   "mounts removes located duplicates",
@@ -83,7 +91,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "located", HostPath: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips located errors",
@@ -98,7 +106,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "error", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips unlocated",
@@ -113,7 +121,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "empty", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0", HostPath: "required0"}, {Path: "required1", HostPath: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts adds multiple",
@@ -129,10 +137,10 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "required0", HostPath: "required0"},
-				{Path: "multiple0", HostPath: "multiple0"},
-				{Path: "multiple1", HostPath: "multiple1"},
-				{Path: "required1", HostPath: "required1"},
+				{Path: "required0", HostPath: "required0", Options: mountOptions},
+				{Path: "multiple0", HostPath: "multiple0", Options: mountOptions},
+				{Path: "multiple1", HostPath: "multiple1", Options: mountOptions},
+				{Path: "required1", HostPath: "required1", Options: mountOptions},
 			},
 		},
 		{
@@ -147,7 +155,7 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "/located", HostPath: "/some/root/located"},
+				{Path: "/located", HostPath: "/some/root/located", Options: mountOptions},
 			},
 		},
 	}

internal/discover/symlinks.go

@@ -21,22 +21,26 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
 )
 
-type symlinks struct {
+type symlinkHook struct {
 	None
-	logger        *logrus.Logger
+	logger        logger.Interface
+	driverRoot    string
 	nvidiaCTKPath string
 	csvFiles      []string
 	mountsFrom    Discover
 }
 
 // NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container
-func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) {
-	d := symlinks{
+func NewCreateSymlinksHook(logger logger.Interface, csvFiles []string, mounts Discover, nvidiaCTKPath string) (Discover, error) {
+	d := symlinkHook{
 		logger:        logger,
-		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		nvidiaCTKPath: nvidiaCTKPath,
 		csvFiles:      csvFiles,
 		mountsFrom:    mounts,
 	}
@@ -45,17 +49,17 @@ func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Disc
 }
 
 // Hooks returns a hook to create the symlinks from the required CSV files
-func (d symlinks) Hooks() ([]Hook, error) {
-	var args []string
-	for _, f := range d.csvFiles {
-		args = append(args, "--csv-filename", f)
-	}
-
-	links, err := d.getSpecificLinkArgs()
+func (d symlinkHook) Hooks() ([]Hook, error) {
+	specificLinks, err := d.getSpecificLinks()
 	if err != nil {
 		return nil, fmt.Errorf("failed to determine specific links: %v", err)
 	}
-	args = append(args, links...)
+
+	csvSymlinks := d.getCSVFileSymlinks()
+	var args []string
+	for _, link := range append(csvSymlinks, specificLinks...) {
+		args = append(args, "--link", link)
+	}
 
 	hook := CreateNvidiaCTKHook(
 		d.nvidiaCTKPath,
@@ -66,8 +70,8 @@ func (d symlinks) Hooks() ([]Hook, error) {
 	return []Hook{hook}, nil
 }
 
-// getSpecificLinkArgs returns the required specic links that need to be created
-func (d symlinks) getSpecificLinkArgs() ([]string, error) {
+// getSpecificLinks returns the required specic links that need to be created
+func (d symlinkHook) getSpecificLinks() ([]string, error) {
 	mounts, err := d.mountsFrom.Mounts()
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
@@ -99,11 +103,63 @@ func (d symlinks) getSpecificLinkArgs() ([]string, error) {
 		if linkProcessed[link] {
 			continue
 		}
+		linkProcessed[link] = true
 
 		linkPath := filepath.Join(filepath.Dir(m.Path), link)
-		links = append(links, "--link", fmt.Sprintf("%v::%v", target, linkPath))
-		linkProcessed[link] = true
+		links = append(links, fmt.Sprintf("%v::%v", target, linkPath))
 	}
 
 	return links, nil
 }
+
+func (d symlinkHook) getCSVFileSymlinks() []string {
+	chainLocator := lookup.NewSymlinkChainLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.driverRoot),
+	)
+
+	var candidates []string
+	for _, file := range d.csvFiles {
+		mountSpecs, err := csv.NewCSVFileParser(d.logger, file).Parse()
+		if err != nil {
+			d.logger.Debugf("Skipping CSV file %v: %v", file, err)
+			continue
+		}
+
+		for _, ms := range mountSpecs {
+			if ms.Type != csv.MountSpecSym {
+				continue
+			}
+			targets, err := chainLocator.Locate(ms.Path)
+			if err != nil {
+				d.logger.Warningf("Failed to locate symlink %v", ms.Path)
+			}
+			candidates = append(candidates, targets...)
+		}
+	}
+
+	var links []string
+	created := make(map[string]bool)
+	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
+	for _, candidate := range candidates {
+		target, err := symlinks.Resolve(candidate)
+		if err != nil {
+			d.logger.Debugf("Skipping invalid link: %v", err)
+			continue
+		} else if target == candidate {
+			d.logger.Debugf("%v is not a symlink", candidate)
+			continue
+		}
+
+		link := fmt.Sprintf("%v::%v", target, candidate)
+		if created[link] {
+			d.logger.Debugf("skipping duplicate link: %v", link)
+			continue
+		}
+		created[link] = true
+
+		links = append(links, link)
+	}
+
+	return links
+}

internal/discover/tegra/tegra.go

@@ -0,0 +1,106 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package tegra
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type tegraOptions struct {
+	logger        logger.Interface
+	csvFiles      []string
+	driverRoot    string
+	nvidiaCTKPath string
+}
+
+// Option defines a functional option for configuring a Tegra discoverer.
+type Option func(*tegraOptions)
+
+// New creates a new tegra discoverer using the supplied options.
+func New(opts ...Option) (discover.Discover, error) {
+	o := &tegraOptions{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	csvDiscoverer, err := discover.NewFromCSVFiles(o.logger, o.csvFiles, o.driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
+	}
+
+	createSymlinksHook, err := discover.NewCreateSymlinksHook(o.logger, o.csvFiles, csvDiscoverer, o.nvidiaCTKPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
+	}
+
+	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(o.logger, csvDiscoverer, o.nvidiaCTKPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
+	}
+
+	tegraSystemMounts := discover.NewMounts(
+		o.logger,
+		lookup.NewFileLocator(lookup.WithLogger(o.logger)),
+		"",
+		[]string{
+			"/etc/nv_tegra_release",
+			"/sys/devices/soc0/family",
+		},
+	)
+
+	d := discover.Merge(
+		csvDiscoverer,
+		createSymlinksHook,
+		// The ldcacheUpdateHook is added last to ensure that the created symlinks are included
+		ldcacheUpdateHook,
+		tegraSystemMounts,
+	)
+
+	return d, nil
+}
+
+// WithLogger sets the logger for the discoverer.
+func WithLogger(logger logger.Interface) Option {
+	return func(o *tegraOptions) {
+		o.logger = logger
+	}
+}
+
+// WithDriverRoot sets the driver root for the discoverer.
+func WithDriverRoot(driverRoot string) Option {
+	return func(o *tegraOptions) {
+		o.driverRoot = driverRoot
+	}
+}
+
+// WithCSVFiles sets the CSV files for the discoverer.
+func WithCSVFiles(csvFiles []string) Option {
+	return func(o *tegraOptions) {
+		o.csvFiles = csvFiles
+	}
+}
+
+// WithNVIDIACTKPath sets the path to the nvidia-container-toolkit binary.
+func WithNVIDIACTKPath(nvidiaCTKPath string) Option {
+	return func(o *tegraOptions) {
+		o.nvidiaCTKPath = nvidiaCTKPath
+	}
+}

internal/dxcore/api.go

@@ -0,0 +1,64 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+import (
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+const (
+	libraryName      = "libdxcore.so"
+	libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+// dxcore stores a reference the dxcore dynamic library
+var dxcore *context
+
+// Init initializes the dxcore dynamic library
+func Init() error {
+	c, err := initContext()
+	if err != nil {
+		return err
+	}
+	dxcore = c
+	return nil
+}
+
+// Shutdown closes the dxcore dynamic library
+func Shutdown() error {
+	if dxcore != nil && dxcore.initialized != 0 {
+		dxcore.deinitContext()
+	}
+	return nil
+}
+
+// GetDriverStorePaths returns the list of driver store paths
+func GetDriverStorePaths() []string {
+	var paths []string
+	selected := make(map[string]bool)
+
+	for i := 0; i < dxcore.getAdapterCount(); i++ {
+		path := dxcore.getAdapter(i).getDriverStorePath()
+		if selected[path] {
+			continue
+		}
+		selected[path] = true
+		paths = append(paths, path)
+	}
+
+	return paths
+}

internal/dxcore/dxcore.c

@@ -0,0 +1,334 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include "dxcore.h"
+
+// We define log_write as an empty macro to allow dxcore to remain unchanged.
+#define log_write(...)
+
+// We define the following macros to allow dxcore to remain largely unchanged.
+#define log_info(msg) log_write('I', __FILE__, __LINE__, msg)
+#define log_warn(msg) log_write('W', __FILE__, __LINE__, msg)
+#define log_err(msg)  log_write('E', __FILE__, __LINE__, msg)
+#define log_infof(fmt, ...) log_write('I', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_warnf(fmt, ...) log_write('W', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_errf(fmt, ...)  log_write('E', __FILE__, __LINE__, fmt, __VA_ARGS__)
+
+
+#define DXCORE_MAX_PATH 260
+
+/*
+ * List of components we expect to find in the driver store that we need to mount
+ */
+static const char * const dxcore_nvidia_driver_store_components[] = {
+        "libcuda.so.1.1",                   /* Core library for cuda support */
+        "libcuda_loader.so",                /* Core library for cuda support on WSL */
+        "libnvidia-ptxjitcompiler.so.1",    /* Core library for PTX Jit support */
+        "libnvidia-ml.so.1",                /* Core library for nvml */
+        "libnvidia-ml_loader.so",           /* Core library for nvml on WSL */
+        "nvidia-smi",                       /* nvidia-smi binary*/
+        "nvcubins.bin",                     /* Binary containing GPU code for cuda */
+};
+
+
+/*
+ * List of functions and structures we need to communicate with libdxcore.
+ * Documentation on these functions can be found on docs.microsoft.com in d3dkmthk.
+ */
+
+struct dxcore_enumAdapters2;
+struct dxcore_queryAdapterInfo;
+
+typedef int(*pfnDxcoreEnumAdapters2)(struct dxcore_enumAdapters2* pParams);
+typedef int(*pfnDxcoreQueryAdapterInfo)(struct dxcore_queryAdapterInfo* pParams);
+
+struct dxcore_lib {
+        void* hDxcoreLib;
+        pfnDxcoreEnumAdapters2 pDxcoreEnumAdapters2;
+        pfnDxcoreQueryAdapterInfo pDxcoreQueryAdapterInfo;
+};
+
+struct dxcore_adapterInfo
+{
+        unsigned int              hAdapter;
+        struct dxcore_luid        AdapterLuid;
+        unsigned int              NumOfSources;
+        unsigned int              bPresentMoveRegionsPreferred;
+};
+
+struct dxcore_enumAdapters2
+{
+        unsigned int                   NumAdapters;
+        struct dxcore_adapterInfo     *pAdapters;
+};
+
+enum dxcore_kmtqueryAdapterInfoType
+{
+        DXCORE_QUERYDRIVERVERSION = 13,
+        DXCORE_QUERYREGISTRY = 48,
+};
+
+enum dxcore_queryregistry_type {
+        DXCORE_QUERYREGISTRY_DRIVERSTOREPATH = 2,
+        DXCORE_QUERYREGISTRY_DRIVERIMAGEPATH = 3,
+};
+
+enum dxcore_queryregistry_status {
+        DXCORE_QUERYREGISTRY_STATUS_SUCCESS = 0,
+        DXCORE_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW = 1,
+        DXCORE_QUERYREGISTRY_STATUS_FAIL = 2,
+};
+
+struct dxcore_queryregistry_info {
+        enum dxcore_queryregistry_type        QueryType;
+        unsigned int                          QueryFlags;
+        wchar_t                               ValueName[DXCORE_MAX_PATH];
+        unsigned int                          ValueType;
+        unsigned int                          PhysicalAdapterIndex;
+        unsigned int                          OutputValueSize;
+        enum dxcore_queryregistry_status      Status;
+        union {
+                unsigned long long                    OutputQword;
+                wchar_t                               Output;
+        };
+};
+
+struct dxcore_queryAdapterInfo
+{
+        unsigned int                           hAdapter;
+        enum dxcore_kmtqueryAdapterInfoType    Type;
+        void                                   *pPrivateDriverData;
+        unsigned int                           PrivateDriverDataSize;
+};
+
+static int dxcore_query_adapter_info_helper(struct dxcore_lib* pLib,
+                                            unsigned int hAdapter,
+                                            enum dxcore_kmtqueryAdapterInfoType type,
+                                            void* pPrivateDriverDate,
+                                            unsigned int privateDriverDataSize)
+{
+        struct dxcore_queryAdapterInfo queryAdapterInfo = { 0 };
+
+        queryAdapterInfo.hAdapter = hAdapter;
+        queryAdapterInfo.Type = type;
+        queryAdapterInfo.pPrivateDriverData = pPrivateDriverDate;
+        queryAdapterInfo.PrivateDriverDataSize = privateDriverDataSize;
+
+        return pLib->pDxcoreQueryAdapterInfo(&queryAdapterInfo);
+}
+
+static int dxcore_query_adapter_wddm_version(struct dxcore_lib* pLib, unsigned int hAdapter, unsigned int* version)
+{
+        return dxcore_query_adapter_info_helper(pLib,
+                                                hAdapter,
+                                                DXCORE_QUERYDRIVERVERSION,
+                                                (void*)version,
+                                                sizeof(*version));
+}
+
+static int dxcore_query_adapter_driverstore(struct dxcore_lib* pLib, unsigned int hAdapter, char** ppDriverStorePath)
+{
+        struct dxcore_queryregistry_info params = {0};
+        struct dxcore_queryregistry_info* pValue = NULL;
+        wchar_t* pOutput;
+        size_t outputSizeInBytes;
+        size_t outputSize;
+
+        params.QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                             hAdapter,
+                                             DXCORE_QUERYREGISTRY,
+                                             (void*)&params,
+                                             sizeof(params)))
+        {
+                log_err("Failed to query driver store path size for the WDDM Adapter");
+                return (-1);
+        }
+
+        if (params.OutputValueSize > DXCORE_MAX_PATH * sizeof(wchar_t)) {
+                log_err("The driver store path size returned by dxcore is not valid");
+                return (-1);
+        }
+
+        outputSizeInBytes = (size_t)params.OutputValueSize;
+        outputSize = outputSizeInBytes / sizeof(wchar_t);
+
+        pValue = calloc(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes + sizeof(wchar_t), 1);
+        if (!pValue) {
+                log_err("Out of memory while allocating temp buffer to query adapter info");
+                return (-1);
+        }
+
+        pValue->QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+        pValue->OutputValueSize = (unsigned int)outputSizeInBytes;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                            hAdapter,
+                                            DXCORE_QUERYREGISTRY,
+                                            (void*)pValue,
+                                            (unsigned int)(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes)))
+        {
+                log_err("Failed to query driver store path data for the WDDM Adapter");
+                free(pValue);
+                return (-1);
+        }
+        pOutput = (wchar_t*)(&pValue->Output);
+
+        // Make sure no matter what happened the wchar_t string is null terminated
+        pOutput[outputSize] = L'\0';
+
+        // Convert the output into a regular c string
+        *ppDriverStorePath = (char*)calloc(outputSize + 1, sizeof(char));
+        if (!*ppDriverStorePath) {
+                log_err("Out of memory while allocating the buffer for the driver store path");
+                free(pValue);
+                return (-1);
+        }
+        wcstombs(*ppDriverStorePath, pOutput, outputSize);
+
+        free(pValue);
+
+    return 0;
+}
+
+static void dxcore_add_adapter(struct dxcore_context* pCtx, struct dxcore_lib* pLib, struct dxcore_adapterInfo *pAdapterInfo)
+{
+        unsigned int wddmVersion = 0;
+        char* driverStorePath = NULL;
+
+        log_infof("Creating a new WDDM Adapter for hAdapter:%x luid:%llx", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid));
+
+        if (dxcore_query_adapter_wddm_version(pLib, pAdapterInfo->hAdapter, &wddmVersion)) {
+                log_err("Failed to query the WDDM version for the specified adapter. Skipping it.");
+                return;
+        }
+
+        if (wddmVersion < 2700) {
+                log_err("Found a WDDM adapter running a driver with pre-WDDM 2.7 . Skipping it.");
+                return;
+        }
+
+        if (dxcore_query_adapter_driverstore(pLib, pAdapterInfo->hAdapter, &driverStorePath)) {
+                log_err("Failed to query driver store path for the WDDM Adapter . Skipping it.");
+                return;
+        }
+
+        // We got all the info we needed. Adding it to the tracking structure.
+        {
+                struct dxcore_adapter* newList;
+                newList = realloc(pCtx->adapterList, sizeof(struct dxcore_adapter) * (pCtx->adapterCount + 1));
+                if (!newList) {
+                        log_err("Out of memory when trying to add a new WDDM Adapter to the list of valid adapters");
+                        free(driverStorePath);
+                        return;
+                }
+
+                pCtx->adapterList = newList;
+
+                pCtx->adapterList[pCtx->adapterCount].hAdapter = pAdapterInfo->hAdapter;
+                pCtx->adapterList[pCtx->adapterCount].pDriverStorePath = driverStorePath;
+                pCtx->adapterList[pCtx->adapterCount].wddmVersion = wddmVersion;
+                pCtx->adapterCount++;
+        }
+
+        log_infof("Adding new adapter via dxcore hAdapter:%x luid:%llx wddm version:%d", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid), wddmVersion);
+}
+
+static void dxcore_enum_adapters(struct dxcore_context* pCtx, struct dxcore_lib* pLib)
+{
+        struct dxcore_enumAdapters2 params = {0};
+        unsigned int adapterIndex = 0;
+
+        params.NumAdapters = 0;
+        params.pAdapters = NULL;
+
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        params.pAdapters = malloc(sizeof(struct dxcore_adapterInfo) * params.NumAdapters);
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                free(params.pAdapters);
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        for (adapterIndex = 0; adapterIndex < params.NumAdapters; adapterIndex++) {
+                dxcore_add_adapter(pCtx, pLib, &params.pAdapters[adapterIndex]);
+        }
+
+        free(params.pAdapters);
+}
+
+int dxcore_init_context(struct dxcore_context* pCtx)
+{
+        struct dxcore_lib lib = {0};
+
+        pCtx->initialized = 0;
+        pCtx->adapterCount = 0;
+        pCtx->adapterList = NULL;
+
+        lib.hDxcoreLib = dlopen("libdxcore.so", RTLD_LAZY);
+        if (!lib.hDxcoreLib) {
+                goto error;
+        }
+
+        lib.pDxcoreEnumAdapters2 = (pfnDxcoreEnumAdapters2)dlsym(lib.hDxcoreLib, "D3DKMTEnumAdapters2");
+        if (!lib.pDxcoreEnumAdapters2) {
+                log_err("dxcore library is present but the symbol D3DKMTEnumAdapters2 is missing");
+                goto error;
+        }
+
+        lib.pDxcoreQueryAdapterInfo = (pfnDxcoreQueryAdapterInfo)dlsym(lib.hDxcoreLib, "D3DKMTQueryAdapterInfo");
+        if (!lib.pDxcoreQueryAdapterInfo) {
+                log_err("dxcore library is present but the symbol D3DKMTQueryAdapterInfo is missing");
+                goto error;
+        }
+
+        dxcore_enum_adapters(pCtx, &lib);
+
+        log_info("dxcore layer initialized successfully");
+        pCtx->initialized = 1;
+
+        dlclose(lib.hDxcoreLib);
+
+        return 0;
+
+error:
+        dxcore_deinit_context(pCtx);
+
+        if (lib.hDxcoreLib)
+                dlclose(lib.hDxcoreLib);
+
+        return (-1);
+}
+
+static void dxcore_deinit_adapter(struct dxcore_adapter* pAdapter)
+{
+        if (!pAdapter)
+            return;
+
+        free(pAdapter->pDriverStorePath);
+}
+
+void dxcore_deinit_context(struct dxcore_context* pCtx)
+{
+        unsigned int adapterIndex = 0;
+
+        if (!pCtx)
+                return;
+
+        for (adapterIndex = 0; adapterIndex < pCtx->adapterCount; adapterIndex++) {
+                dxcore_deinit_adapter(&pCtx->adapterList[adapterIndex]);
+        }
+
+        free(pCtx->adapterList);
+
+        pCtx->initialized = 0;
+}

internal/dxcore/dxcore.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+/*
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
+#include <dxcore.h>
+*/
+import "C"
+import (
+	"fmt"
+	"unsafe"
+)
+
+type context C.struct_dxcore_context
+type adapter C.struct_dxcore_adapter
+
+// initContext initializes the dxcore context and populates the list of adapters.
+func initContext() (*context, error) {
+	cContext := C.struct_dxcore_context{}
+	if C.dxcore_init_context(&cContext) != 0 {
+		return nil, fmt.Errorf("failed to initialize dxcore context")
+	}
+	c := (*context)(&cContext)
+	return c, nil
+}
+
+// deinitContext deinitializes the dxcore context and frees the list of adapters.
+func (c context) deinitContext() {
+	cContext := C.struct_dxcore_context(c)
+	C.dxcore_deinit_context(&cContext)
+}
+
+func (c context) getAdapterCount() int {
+	return int(c.adapterCount)
+}
+
+func (c context) getAdapter(index int) adapter {
+	arrayPointer := (*[1 << 30]C.struct_dxcore_adapter)(unsafe.Pointer(c.adapterList))
+	return adapter(arrayPointer[index])
+}
+
+func (a adapter) getDriverStorePath() string {
+	return C.GoString(a.pDriverStorePath)
+}

internal/dxcore/dxcore.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#ifndef HEADER_DXCORE_H_
+#define HEADER_DXCORE_H_
+
+#define MAX_DXCORE_DRIVERSTORE_LIBRAIRIES (16)
+
+struct dxcore_luid
+{
+        unsigned int lowPart;
+        int highPart;
+};
+
+struct dxcore_adapter
+{
+        unsigned int             hAdapter;
+        unsigned int             wddmVersion;
+        char*                    pDriverStorePath;
+        unsigned int             driverStoreComponentCount;
+        const char*              pDriverStoreComponents[MAX_DXCORE_DRIVERSTORE_LIBRAIRIES];
+        struct dxcore_context    *pContext;
+};
+
+struct dxcore_context
+{
+        unsigned int adapterCount;
+        struct dxcore_adapter *adapterList;
+
+        int initialized;
+};
+
+
+
+int dxcore_init_context(struct dxcore_context* pDxcore_context);
+void dxcore_deinit_context(struct dxcore_context* pDxcore_context);
+
+#endif // HEADER_DXCORE_H_

internal/edits/edits.go

@@ -20,21 +20,21 @@ import (
 	"fmt"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
 	ociSpecs "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
 )
 
 type edits struct {
 	cdi.ContainerEdits
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewSpecEdits creates a SpecModifier that defines the required OCI spec edits (as CDI ContainerEdits) from the specified
 // discoverer.
-func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+func NewSpecEdits(logger logger.Interface, d discover.Discover) (oci.SpecModifier, error) {
 	c, err := FromDiscoverer(d)
 	if err != nil {
 		return nil, fmt.Errorf("error constructing container edits: %v", err)
@@ -108,7 +108,7 @@ func (e *edits) Modify(spec *ociSpecs.Spec) error {
 	}
 	e.logger.Infof("Hooks:")
 	for _, hook := range e.Hooks {
-		e.logger.Infof("Injecting %v", hook.Args)
+		e.logger.Infof("Injecting %v %v", hook.Path, hook.Args)
 	}
 
 	return e.Apply(spec)

internal/edits/mount.go

@@ -40,12 +40,7 @@ func (d mount) toSpec() *specs.Mount {
 	s := specs.Mount{
 		HostPath:      d.HostPath,
 		ContainerPath: d.Path,
-		Options: []string{
-			"ro",
-			"nosuid",
-			"nodev",
-			"bind",
-		},
+		Options:       d.Options,
 	}
 
 	return &s

internal/info/auto.go

@@ -16,31 +16,51 @@
 
 package info
 
-import "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/info"
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/info"
+)
 
-// Logger is a basic interface for logging to allow these functions to be called
-// from code where logrus is not used.
-type Logger interface {
-	Infof(string, ...interface{})
-	Debugf(string, ...interface{})
+// infoInterface provides an alias for mocking.
+//
+//go:generate moq -stub -out info-interface_mock.go . infoInterface
+type infoInterface info.Interface
+
+type resolver struct {
+	logger logger.Interface
+	info   info.Interface
 }
 
 // ResolveAutoMode determines the correct mode for the platform if set to "auto"
-func ResolveAutoMode(logger Logger, mode string) (rmode string) {
+func ResolveAutoMode(logger logger.Interface, mode string, image image.CUDA) (rmode string) {
+	nvinfo := info.New()
+	r := resolver{
+		logger: logger,
+		info:   nvinfo,
+	}
+	return r.resolveMode(mode, image)
+}
+
+// resolveMode determines the correct mode for the platform if set to "auto"
+func (r resolver) resolveMode(mode string, image image.CUDA) (rmode string) {
 	if mode != "auto" {
 		return mode
 	}
 	defer func() {
-		logger.Infof("Auto-detected mode as '%v'", rmode)
+		r.logger.Infof("Auto-detected mode as '%v'", rmode)
 	}()
 
-	nvinfo := info.New()
+	if onlyFullyQualifiedCDIDevices(image) {
+		return "cdi"
+	}
 
-	isTegra, reason := nvinfo.IsTegraSystem()
-	logger.Debugf("Is Tegra-based system? %v: %v", isTegra, reason)
+	isTegra, reason := r.info.IsTegraSystem()
+	r.logger.Debugf("Is Tegra-based system? %v: %v", isTegra, reason)
 
-	hasNVML, reason := nvinfo.HasNvml()
-	logger.Debugf("Has NVML? %v: %v", hasNVML, reason)
+	hasNVML, reason := r.info.HasNvml()
+	r.logger.Debugf("Has NVML? %v: %v", hasNVML, reason)
 
 	if isTegra && !hasNVML {
 		return "csv"
@@ -48,3 +68,14 @@ func ResolveAutoMode(logger Logger, mode string) (rmode string) {
 
 	return "legacy"
 }
+
+func onlyFullyQualifiedCDIDevices(image image.CUDA) bool {
+	var hasCDIdevice bool
+	for _, device := range image.DevicesFromEnvvars("NVIDIA_VISIBLE_DEVICES").List() {
+		if !cdi.IsQualifiedName(device) {
+			return false
+		}
+		hasCDIdevice = true
+	}
+	return hasCDIdevice
+}

internal/info/auto_test.go

@@ -19,8 +19,10 @@ package info
 import (
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/info"
 )
 
 func TestResolveAutoMode(t *testing.T) {
@@ -30,23 +32,123 @@ func TestResolveAutoMode(t *testing.T) {
 		description  string
 		mode         string
 		expectedMode string
+		info         info.Interface
+		image        image.CUDA
 	}{
 		{
 			description:  "non-auto resolves to input",
 			mode:         "not-auto",
 			expectedMode: "not-auto",
 		},
-		// TODO: The following test is brittle in that it will break on Tegra-based systems.
-		// {
-		// 	description:  "auto resolves to legacy",
-		// 	mode:         "auto",
-		// 	expectedMode: "legacy",
-		// },
+		{
+			description: "nvml non-tegra resolves to legacy",
+			mode:        "auto",
+			info: &infoInterfaceMock{
+				HasNvmlFunc: func() (bool, string) {
+					return true, "nvml"
+				},
+				IsTegraSystemFunc: func() (bool, string) {
+					return false, "tegra"
+				},
+			},
+			expectedMode: "legacy",
+		},
+		{
+			description: "non-nvml non-tegra resolves to legacy",
+			mode:        "auto",
+			info: &infoInterfaceMock{
+				HasNvmlFunc: func() (bool, string) {
+					return false, "nvml"
+				},
+				IsTegraSystemFunc: func() (bool, string) {
+					return false, "tegra"
+				},
+			},
+			expectedMode: "legacy",
+		},
+		{
+			description: "nvml tegra resolves to legacy",
+			mode:        "auto",
+			info: &infoInterfaceMock{
+				HasNvmlFunc: func() (bool, string) {
+					return true, "nvml"
+				},
+				IsTegraSystemFunc: func() (bool, string) {
+					return true, "tegra"
+				},
+			},
+			expectedMode: "legacy",
+		},
+		{
+			description: "non-nvml tegra resolves to csv",
+			mode:        "auto",
+			info: &infoInterfaceMock{
+				HasNvmlFunc: func() (bool, string) {
+					return false, "nvml"
+				},
+				IsTegraSystemFunc: func() (bool, string) {
+					return true, "tegra"
+				},
+			},
+			expectedMode: "csv",
+		},
+		{
+			description:  "cdi devices resolves to cdi",
+			mode:         "auto",
+			expectedMode: "cdi",
+			image: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES": "nvidia.com/gpu=all",
+			},
+		},
+		{
+			description:  "multiple cdi devices resolves to cdi",
+			mode:         "auto",
+			expectedMode: "cdi",
+			image: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES": "nvidia.com/gpu=0,nvidia.com/gpu=1",
+			},
+		},
+		{
+			description: "at least one non-cdi device resolves to legacy",
+			mode:        "auto",
+			image: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES": "nvidia.com/gpu=0,0",
+			},
+			info: &infoInterfaceMock{
+				HasNvmlFunc: func() (bool, string) {
+					return true, "nvml"
+				},
+				IsTegraSystemFunc: func() (bool, string) {
+					return true, "tegra"
+				},
+			},
+			expectedMode: "legacy",
+		},
+		{
+			description: "at least one non-cdi device resolves to csv",
+			mode:        "auto",
+			image: image.CUDA{
+				"NVIDIA_VISIBLE_DEVICES": "nvidia.com/gpu=0,0",
+			},
+			info: &infoInterfaceMock{
+				HasNvmlFunc: func() (bool, string) {
+					return false, "nvml"
+				},
+				IsTegraSystemFunc: func() (bool, string) {
+					return true, "tegra"
+				},
+			},
+			expectedMode: "csv",
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			mode := ResolveAutoMode(logger, tc.mode)
+			r := resolver{
+				logger: logger,
+				info:   tc.info,
+			}
+			mode := r.resolveMode(tc.mode, tc.image)
 			require.EqualValues(t, tc.expectedMode, mode)
 		})
 	}

internal/info/info-interface_mock.go

@@ -0,0 +1,153 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package info
+
+import (
+	"sync"
+)
+
+// Ensure, that infoInterfaceMock does implement infoInterface.
+// If this is not the case, regenerate this file with moq.
+var _ infoInterface = &infoInterfaceMock{}
+
+// infoInterfaceMock is a mock implementation of infoInterface.
+//
+//	func TestSomethingThatUsesinfoInterface(t *testing.T) {
+//
+//		// make and configure a mocked infoInterface
+//		mockedinfoInterface := &infoInterfaceMock{
+//			HasDXCoreFunc: func() (bool, string) {
+//				panic("mock out the HasDXCore method")
+//			},
+//			HasNvmlFunc: func() (bool, string) {
+//				panic("mock out the HasNvml method")
+//			},
+//			IsTegraSystemFunc: func() (bool, string) {
+//				panic("mock out the IsTegraSystem method")
+//			},
+//		}
+//
+//		// use mockedinfoInterface in code that requires infoInterface
+//		// and then make assertions.
+//
+//	}
+type infoInterfaceMock struct {
+	// HasDXCoreFunc mocks the HasDXCore method.
+	HasDXCoreFunc func() (bool, string)
+
+	// HasNvmlFunc mocks the HasNvml method.
+	HasNvmlFunc func() (bool, string)
+
+	// IsTegraSystemFunc mocks the IsTegraSystem method.
+	IsTegraSystemFunc func() (bool, string)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// HasDXCore holds details about calls to the HasDXCore method.
+		HasDXCore []struct {
+		}
+		// HasNvml holds details about calls to the HasNvml method.
+		HasNvml []struct {
+		}
+		// IsTegraSystem holds details about calls to the IsTegraSystem method.
+		IsTegraSystem []struct {
+		}
+	}
+	lockHasDXCore     sync.RWMutex
+	lockHasNvml       sync.RWMutex
+	lockIsTegraSystem sync.RWMutex
+}
+
+// HasDXCore calls HasDXCoreFunc.
+func (mock *infoInterfaceMock) HasDXCore() (bool, string) {
+	callInfo := struct {
+	}{}
+	mock.lockHasDXCore.Lock()
+	mock.calls.HasDXCore = append(mock.calls.HasDXCore, callInfo)
+	mock.lockHasDXCore.Unlock()
+	if mock.HasDXCoreFunc == nil {
+		var (
+			bOut bool
+			sOut string
+		)
+		return bOut, sOut
+	}
+	return mock.HasDXCoreFunc()
+}
+
+// HasDXCoreCalls gets all the calls that were made to HasDXCore.
+// Check the length with:
+//
+//	len(mockedinfoInterface.HasDXCoreCalls())
+func (mock *infoInterfaceMock) HasDXCoreCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockHasDXCore.RLock()
+	calls = mock.calls.HasDXCore
+	mock.lockHasDXCore.RUnlock()
+	return calls
+}
+
+// HasNvml calls HasNvmlFunc.
+func (mock *infoInterfaceMock) HasNvml() (bool, string) {
+	callInfo := struct {
+	}{}
+	mock.lockHasNvml.Lock()
+	mock.calls.HasNvml = append(mock.calls.HasNvml, callInfo)
+	mock.lockHasNvml.Unlock()
+	if mock.HasNvmlFunc == nil {
+		var (
+			bOut bool
+			sOut string
+		)
+		return bOut, sOut
+	}
+	return mock.HasNvmlFunc()
+}
+
+// HasNvmlCalls gets all the calls that were made to HasNvml.
+// Check the length with:
+//
+//	len(mockedinfoInterface.HasNvmlCalls())
+func (mock *infoInterfaceMock) HasNvmlCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockHasNvml.RLock()
+	calls = mock.calls.HasNvml
+	mock.lockHasNvml.RUnlock()
+	return calls
+}
+
+// IsTegraSystem calls IsTegraSystemFunc.
+func (mock *infoInterfaceMock) IsTegraSystem() (bool, string) {
+	callInfo := struct {
+	}{}
+	mock.lockIsTegraSystem.Lock()
+	mock.calls.IsTegraSystem = append(mock.calls.IsTegraSystem, callInfo)
+	mock.lockIsTegraSystem.Unlock()
+	if mock.IsTegraSystemFunc == nil {
+		var (
+			bOut bool
+			sOut string
+		)
+		return bOut, sOut
+	}
+	return mock.IsTegraSystemFunc()
+}
+
+// IsTegraSystemCalls gets all the calls that were made to IsTegraSystem.
+// Check the length with:
+//
+//	len(mockedinfoInterface.IsTegraSystemCalls())
+func (mock *infoInterfaceMock) IsTegraSystemCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockIsTegraSystem.RLock()
+	calls = mock.calls.IsTegraSystem
+	mock.lockIsTegraSystem.RUnlock()
+	return calls
+}