Merge branch 'skip-for-point-release' into 'main'

Skip components for patch releases

See merge request nvidia/container-toolkit/container-toolkit!374

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker:stable
+  image: docker
   services:
-    - name: docker:stable-dind
+    - name: docker:dind
       command: ["--experimental"]
 
 variables:
@@ -23,6 +23,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -34,51 +35,99 @@ stages:
   - scan
   - release
 
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
+.main-or-manual:
+  rules:
+    - !reference [.pipeline-trigger-rules, rules]
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: manual
+
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
+  rules:
+    - !reference [.main-or-manual, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
+
 # Define the distribution targets
 .dist-amazonlinux2:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: amazonlinux2
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-centos7:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: centos7
     CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-centos8:
   variables:
     DIST: centos8
     CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-debian10:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: debian10
-
-.dist-debian9:
-  variables:
-    DIST: debian9
+    PACKAGE_REPO_TYPE: debian
 
 .dist-opensuse-leap15.1:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: opensuse-leap15.1
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-ubi8:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubi8
     CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-ubuntu16.04:
-  variables:
-    DIST: ubuntu16.04
+    PACKAGE_REPO_TYPE: rpm
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
     CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+    PACKAGE_REPO_TYPE: debian
 
 .dist-ubuntu20.04:
   variables:
     DIST: ubuntu20.04
     CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+    PACKAGE_REPO_TYPE: debian
 
 .dist-packaging:
   variables:
@@ -98,6 +147,8 @@ stages:
     ARCH: arm64
 
 .arch-ppc64le:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     ARCH: ppc64le
 
@@ -138,7 +189,7 @@ test-packaging:
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.3.10
+    - export REGCTL_VERSION=v0.4.5
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
@@ -210,13 +261,6 @@ release:staging-centos7:
   needs:
     - image-centos7
 
-release:staging-centos8:
-  extends:
-    - .release:staging
-    - .dist-centos8
-  needs:
-    - image-centos8
-
 release:staging-ubi8:
   extends:
     - .release:staging
@@ -224,16 +268,6 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
 release:staging-ubuntu20.04:
   extends:
     - .release:staging

.github/workflows/blossom-ci.yaml

@@ -0,0 +1,113 @@
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to trigger ci on hybrid infra (github + self hosted runner)
+name: Blossom-CI
+on:
+  issue_comment:
+    types: [created]
+  workflow_dispatch:
+      inputs:
+          platform:
+            description: 'runs-on argument'     
+            required: false
+          args:
+            description: 'argument'     
+            required: false
+jobs:
+  Authorization:
+    name: Authorization
+    runs-on: blossom 
+    outputs:
+      args: ${{ env.args }}
+     
+    # This job only runs for pull request comments
+    if: |
+         contains( '\
+         anstockatnv,\
+         rohitrajani2018,\
+         cdesiniotis,\
+         shivamerla,\
+         ArangoGutierrez,\
+         elezar,\
+         klueska,\
+         zvonkok,\
+         ', format('{0},', github.actor)) && 
+         github.event.comment.body == '/blossom-ci'  
+    steps:
+      - name: Check if comment is issued by authorized person
+        run: blossom-ci
+        env:
+          OPERATION: 'AUTH'
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+      
+  Vulnerability-scan:
+    name: Vulnerability scan
+    needs: [Authorization]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
+          ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
+          lfs: 'true'
+      
+      # repo specific steps 
+      #- name: Setup java
+      #  uses: actions/setup-java@v1
+      #  with:
+      #    java-version: 1.8
+      
+      # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file
+      #- name: Setup blackduck properties
+      #  run: |
+      #       PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g')
+      #       echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties
+      #       echo detect.maven.included.scopes=compile >> application.properties
+          
+      - name: Run blossom action
+        uses: NVIDIA/blossom-action@main
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
+        with:
+          args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }}
+          args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }}
+          args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }}
+          
+  Job-trigger:
+    name: Start ci job
+    needs: [Vulnerability-scan]
+    runs-on: blossom
+    steps:
+      - name: Start ci job
+        run: blossom-ci
+        env:
+          OPERATION: 'START-CI-JOB'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              
+  Upload-Log:
+    name: Upload log
+    runs-on: blossom
+    if : github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here)
+        run: blossom-ci
+        env:
+          OPERATION: 'POST-PROCESSING'
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,9 +1,13 @@
 dist
+artifacts
 *.swp
 *.swo
 /coverage.out*
 /test/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
+/nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk
 /shared-*
+/release-*

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,7 +94,7 @@ unit-tests:
     - .multi-arch-build
     - .package-artifacts
   stage: package-build
-  timeout: 2h 30m
+  timeout: 3h
   script:
     - ./scripts/build-packages.sh ${DIST}-${ARCH}
 
@@ -116,12 +116,6 @@ package-amazonlinux2-x86_64:
     - .dist-amazonlinux2
     - .arch-x86_64
 
-package-centos7-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-ppc64le
-
 package-centos7-x86_64:
   extends:
     - .package-build
@@ -152,30 +146,12 @@ package-debian10-amd64:
     - .dist-debian10
     - .arch-amd64
 
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
 package-opensuse-leap15.1-x86_64:
   extends:
     - .package-build
     - .dist-opensuse-leap15.1
     - .arch-x86_64
 
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -216,7 +192,7 @@ package-ubuntu18.04-ppc64le:
   before_script:
     - !reference [.buildx-setup, before_script]
 
-    - apk add --no-cache bash make
+    - apk add --no-cache bash make git
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
@@ -228,19 +204,8 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
-    - package-centos7-ppc64le
     - package-centos7-x86_64
 
-image-centos8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos8
-  needs:
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
 image-ubi8:
   extends:
     - .image-build
@@ -252,16 +217,6 @@ image-ubi8:
     - package-centos8-x86_64
     - package-centos8-ppc64le
 
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
-
 image-ubuntu20.04:
   extends:
     - .image-build
@@ -270,7 +225,8 @@ image-ubuntu20.04:
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -279,21 +235,24 @@ image-packaging:
     - .package-artifacts
     - .dist-packaging
   needs:
-    - package-amazonlinux2-aarch64
-    - package-amazonlinux2-x86_64
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-    - package-centos8-aarch64
-    - package-centos8-ppc64le
-    - package-centos8-x86_64
-    - package-debian10-amd64
-    - package-debian9-amd64
-    - package-opensuse-leap15.1-x86_64
-    - package-ubuntu16.04-amd64
-    - package-ubuntu16.04-ppc64le
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-centos8-aarch64
+    - job: package-centos8-x86_64
+    - job: package-ubuntu18.04-amd64
+    - job: package-ubuntu18.04-arm64
+    - job: package-amazonlinux2-aarch64
+      optional: true
+    - job: package-amazonlinux2-x86_64
+      optional: true
+    - job: package-centos7-x86_64
+      optional: true
+    - job: package-centos8-ppc64le
+      optional: true
+    - job: package-debian10-amd64
+      optional: true
+    - job: package-opensuse-leap15.1-x86_64
+      optional: true
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # Define publish test helpers
 .test:toolkit:
@@ -325,34 +284,6 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu18.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-containerd-ubuntu18.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-crio-ubuntu18.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-docker-ubuntu18.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
 test-toolkit-ubuntu20.04:
   extends:
     - .test:toolkit

.gitmodules

@@ -5,6 +5,8 @@
 [submodule "third_party/nvidia-container-runtime"]
 	path = third_party/nvidia-container-runtime
 	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
+	branch = main
 [submodule "third_party/nvidia-docker"]
 	path = third_party/nvidia-docker
 	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
+	branch = main

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,6 +35,8 @@ variables:
   # Define the public staging registry
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
 
 .image-pull:
   stage: image-build
@@ -48,8 +50,9 @@ variables:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PUSH_MULTIPLE_TAGS: "false"
   # We delay the job start to allow the public pipeline to generate the required images.
-  when: delayed
-  start_in: 30 minutes
+  rules:
+    - when: delayed
+      start_in: 30 minutes
   timeout: 30 minutes
   retry:
     max: 2
@@ -67,34 +70,24 @@ variables:
 
 image-centos7:
   extends:
-    - .image-pull
     - .dist-centos7
-
-image-centos8:
-  extends:
     - .image-pull
-    - .dist-centos8
 
 image-ubi8:
   extends:
-    - .image-pull
     - .dist-ubi8
-
-image-ubuntu18.04:
-  extends:
     - .image-pull
-    - .dist-ubuntu18.04
 
 image-ubuntu20.04:
   extends:
-    - .image-pull
     - .dist-ubuntu20.04
+    - .image-pull
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
   extends:
-    - .image-pull
     - .dist-packaging
+    - .image-pull
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -111,10 +104,10 @@ image-packaging:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
-  except:
-    variables:
-      - $SKIP_SCANS && $SKIP_SCANS == "yes"
+    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
+  rules:
+    - if: $SKIP_SCANS != "yes"
+    - when: manual
   before_script:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     # TODO: We should specify the architecture here and scan all architectures
@@ -139,85 +132,51 @@ image-packaging:
 # Define the scan targets
 scan-centos7-amd64:
   extends:
-    - .scan
     - .dist-centos7
     - .platform-amd64
+    - .scan
   needs:
     - image-centos7
 
 scan-centos7-arm64:
   extends:
-    - .scan
     - .dist-centos7
     - .platform-arm64
+    - .scan
   needs:
     - image-centos7
     - scan-centos7-amd64
 
-scan-centos8-amd64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-amd64
-  needs:
-    - image-centos8
-
-scan-centos8-arm64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-arm64
-  needs:
-    - image-centos8
-    - scan-centos8-amd64
-
-scan-ubuntu18.04-amd64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-amd64
-  needs:
-    - image-ubuntu18.04
-
-scan-ubuntu18.04-arm64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-arm64
-  needs:
-    - image-ubuntu18.04
-    - scan-ubuntu18.04-amd64
-
 scan-ubuntu20.04-amd64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-amd64
+    - .scan
   needs:
     - image-ubuntu20.04
 
 scan-ubuntu20.04-arm64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-arm64
+    - .scan
   needs:
     - image-ubuntu20.04
     - scan-ubuntu20.04-amd64
 
 scan-ubi8-amd64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-amd64
+    - .scan
   needs:
     - image-ubi8
 
 scan-ubi8-arm64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-arm64
+    - .scan
   needs:
     - image-ubi8
     - scan-ubi8-amd64
@@ -232,12 +191,30 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
+.release:packages:
+  stage: release
   needs:
-    - image-ubuntu18.04
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash git
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
+    # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
 
 release:staging-ubuntu20.04:
   extends:
@@ -250,25 +227,15 @@ release:staging-ubuntu20.04:
 # Release to NGC
 release:ngc-centos7:
   extends:
-    - .release:ngc
     - .dist-centos7
-
-release:ngc-centos8:
-  extends:
     - .release:ngc
-    - .dist-centos8
-
-release:ngc-ubuntu18.04:
-  extends:
-    - .release:ngc
-    - .dist-ubuntu18.04
 
 release:ngc-ubuntu20.04:
   extends:
-    - .release:ngc
     - .dist-ubuntu20.04
+    - .release:ngc
 
 release:ngc-ubi8:
   extends:
-    - .release:ngc
     - .dist-ubi8
+    - .release:ngc

CHANGELOG.md

@@ -0,0 +1,310 @@
+# NVIDIA Container Toolkit Changelog
+
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
+## v1.12.0-rc.1
+
+* Add support for multiple Docker Swarm resources
+* Improve injection of Vulkan configurations and libraries
+* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
+* [libnvidia-container] Include NVVM compiler library in compute libs
+
+## v1.11.0
+
+* Promote v1.11.0-rc.3 to v1.11.0
+
+## v1.11.0-rc.3
+
+* Build fedora35 packages
+* Introduce an `nvidia-container-toolkit-base` package for better dependency management
+* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
+* Inject platform files into container on Tegra-based systems
+* [toolkit container] Update CUDA base images to 11.7.1
+* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
+
+## v1.11.0-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.0-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
+## v1.10.0
+
+* Promote v1.10.0-rc.3 to v1.10.0
+
+## v1.10.0-rc.3
+
+* Use default config instead of raising an error if config file cannot be found
+* Ignore NVIDIA_REQUIRE_JETPACK* environment variables for requirement checks
+* Fix bug in detection of Tegra systems where `/sys/devices/soc0/family` is ignored
+* Fix bug where links to devices were detected as devices
+* [libnvida-container] Fix bug introduced when adding libcudadebugger.so to list of libraries
+
+## v1.10.0-rc.2
+
+* Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
+* Switch to debug logging to reduce log verbosity
+* Support logging to logs requested in command line
+* Fix bug when launching containers with relative root path (e.g. using containerd)
+* Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
+* Fix failure to locate low-level runtime if PATH envvar is unset
+* Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
+* Use csv as default mode on Tegra systems without NVML
+* Add --version flag to all CLIs
+* [libnvidia-container] Bump libtirpc to 1.3.2
+* [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
+* [libnvidia-container] Add libcudadebugger.so to list of compute libraries
+
+## v1.10.0-rc.1
+
+* Include nvidia-ctk CLI in installed binaries
+* Add experimental option to NVIDIA Container Runtime
+
+## v1.9.0
+
+* [libnvidia-container] Add additional check for Tegra in /sys/.../family file in CLI
+* [libnvidia-container] Update jetpack-specific CLI option to only load Base CSV files by default
+* [libnvidia-container] Fix bug (from 1.8.0) when mounting GSP firmware into containers without /lib to /usr/lib symlinks
+* [libnvidia-container] Update nvml.h to CUDA 11.6.1 nvML_DEV 11.6.55
+* [libnvidia-container] Update switch statement to include new brands from latest nvml.h
+* [libnvidia-container] Process all --require flags on Jetson platforms
+* [libnvidia-container] Fix long-standing issue with running ldconfig on Debian systems
+
+## v1.8.1
+
+* [libnvidia-container] Fix bug in determining cgroup root when running in nested containers
+* [libnvidia-container] Fix permission issue when determining cgroup version
+
+## v1.8.0
+
+* Promote 1.8.0-rc.2-1 to 1.8.0
+
+## v1.8.0-rc.2
+
+* Remove support for building amazonlinux1 packages
+
+## v1.8.0-rc.1
+
+* [libnvidia-container] Add support for cgroupv2
+* Release toolkit-container images from nvidia-container-toolkit repository
+
+## v1.7.0
+
+* Promote 1.7.0-rc.1-1 to 1.7.0
+ * Bump Golang version to 1.16.4
+
+## v1.7.0-rc.1
+
+* Specify containerd runtime type as string in config tools to remove dependency on containerd package
+* Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
+
+## v1.6.0
+
+* Promote 1.6.0-rc.3-1 to 1.6.0
+ * Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
+
+## v1.6.0-rc.3
+
+* Add supported-driver-capabilities config option to the nvidia-container-toolkit
+* Move OCI and command line checks for runtime to internal oci package
+
+## v1.6.0-rc.2
+
+* Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+## v1.6.0-rc.1
+
+* Add AARCH64 package for Amazon Linux 2
+* Include nvidia-container-runtime into nvidia-container-toolkit package
+
+## v1.5.1
+
+* Fix bug where Docker Swarm device selection is ignored if NVIDIA_VISIBLE_DEVICES is also set
+* Improve unit testing by using require package and adding coverage reports
+* Remove unneeded go dependencies by running go mod tidy
+* Move contents of pkg directory to cmd for CLI tools
+* Ensure make binary target explicitly sets GOOS
+
+## v1.5.0
+
+* Add dependence on libnvidia-container-tools >= 1.4.0
+* Add golang check targets to Makefile
+* Add Jenkinsfile definition for build targets
+* Move docker.mk to docker folder
+
+## v1.4.2
+
+* Add dependence on libnvidia-container-tools >= 1.3.3
+
+## v1.4.1
+
+* Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
+* Add dependence on libnvidia-container-tools >= 1.3.2
+
+## v1.4.0
+
+* Add 'compute' capability to list of defaults
+* Add dependence on libnvidia-container-tools >= 1.3.1
+
+## v1.3.0
+
+* Promote 1.3.0-rc.2-1 to 1.3.0
+* Add dependence on libnvidia-container-tools >= 1.3.0
+
+## v1.3.0-rc.2
+
+* 2c180947 Add more tests for new semantics with device list from volume mounts
+* 7c003857 Refactor accepting device lists from volume mounts as a boolean
+
+## v1.3.0-rc.1
+
+* b50d86c1 Update build system to accept a TAG variable for things like rc.x
+* fe65573b Add common CI tests for things like golint, gofmt, unit tests, etc.
+* da6fbb34 Revert "Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*"
+* a7fb3330 Flip build-all targets to run automatically on merge requests
+* 8b248b66 Rename github.com/NVIDIA/container-toolkit to nvidia-container-toolkit
+* da36874e Add new config options to pull device list from mounted files instead of ENVVAR
+
+## v1.2.1
+
+* 4e6e0ed4 Add 'ngx' to list of*all* driver capabilities
+* 2f4af743 List config.toml as a config file in the RPM SPEC
+
+## v1.2.0
+
+*  8e0aab46 Fix repo listed in changelog for debian distributions
+*  320bb6e4 Update dependence on libnvidia-container to 1.2.0
+*  6cfc8097 Update package license to match source license
+*  e7dc3cbb Fix debian copyright file
+*  d3aee3e0 Add the 'ngx' driver capability
+
+## v1.1.2
+
+* c32237f3 Add support for parsing Linux Capabilities for older OCI specs
+
+## v1.1.1
+
+* d202aded Update dependence to libnvidia-container 1.1.1
+
+## v1.1.0
+
+* 4e4de762 Update build system to support multi-arch builds
+* fcc1d116 Add support for MIG (Multi-Instance GPUs)
+* d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
+* 60f165ad Add no-pivot option to toolkit
+
+## v1.0.5
+
+* Initial release. Replaces older package nvidia-container-runtime-hook. (Closes: #XXXXXX)

Makefile

@@ -39,7 +39,7 @@ CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
 CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate licenses $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
@@ -51,6 +51,7 @@ CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 else
 CLI_VERSION = $(VERSION)
 endif
+CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
 
 GOOS ?= linux
 
@@ -60,7 +61,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.gitCommit=$(GIT_COMMIT) -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-extldflags=-Wl,-z,lazy -s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...
@@ -102,6 +103,9 @@ misspell:
 vet:
 	go vet $(MODULE)/...
 
+licenses:
+	go-licenses csv $(MODULE)/...
+
 COVERAGE_FILE := coverage.out
 test: build cmds
 	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...

build/container/Dockerfile.centos

@@ -67,9 +67,9 @@ ARG TARGETARCH
 ENV PACKAGE_ARCH ${TARGETARCH}
 RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
     yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
 
@@ -85,7 +85,7 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-COPY ./LICENSE /licenses/LICENSE
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES

build/container/Dockerfile.packaging

@@ -15,15 +15,27 @@
 ARG BASE_DIST
 ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
-ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
-
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
-COPY ./LICENSE /licenses/LICENSE
+# build-args are added to the manifest.txt file below.
+ARG BASE_DIST
+ARG PACKAGE_DIST
+ARG PACKAGE_VERSION
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
+ARG SOURCE_DATE_EPOCH
+ARG VERSION
+
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

build/container/Dockerfile.ubuntu

@@ -75,9 +75,9 @@ RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
     fi
 
 RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
 
@@ -93,7 +93,7 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-COPY ./LICENSE /licenses/LICENSE
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES

build/container/Makefile

@@ -43,14 +43,14 @@ OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu18.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+DEFAULT_PUSH_TARGET := ubuntu20.04
+DISTRIBUTIONS := ubuntu20.04 ubi8 centos7
 
 META_TARGETS := packaging
 
 BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
 PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
 
 .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -94,6 +94,10 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
+		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
+		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
+		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
 		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
@@ -102,24 +106,20 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
 build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
 
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
 build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
-build-centos%: BASE_DIST = $(*)
-build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: PACKAGE_DIST = $(BASE_DIST)
-build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-centos7: BASE_DIST = $(*)
+build-centos7: DOCKERFILE_SUFFIX := centos
+build-centos7: PACKAGE_DIST = $(BASE_DIST)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
@@ -143,10 +143,7 @@ test-packaging:
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/multi-arch.mk

@@ -30,5 +30,8 @@ push-short:
 # We only have x86_64 packages for centos7
 build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
 
+# We only generate amd64 image for ubuntu18.04
+build-ubuntu18.04: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
 # We only generate a single image for packaging targets
 build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64

cmd/nvidia-container-runtime-hook/container_config.go

@@ -10,11 +10,10 @@ import (
 	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
 )
 
-var envSwarmGPU *string
-
 const (
 	envCUDAVersion          = "CUDA_VERSION"
 	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
@@ -132,7 +131,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -141,67 +140,58 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		for _, c := range caps {
+			if c == capSysAdmin {
+				return true
+			}
 		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
+		return false
 	}
 
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	process := specs.Process{
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
-func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
-	// Build a list of envvars to consider.
-	envVars := []string{envNVVisibleDevices}
-	if envSwarmGPU != nil {
-		// The Swarm envvar has higher precedence.
-		envVars = append([]string{*envSwarmGPU}, envVars...)
-	}
-
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
-	for _, envVar := range envVars {
-		if devs, ok := env[envVar]; ok {
-			devices = &devs
+func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
+	// We check if the image has at least one of the Swarm resource envvars defined and use this
+	// if specified.
+	var hasSwarmEnvvar bool
+	for _, envvar := range swarmResourceEnvvars {
+		if _, exists := image[envvar]; exists {
+			hasSwarmEnvvar = true
 			break
 		}
 	}
 
-	// Environment variable unset with legacy image: default to "all".
-	if devices == nil && legacyImage {
-		all := "all"
-		return &all
+	var devices []string
+	if hasSwarmEnvvar {
+		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
+	} else {
+		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
 	}
 
-	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
+	if len(devices) == 0 {
 		return nil
 	}
 
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		empty := ""
-		return &empty
-	}
+	devicesString := strings.Join(devices, ",")
 
-	// Any other value.
-	return devices
+	return &devicesString
 }
 
 func getDevicesFromMounts(mounts []Mount) *string {
@@ -241,7 +231,7 @@ func getDevicesFromMounts(mounts []Mount) *string {
 	return &ret
 }
 
-func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
+func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
 		devices := getDevicesFromMounts(mounts)
@@ -251,7 +241,7 @@ func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, p
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(env, legacyImage)
+	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
 	if devices == nil {
 		return nil
 	}
@@ -307,7 +297,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	legacyImage := image.IsLegacy()
 
 	var devices string
-	if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil {
+	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
 		devices = *d
 	} else {
 		// 'nil' devices means this is not a GPU container.
@@ -369,7 +359,6 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 	}
 
 	privileged := isPrivileged(s)
-	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"fmt"
 	"path/filepath"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -68,7 +70,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Legacy image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
+				envNVVisibleDevices: "void",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -225,7 +227,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Modern image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
+				envNVVisibleDevices: "void",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -448,6 +450,44 @@ func TestGetNvidiaConfig(t *testing.T) {
 				DriverCapabilities: defaultDriverCapabilities.String(),
 			},
 		},
+		{
+			description: "Hook config set, swarmResource overrides device selection",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SwarmResource: func() *string {
+					s := "DOCKER_SWARM_RESOURCE"
+					return &s
+				}(),
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "GPU1,GPU2",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, comma separated swarmResource is split and overrides device selection",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SwarmResource: func() *string {
+					s := "NOT_DOCKER_SWARM_RESOURCE,DOCKER_SWARM_RESOURCE"
+					return &s
+				}(),
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "GPU1,GPU2",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+			},
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
@@ -671,7 +711,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				hookConfig := getDefaultHookConfig()
 				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
+				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)
 			}
 
 			// For all other tests, just grab the devices and check the results
@@ -688,13 +728,13 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
 	gpuID := "GPU-12345"
 	anotherGPUID := "GPU-67890"
+	thirdGPUID := "MIG-12345"
 
 	var tests = []struct {
-		description     string
-		envSwarmGPU     *string
-		env             map[string]string
-		legacyImage     bool
-		expectedDevices *string
+		description          string
+		swarmResourceEnvvars []string
+		env                  map[string]string
+		expectedDevices      *string
 	}{
 		{
 			description: "empty env returns nil for non-legacy image",
@@ -729,13 +769,15 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
 			env: map[string]string{
 				envNVVisibleDevices: gpuID,
+				envCUDAVersion:      "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description:     "empty env returns all for legacy image",
-			legacyImage:     true,
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envCUDAVersion: "legacy",
+			},
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
@@ -781,86 +823,116 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
 			description: "empty env returns all for legacy image",
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
 		// enabled
 		{
-			description: "empty env returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "empty env returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 		},
 		{
-			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "",
 			},
 		},
 		{
-			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "void",
 			},
 		},
 		{
-			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: "none",
 			},
 			expectedDevices: &empty,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
 			},
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
+				envCUDAVersion:        "legacy",
 			},
-			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS is selected if present",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS is selected if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
 		{
-			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
-			envSwarmGPU: &envDockerResourceGPUs,
+			description:          "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
+		{
+			description:          "All available swarm resource envvars are selected and override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS":            thirdGPUID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: func() *string {
+				result := fmt.Sprintf("%s,%s", thirdGPUID, anotherGPUID)
+				return &result
+			}(),
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				envNVVisibleDevices:               gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
 	}
 
 	for i, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			envSwarmGPU = tc.envSwarmGPU
-			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
+			devices := getDevicesFromEnvvar(image.CUDA(tc.env), tc.swarmResourceEnvvars)
 			if tc.expectedDevices == nil {
 				require.Nil(t, devices, "%d: %v", i, tc)
 				return

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"strings"
 
 	"github.com/BurntSushi/toml"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
@@ -34,7 +35,7 @@ type CLIConfig struct {
 	Ldconfig    *string  `toml:"ldconfig"`
 }
 
-// HookConfig : options for the nvidia-container-toolkit.
+// HookConfig : options for the nvidia-container-runtime-hook.
 type HookConfig struct {
 	DisableRequire                 bool               `toml:"disable-require"`
 	SwarmResource                  *string            `toml:"swarm-resource"`
@@ -42,8 +43,9 @@ type HookConfig struct {
 	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
 	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
+	NvidiaContainerCLI         CLIConfig                `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime     config.RuntimeConfig     `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHook config.RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
 }
 
 func getDefaultHookConfig() HookConfig {
@@ -65,7 +67,8 @@ func getDefaultHookConfig() HookConfig {
 			User:        nil,
 			Ldconfig:    nil,
 		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntime:     *config.GetDefaultRuntimeConfig(),
+		NVIDIAContainerRuntimeHook: *config.GetDefaultRuntimeHookConfig(),
 	}
 }
 
@@ -116,3 +119,22 @@ func (c HookConfig) getConfigOption(fieldName string) string {
 	}
 	return v
 }
+
+// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
+func (c *HookConfig) getSwarmResourceEnvvars() []string {
+	if c.SwarmResource == nil {
+		return nil
+	}
+
+	candidates := strings.Split(*c.SwarmResource, ",")
+
+	var envvars []string
+	for _, c := range candidates {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) > 0 {
+			envvars = append(envvars, trimmed)
+		}
+	}
+
+	return envvars
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -103,3 +103,59 @@ func TestGetHookConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSwarmResourceEnvvars(t *testing.T) {
+	testCases := []struct {
+		value    string
+		expected []string
+	}{
+		{
+			value:    "nil",
+			expected: nil,
+		},
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value:    " ",
+			expected: nil,
+		},
+		{
+			value:    "single",
+			expected: []string{"single"},
+		},
+		{
+			value:    "single ",
+			expected: []string{"single"},
+		},
+		{
+			value:    "one,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one ,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one, two",
+			expected: []string{"one", "two"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			c := &HookConfig{
+				SwarmResource: func() *string {
+					if tc.value == "nil" {
+						return nil
+					}
+					return &tc.value
+				}(),
+			}
+
+			envvars := c.getSwarmResourceEnvvars()
+			require.EqualValues(t, tc.expected, envvars)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/main.go

@@ -74,8 +74,8 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+	if !hook.NVIDIAContainerRuntimeHook.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
 	}
 
 	container := getContainerConfig(hook)

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/README.md

@@ -2,24 +2,86 @@
 
 The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
 
-## Standard Mode
+## Configuration
 
-In the standard mode configuration, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from project [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+The NVIDIA Container Runtime uses file-based configuration, with the config stored in `/etc/nvidia-container-runtime/config.toml`. The `/etc` path can be overridden using the `XDG_CONFIG_HOME` environment variable with the `${XDG_CONFIG_HOME}/nvidia-container-runtime/config.toml` file used instead if this environment variable is set.
 
-## Experimental Mode
+This config file may contain options for other components of the NVIDIA container stack and for the NVIDIA Container Runtime, the relevant config section is `nvidia-container-runtime`
 
-The NVIDIA Container Runtime can be configured in an experimental mode by setting the following options in the runtime's `config.toml` file:
+### Logging
+
+The `log-level` config option (default: `"info"`) specifies the log level to use and the `debug` option, if set, specifies a log file to which logs for the NVIDIA Container Runtime must be written.
+
+In addition to this, the NVIDIA Container Runtime considers the value of `--log` and `--log-format` flags that may be passed to it by a container runtime such as docker or containerd. If the `--debug` flag is present the log-level specified in the config file is overridden as `"debug"`.
+
+### Low-level Runtime Path
+
+The `runtimes` config option allows for the low-level runtime to be specified. The first entry in this list that is an existing executable file is used as the low-level runtime. If the entry is not a path, the `PATH` is searched for a matching executable. If the entry is a path this is checked instead.
+
+The default value for this setting is:
+```toml
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+```
+
+and if, for example, `crun` is to be used instead this can be changed to:
+```toml
+runtimes = [
+    "crun",
+]
+```
+
+### Runtime Mode
+
+The `mode` config option (default `"auto"`) controls the high-level behaviour of the runtime.
+
+#### Auto Mode
+
+When `mode` is set to `"auto"`, the runtime employs heuristics to determine which mode to use based on, for example, the platform where the runtime is being run.
+
+#### Legacy Mode
+
+When `mode` is set to `"legacy"`, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from the [libnvidia-container](https://github.com/NVIDIA/libnvidia-container) project.
+
+#### CSV Mode
+
+When `mode` is set to `"csv"`, CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` define the devices and mounts that are to be injected into a container when it is created. The search path for the files can be overridden by modifying the `nvidia-container-runtime.modes.csv.mount-spec-path` in the config as below:
 
 ```toml
 [nvidia-container-runtime]
-experimental = true
+    [nvidia-container-runtime.modes.csv]
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
 ```
 
-When this setting is enabled, the modifications made to the OCI specification are controlled by the `nvidia-container-runtime.discover-mode` option, with the following mode supported:
-
-* `"legacy"`: This mode mirrors the behaviour of the standard mode, inserting the NVIDIA Container Runtime Hook as a `prestart` hook into the container's OCI specification.
-* `"csv"`: This mode uses CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` to define the devices and mounts that are to be injected into a container when it is created.
+This mode is primarily targeted at Tegra-based systems without NVML available.
 
 ### Notes on using the docker CLI
 
-The `docker` CLI supports the `--gpus` flag to select GPUs for inclusion in a container. Since specifying this flag inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification. When experimental mode is activated, the NVIDIA Container Runtime detects the presence of the hook and raises an error. This requirement will be relaxed in the near future.
+Note that only the `"legacy"` NVIDIA Container Runtime mode is directly compatible with the `--gpus` flag implemented by the `docker` CLI (assuming the NVIDIA Container Runtime is not used). The reason for this is that `docker` inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification.
+
+
+If a different mode is explicitly set or detected, the NVIDIA Container Runtime Hook will raise the following error when `--gpus` is set:
+```
+$ docker run --rm --gpus all ubuntu:18.04
+docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'csv'
+invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.: unknown.
+```
+Here NVIDIA Container Runtime must be used explicitly. The recommended way to do this is to specify the `--runtime=nvidia` command line argument as part of the `docker run` commmand as follows:
+```
+$ docker run --rm --gpus all --runtime=nvidia ubuntu:18.04
+```
+
+Alternatively the NVIDIA Container Runtime can be set as the default runtime for docker. This can be done by modifying the `/etc/docker/daemon.json` file as follows:
+```json
+{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}
+```

cmd/nvidia-container-runtime/main.go

@@ -1,84 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer logger.Reset()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -10,9 +10,10 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,7 +42,7 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		logrus.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
@@ -53,11 +54,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		logrus.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		logrus.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,7 +78,7 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -91,9 +92,10 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -170,7 +172,7 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	m := modifier.NewStableRuntimeModifier(logrus.StandardLogger())
 	return m.Modify(spec)
 }
 

cmd/nvidia-ctk/README.md

@@ -1,3 +1,48 @@
 # NVIDIA Container Toolkit CLI
 
 The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,52 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,283 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	allDeviceName = "all"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	output             string
+	format             string
+	deviceNameStrategy string
+	driverRoot         string
+	nvidiaCTKPath      string
+	mode               string
+	vendor             string
+	class              string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &cfg.output,
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       spec.FormatYAML,
+			Destination: &cfg.format,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Aliases:     []string{"discovery-mode"},
+			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       nvcdi.ModeAuto,
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
+			Value:       nvcdi.DeviceNameStrategyIndex,
+			Destination: &cfg.deviceNameStrategy,
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &cfg.driverRoot,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-ctk-path",
+			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
+			Destination: &cfg.nvidiaCTKPath,
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &cfg.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &cfg.class,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+
+	cfg.format = strings.ToLower(cfg.format)
+	switch cfg.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", cfg.format)
+	}
+
+	cfg.mode = strings.ToLower(cfg.mode)
+	switch cfg.mode {
+	case nvcdi.ModeAuto:
+	case nvcdi.ModeNvml:
+	case nvcdi.ModeWsl:
+	case nvcdi.ModeManagement:
+	default:
+		return fmt.Errorf("invalid discovery mode: %v", cfg.mode)
+	}
+
+	_, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
+	if err != nil {
+		return err
+	}
+
+	cfg.nvidiaCTKPath = discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath)
+
+	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			cfg.format = outputFileFormat
+		} else if outputFileFormat != cfg.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", cfg.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(cfg.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(cfg.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	spec, err := m.generateSpec(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
+
+	if cfg.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return spec.Save(cfg.output)
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
+	}
+
+	return ""
+}
+
+func (m command) generateSpec(cfg *config) (spec.Interface, error) {
+	deviceNamer, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device namer: %v", err)
+	}
+
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(cfg.driverRoot),
+		nvcdi.WithNVIDIACTKPath(cfg.nvidiaCTKPath),
+		nvcdi.WithDeviceNamer(deviceNamer),
+		nvcdi.WithMode(string(cfg.mode)),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
+	}
+
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+	var hasAll bool
+	for _, deviceSpec := range deviceSpecs {
+		if deviceSpec.Name == allDeviceName {
+			hasAll = true
+			break
+		}
+	}
+	if !hasAll {
+		allDevice, err := MergeDeviceSpecs(deviceSpecs, allDeviceName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create CDI specification for %q device: %v", allDeviceName, err)
+		}
+		deviceSpecs = append(deviceSpecs, allDevice)
+	}
+
+	commonEdits, err := cdilib.GetCommonEdits()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
+	}
+
+	return spec.New(
+		spec.WithVendor(cfg.vendor),
+		spec.WithClass(cfg.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(cfg.format),
+	)
+}
+
+// MergeDeviceSpecs creates a device with the specified name which combines the edits from the previous devices.
+// If a device of the specified name already exists, an error is returned.
+func MergeDeviceSpecs(deviceSpecs []specs.Device, mergedDeviceName string) (specs.Device, error) {
+	if err := cdi.ValidateDeviceName(mergedDeviceName); err != nil {
+		return specs.Device{}, fmt.Errorf("invalid device name %q: %v", mergedDeviceName, err)
+	}
+	for _, d := range deviceSpecs {
+		if d.Name == mergedDeviceName {
+			return specs.Device{}, fmt.Errorf("device %q already exists", mergedDeviceName)
+		}
+	}
+
+	mergedEdits := edits.NewContainerEdits()
+
+	for _, d := range deviceSpecs {
+		edit := cdi.ContainerEdits{
+			ContainerEdits: &d.ContainerEdits,
+		}
+		mergedEdits.Append(&edit)
+	}
+
+	merged := specs.Device{
+		Name:           mergedDeviceName,
+		ContainerEdits: *mergedEdits.ContainerEdits,
+	}
+	return merged, nil
+}

cmd/nvidia-ctk/cdi/generate/generate_test.go

@@ -0,0 +1,117 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeDeviceSpecs(t *testing.T) {
+	testCases := []struct {
+		description      string
+		deviceSpecs      []specs.Device
+		mergedDeviceName string
+		expectedError    error
+		expected         specs.Device
+	}{
+		{
+			description:      "no devices",
+			mergedDeviceName: "all",
+			expected: specs.Device{
+				Name: "all",
+			},
+		},
+		{
+			description:      "one device",
+			mergedDeviceName: "all",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+			},
+			expected: specs.Device{
+				Name: "all",
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"GPU=0"},
+				},
+			},
+		},
+		{
+			description:      "two devices",
+			mergedDeviceName: "all",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+				{
+					Name: "gpu1",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=1"},
+					},
+				},
+			},
+			expected: specs.Device{
+				Name: "all",
+				ContainerEdits: specs.ContainerEdits{
+					Env: []string{"GPU=0", "GPU=1"},
+				},
+			},
+		},
+		{
+			description:      "has merged device",
+			mergedDeviceName: "gpu0",
+			deviceSpecs: []specs.Device{
+				{
+					Name: "gpu0",
+					ContainerEdits: specs.ContainerEdits{
+						Env: []string{"GPU=0"},
+					},
+				},
+			},
+			expectedError: fmt.Errorf("device %q already exists", "gpu0"),
+		},
+		{
+			description:      "invalid merged device name",
+			mergedDeviceName: ".-not-valid",
+			expectedError:    fmt.Errorf("invalid device name %q", ".-not-valid"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mergedDevice, err := MergeDeviceSpecs(tc.deviceSpecs, tc.mergedDeviceName)
+
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expected, mergedDevice)
+		})
+	}
+}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,159 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type loadSaver interface {
+	Load() (spec.Interface, error)
+	Save(spec.Interface) error
+}
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from string
+	to   string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transform.NewRootTransformer(
+		opts.from,
+		opts.to,
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -0,0 +1,146 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	paths         cli.StringSlice
+	mode          string
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specifiy a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.mode) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value())
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	locator := lookup.NewExecutableLocator(m.logger, "")
+	targets, err := locator.Locate("chmod")
+	if err != nil {
+		return fmt.Errorf("failed to locate chmod: %v", err)
+	}
+	chmodPath := targets[0]
+
+	args := append([]string{filepath.Base(chmodPath), cfg.mode}, paths...)
+
+	return syscall.Exec(chmodPath, args, nil)
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		path := filepath.Join(root, f)
+		if _, err := os.Stat(path); err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -74,7 +74,7 @@ func (m command) build() *cli.Command {
 		},
 		&cli.StringSliceFlag{
 			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as source:target",
+			Usage:       "Specify a specific link to create. The link is specified as target::link",
 			Destination: &cfg.links,
 		},
 		&cli.StringFlag{
@@ -145,7 +145,7 @@ func (m command) run(c *cli.Context, cfg *config) error {
 
 	links := cfg.links.Value()
 	for _, l := range links {
-		parts := strings.Split(l, ":")
+		parts := strings.Split(l, "::")
 		if len(parts) != 2 {
 			m.logger.Warnf("Invalid link specification %v", l)
 			continue

cmd/nvidia-ctk/hook/hook.go

@@ -17,6 +17,8 @@
 package hook
 
 import (
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
 	"github.com/sirupsen/logrus"
@@ -46,6 +48,7 @@ func (m hookCommand) build() *cli.Command {
 	hook.Subcommands = []*cli.Command{
 		ldcache.NewCommand(m.logger),
 		symlinks.NewCommand(m.logger),
+		chmod.NewCommand(m.logger),
 	}
 
 	return &hook

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -84,6 +84,12 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		return fmt.Errorf("failed to determined container root: %v", err)
 	}
 
+	_, err = os.Stat(filepath.Join(containerRoot, "/etc/ld.so.cache"))
+	if err != nil && os.IsNotExist(err) {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		return nil
+	}
+
 	err = m.createConfig(containerRoot, cfg.folders.Value())
 	if err != nil {
 		return fmt.Errorf("failed to update ld.so.conf: %v", err)
@@ -105,6 +111,10 @@ func (m command) createConfig(root string, folders []string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
 	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
 	if err != nil {
 		return fmt.Errorf("failed to create config file: %v", err)

cmd/nvidia-ctk/info/info.go

@@ -0,0 +1,47 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "info",
+		Usage: "Provide information about the system",
+	}
+
+	hook.Subcommands = []*cli.Command{}
+
+	return &hook
+}

cmd/nvidia-ctk/main.go

@@ -19,8 +19,13 @@ package main
 import (
 	"os"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 )
@@ -70,6 +75,10 @@ func main() {
 	// Define the subcommands
 	c.Commands = []*cli.Command{
 		hook.NewCommand(logger),
+		runtime.NewCommand(logger),
+		infoCLI.NewCommand(logger),
+		cdi.NewCommand(logger),
+		system.NewCommand(logger),
 	}
 
 	// Run the CLI

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -0,0 +1,213 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package configure
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
+	"github.com/pelletier/go-toml"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultRuntime = "docker"
+
+	defaultDockerConfigFilePath = "/etc/docker/daemon.json"
+	defaultCrioConfigFilePath   = "/etc/crio/crio.conf"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an configure command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line config
+type config struct {
+	dryRun         bool
+	runtime        string
+	configFilePath string
+	nvidiaOptions  nvidia.Options
+}
+
+func (m command) build() *cli.Command {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the 'configure' command
+	configure := cli.Command{
+		Name:  "configure",
+		Usage: "Add a runtime to the specified container engine",
+		Action: func(c *cli.Context) error {
+			return m.configureWrapper(c, &config)
+		},
+	}
+
+	configure.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "update the runtime configuration as required but don't write changes to disk",
+			Destination: &config.dryRun,
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Usage:       "the target runtime engine. One of [crio, docker]",
+			Value:       defaultRuntime,
+			Destination: &config.runtime,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to the config file for the target runtime",
+			Destination: &config.configFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-name",
+			Usage:       "specify the name of the NVIDIA runtime that will be added",
+			Value:       nvidia.RuntimeName,
+			Destination: &config.nvidiaOptions.RuntimeName,
+		},
+		&cli.StringFlag{
+			Name:        "runtime-path",
+			Usage:       "specify the path to the NVIDIA runtime executable",
+			Value:       nvidia.RuntimeExecutable,
+			Destination: &config.nvidiaOptions.RuntimePath,
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Usage:       "set the specified runtime as the default runtime",
+			Destination: &config.nvidiaOptions.SetAsDefault,
+		},
+	}
+
+	return &configure
+}
+
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.runtime {
+	case "crio":
+		return m.configureCrio(c, config)
+	case "docker":
+		return m.configureDocker(c, config)
+	}
+
+	return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+}
+
+// configureDocker updates the docker config to enable the NVIDIA Container Runtime
+func (m command) configureDocker(c *cli.Context, config *config) error {
+	configFilePath := config.configFilePath
+	if configFilePath == "" {
+		configFilePath = defaultDockerConfigFilePath
+	}
+
+	cfg, err := docker.New(
+		docker.WithPath(configFilePath),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = cfg.AddRuntime(
+		config.nvidiaOptions.RuntimeName,
+		config.nvidiaOptions.RuntimePath,
+		config.nvidiaOptions.SetAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.dryRun {
+		output, err := json.MarshalIndent(cfg, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to convert to JSON: %v", err)
+		}
+		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
+		return nil
+	}
+	n, err := cfg.Save(configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	if n == 0 {
+		m.logger.Infof("Removed empty config from %v", configFilePath)
+	} else {
+		m.logger.Infof("Wrote updated config to %v", configFilePath)
+	}
+	m.logger.Infof("It is recommended that the docker daemon be restarted.")
+
+	return nil
+}
+
+// configureCrio updates the crio config to enable the NVIDIA Container Runtime
+func (m command) configureCrio(c *cli.Context, config *config) error {
+	configFilePath := config.configFilePath
+	if configFilePath == "" {
+		configFilePath = defaultCrioConfigFilePath
+	}
+
+	cfg, err := crio.New(
+		crio.WithPath(configFilePath),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = cfg.AddRuntime(
+		config.nvidiaOptions.RuntimeName,
+		config.nvidiaOptions.RuntimePath,
+		config.nvidiaOptions.SetAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.dryRun {
+		output, err := toml.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("unable to convert to TOML: %v", err)
+		}
+		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
+		return nil
+	}
+	n, err := cfg.Save(configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	if n == 0 {
+		m.logger.Infof("Removed empty config from %v", configFilePath)
+	} else {
+		m.logger.Infof("Wrote updated config to %v", configFilePath)
+	}
+	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
+
+	return nil
+}

cmd/nvidia-ctk/runtime/nvidia/nvidia.go

@@ -0,0 +1,75 @@
+/*
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package nvidia
+
+const (
+	// RuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	RuntimeName = "nvidia"
+	// RuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	RuntimeExecutable = "nvidia-container-runtime"
+)
+
+// Options specifies the options for the NVIDIA Container Runtime w.r.t a container engine such as docker.
+type Options struct {
+	SetAsDefault bool
+	RuntimeName  string
+	RuntimePath  string
+}
+
+// Runtime defines an NVIDIA runtime with a name and a executable
+type Runtime struct {
+	Name string
+	Path string
+}
+
+// DefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o Options) DefaultRuntime() string {
+	if !o.SetAsDefault {
+		return ""
+	}
+
+	return o.RuntimeName
+}
+
+// Runtime creates a runtime struct based on the options.
+func (o Options) Runtime() Runtime {
+	path := o.RuntimePath
+
+	if o.RuntimePath == "" {
+		path = RuntimeExecutable
+	}
+
+	r := Runtime{
+		Name: o.RuntimeName,
+		Path: path,
+	}
+
+	return r
+}
+
+// DockerRuntimesConfig generatest the expected docker config for the specified runtime
+func (r Runtime) DockerRuntimesConfig() map[string]interface{} {
+	runtimes := make(map[string]interface{})
+	runtimes[r.Name] = map[string]interface{}{
+		"path": r.Path,
+		"args": []string{},
+	}
+
+	return runtimes
+}

cmd/nvidia-ctk/runtime/runtime.go

@@ -0,0 +1,49 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type runtimeCommand struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := runtimeCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m runtimeCommand) build() *cli.Command {
+	// Create the 'runtime' command
+	runtime := cli.Command{
+		Name:  "runtime",
+		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	runtime.Subcommands = []*cli.Command{
+		configure.NewCommand(m.logger),
+	}
+
+	return &runtime
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -0,0 +1,175 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
+)
+
+type allPossible struct {
+	logger       *logrus.Logger
+	driverRoot   string
+	deviceMajors devices.Devices
+	migCaps      nvcaps.MigCaps
+}
+
+// newAllPossible returns a new allPossible device node lister.
+// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error) {
+	deviceMajors, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed reading device majors: %v", err)
+	}
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
+	}
+	if migCaps == nil {
+		migCaps = make(nvcaps.MigCaps)
+	}
+
+	l := allPossible{
+		logger:       logger,
+		driverRoot:   driverRoot,
+		deviceMajors: deviceMajors,
+		migCaps:      migCaps,
+	}
+
+	return l, nil
+}
+
+// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func (m allPossible) DeviceNodes() ([]deviceNode, error) {
+	gpus, err := nvpci.NewFrom(
+		filepath.Join(m.driverRoot, nvpci.PCIDevicesRoot),
+	).GetGPUs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GPU information: %v", err)
+	}
+
+	count := len(gpus)
+	if count == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		return nil, nil
+	}
+
+	deviceNodes, err := m.getControlDeviceNodes()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
+	}
+
+	for gpu := 0; gpu < count; gpu++ {
+		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
+		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
+	}
+
+	return deviceNodes, nil
+}
+
+// getControlDeviceNodes generates a list of control devices
+func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
+	var deviceNodes []deviceNode
+
+	// Define the control devices for standard GPUs.
+	controlDevices := []deviceNode{
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
+	}
+
+	deviceNodes = append(deviceNodes, controlDevices...)
+
+	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
+		migControlMinor, exist := m.migCaps[migControlDevice]
+		if !exist {
+			continue
+		}
+
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			migControlMinor.DevicePath(),
+			int(migControlMinor),
+		)
+
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes, nil
+}
+
+// getGPUDeviceNodes generates a list of device nodes for a given GPU.
+func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
+	d := m.newDeviceNode(
+		devices.NVIDIAGPU,
+		fmt.Sprintf("/dev/nvidia%d", gpu),
+		gpu,
+	)
+
+	return []deviceNode{d}
+}
+
+// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
+func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
+	var selectedCapMinors []nvcaps.MigMinor
+	for gi := 0; ; gi++ {
+		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+		giMinor, exist := m.migCaps[giCap]
+		if !exist {
+			break
+		}
+		selectedCapMinors = append(selectedCapMinors, giMinor)
+		for ci := 0; ; ci++ {
+			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+			ciMinor, exist := m.migCaps[ciCap]
+			if !exist {
+				break
+			}
+			selectedCapMinors = append(selectedCapMinors, ciMinor)
+		}
+	}
+
+	var deviceNodes []deviceNode
+	for _, capMinor := range selectedCapMinors {
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			capMinor.DevicePath(),
+			int(capMinor),
+		)
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes
+}
+
+// newDeviceNode creates a new device node with the specified path and major/minor numbers.
+// The path is adjusted for the specified driver root.
+func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
+	major, _ := m.deviceMajors.Get(deviceName)
+
+	return deviceNode{
+		path:  filepath.Join(m.driverRoot, path),
+		major: uint32(major),
+		minor: uint32(minor),
+	}
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -0,0 +1,332 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultDevCharPath = "/dev/char"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	devCharPath string
+	driverRoot  string
+	dryRun      bool
+	watch       bool
+	createAll   bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'create-dev-char-symlinks' command
+	c := cli.Command{
+		Name:  "create-dev-char-symlinks",
+		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "dev-char-path",
+			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
+			Value:       defaultDevCharPath,
+			Destination: &cfg.devCharPath,
+			EnvVars:     []string{"DEV_CHAR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
+			Value:       "/",
+			Destination: &cfg.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "watch",
+			Usage:       "If set, the command will watch for changes to the driver root and recreate the symlinks when changes are detected.",
+			Value:       false,
+			Destination: &cfg.watch,
+			EnvVars:     []string{"WATCH"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-all",
+			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
+			Destination: &cfg.createAll,
+			EnvVars:     []string{"CREATE_ALL"},
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "If set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &cfg.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	if cfg.createAll && cfg.watch {
+		return fmt.Errorf("create-all and watch are mutually exclusive")
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	var watcher *fsnotify.Watcher
+	var sigs chan os.Signal
+
+	if cfg.watch {
+		watcher, err := newFSWatcher(filepath.Join(cfg.driverRoot, "dev"))
+		if err != nil {
+			return fmt.Errorf("failed to create FS watcher: %v", err)
+		}
+		defer watcher.Close()
+
+		sigs = newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	}
+
+	l, err := NewSymlinkCreator(
+		WithLogger(m.logger),
+		WithDevCharPath(cfg.devCharPath),
+		WithDriverRoot(cfg.driverRoot),
+		WithDryRun(cfg.dryRun),
+		WithCreateAll(cfg.createAll),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink creator: %v", err)
+	}
+
+create:
+	err = l.CreateLinks()
+	if err != nil {
+		return fmt.Errorf("failed to create links: %v", err)
+	}
+	if !cfg.watch {
+		return nil
+	}
+	for {
+		select {
+
+		case event := <-watcher.Events:
+			deviceNode := filepath.Base(event.Name)
+			if !strings.HasPrefix(deviceNode, "nvidia") {
+				continue
+			}
+			if event.Op&fsnotify.Create == fsnotify.Create {
+				m.logger.Infof("%s created, restarting.", event.Name)
+				goto create
+			}
+			if event.Op&fsnotify.Create == fsnotify.Remove {
+				m.logger.Infof("%s removed. Ignoring", event.Name)
+
+			}
+
+		// Watch for any other fs errors and log them.
+		case err := <-watcher.Errors:
+			m.logger.Errorf("inotify: %s", err)
+
+		// React to signals
+		case s := <-sigs:
+			switch s {
+			case syscall.SIGHUP:
+				m.logger.Infof("Received SIGHUP, recreating symlinks.")
+				goto create
+			default:
+				m.logger.Infof("Received signal %q, shutting down.", s)
+				return nil
+			}
+		}
+	}
+}
+
+type linkCreator struct {
+	logger      *logrus.Logger
+	lister      nodeLister
+	driverRoot  string
+	devCharPath string
+	dryRun      bool
+	createAll   bool
+}
+
+// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
+type Creator interface {
+	CreateLinks() error
+}
+
+// Option is a functional option for configuring the linkCreator.
+type Option func(*linkCreator)
+
+// NewSymlinkCreator creates a new linkCreator.
+func NewSymlinkCreator(opts ...Option) (Creator, error) {
+	c := linkCreator{}
+	for _, opt := range opts {
+		opt(&c)
+	}
+	if c.logger == nil {
+		c.logger = logrus.StandardLogger()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+	if c.devCharPath == "" {
+		c.devCharPath = defaultDevCharPath
+	}
+
+	if c.createAll {
+		lister, err := newAllPossible(c.logger, c.driverRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
+		}
+		c.lister = lister
+	} else {
+		c.lister = existing{c.logger, c.driverRoot}
+	}
+	return c, nil
+}
+
+// WithDriverRoot sets the driver root path.
+func WithDriverRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.driverRoot = root
+	}
+}
+
+// WithDevCharPath sets the path at which the symlinks will be created.
+func WithDevCharPath(path string) Option {
+	return func(c *linkCreator) {
+		c.devCharPath = path
+	}
+}
+
+// WithDryRun sets the dry run flag.
+func WithDryRun(dryRun bool) Option {
+	return func(c *linkCreator) {
+		c.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger.
+func WithLogger(logger *logrus.Logger) Option {
+	return func(c *linkCreator) {
+		c.logger = logger
+	}
+}
+
+// WithCreateAll sets the createAll flag for the linkCreator.
+func WithCreateAll(createAll bool) Option {
+	return func(lc *linkCreator) {
+		lc.createAll = createAll
+	}
+}
+
+// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
+func (m linkCreator) CreateLinks() error {
+	deviceNodes, err := m.lister.DeviceNodes()
+	if err != nil {
+		return fmt.Errorf("failed to get device nodes: %v", err)
+	}
+
+	if len(deviceNodes) != 0 && !m.dryRun {
+		err := os.MkdirAll(m.devCharPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
+		}
+	}
+
+	for _, deviceNode := range deviceNodes {
+		target := deviceNode.path
+		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
+
+		m.logger.Infof("Creating link %s => %s", linkPath, target)
+		if m.dryRun {
+			continue
+		}
+
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warnf("Could not create symlink: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type deviceNode struct {
+	path  string
+	major uint32
+	minor uint32
+}
+
+func (d deviceNode) devCharName() string {
+	return fmt.Sprintf("%d:%d", d.major, d.minor)
+}
+
+func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, f := range files {
+		err = watcher.Add(f)
+		if err != nil {
+			watcher.Close()
+			return nil, err
+		}
+	}
+
+	return watcher, nil
+}
+
+func newOSWatcher(sigs ...os.Signal) chan os.Signal {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, sigs...)
+
+	return sigChan
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -0,0 +1,95 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+type nodeLister interface {
+	DeviceNodes() ([]deviceNode, error)
+}
+
+type existing struct {
+	logger     *logrus.Logger
+	driverRoot string
+}
+
+// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
+// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
+func (m existing) DeviceNodes() ([]deviceNode, error) {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(m.driverRoot),
+		lookup.WithOptional(true),
+	)
+
+	devices, err := locator.Locate("/dev/nvidia*")
+	if err != nil {
+		m.logger.Warnf("Error while locating device: %v", err)
+	}
+
+	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
+	if err != nil {
+		m.logger.Warnf("Error while locating caps device: %v", err)
+	}
+
+	if len(devices) == 0 && len(capDevices) == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		return nil, nil
+	}
+
+	var deviceNodes []deviceNode
+	for _, d := range append(devices, capDevices...) {
+		if m.nodeIsBlocked(d) {
+			continue
+		}
+
+		var stat unix.Stat_t
+		err := unix.Stat(d, &stat)
+		if err != nil {
+			m.logger.Warnf("Could not stat device: %v", err)
+			continue
+		}
+		deviceNode := deviceNode{
+			path:  d,
+			major: unix.Major(uint64(stat.Rdev)),
+			minor: unix.Minor(uint64(stat.Rdev)),
+		}
+
+		deviceNodes = append(deviceNodes, deviceNode)
+	}
+
+	return deviceNodes, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m existing) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,107 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type options struct {
+	driverRoot string
+
+	dryRun bool
+
+	control bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device ndoes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
+			Value:       "/",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	s, err := system.New(
+		system.WithLogger(m.logger),
+		system.WithDryRun(opts.dryRun),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create library: %v", err)
+	}
+
+	if opts.control {
+		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
+		if err := s.CreateNVIDIAControlDeviceNodesAt(opts.driverRoot); err != nil {
+			return fmt.Errorf("failed to create control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -0,0 +1,51 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m command) build() *cli.Command {
+	// Create the 'system' command
+	system := cli.Command{
+		Name:  "system",
+		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	system.Subcommands = []*cli.Command{
+		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
+	}
+
+	return &system
+}

config/config.toml.centos

@@ -1,19 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false

config/config.toml.debian

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.opensuse-leap

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.rpm-yum

@@ -16,4 +16,17 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu

@@ -16,7 +16,7 @@ ldconfig = "@/sbin/ldconfig.real"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-#experimental = false
+log-level = "info"
 
 # Specify the runtimes to consider. This list is processed in order and the PATH
 # searched for matching executables unless the entry is an absolute path.
@@ -24,3 +24,9 @@ runtimes = [
     "docker-runc",
     "runc",
 ]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

docker/Dockerfile.amazonlinux

@@ -1,70 +0,0 @@
-ARG BASEIMAGE
-FROM ${BASEIMAGE}
-
-RUN yum install -y \
-        ca-certificates \
-        gcc \
-        wget \
-        git \
-        rpm-build \
-        make && \
-    rm -rf /var/cache/yum/*
-
-ARG GOLANG_VERSION=0.0.0
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture"; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
-
-# packaging
-ARG PKG_VERS
-ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
-
-# output directory
-ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
-RUN mkdir -p $DIST_DIR /dist
-
-# nvidia-container-toolkit
-WORKDIR $GOPATH/src/nvidia-container-toolkit
-COPY . .
-
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
-RUN make PREFIX=${DIST_DIR} cmds
-
-ARG CONFIG_TOML_SUFFIX
-ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
-COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
-
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-# This might not be useful on Amazon Linux, but it's simpler to keep the RHEL
-# and Amazon Linux packages identical.
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
-WORKDIR $DIST_DIR/..
-COPY packaging/rpm .
-
-CMD arch=$(uname -m) && \
-    rpmbuild --clean --target=$arch -bb \
-             -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
-             SPECS/nvidia-container-toolkit.spec && \
-    mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -32,6 +32,7 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -64,12 +65,17 @@ RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
-    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
-    dch --changelog debian/changelog -r "" && \
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
+RUN dch --create --package="${PKG_NAME}" \
+        --newversion "${REVISION}" \
+            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER1_VERSION}" && \
+    dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
-    mv /tmp/nvidia-container-toolkit_*.deb /dist
+    mv /tmp/*.deb /dist

docker/Dockerfile.devel

@@ -14,7 +14,8 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go install golang.org/x/lint/golint@latest
+RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
 RUN go install github.com/matryer/moq@latest
-RUN go install github.com/gordonklaus/ineffassign@latest
+RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
 RUN go install github.com/client9/misspell/cmd/misspell@latest
+RUN go install github.com/google/go-licenses@latest

docker/Dockerfile.opensuse-leap

@@ -25,11 +25,12 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -56,11 +57,16 @@ COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
+             -D "release_date $(date +'%a %b %d %Y')" \
+             -D "git_commit ${GIT_COMMIT}" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.rpm-yum

@@ -1,3 +1,19 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the dockerfile for building packages on yum-based RPM systems.
+
 ARG BASEIMAGE
 FROM ${BASEIMAGE}
 
@@ -27,11 +43,12 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -58,11 +75,16 @@ COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
-             -D "release $RELEASE" \
+             -D "release_date $(date +'%a %b %d %Y')" \
+             -D "git_commit ${GIT_COMMIT}" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -30,6 +30,7 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -57,12 +58,17 @@ COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
-    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
-    dch --changelog debian/changelog -r "" && \
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
+RUN dch --create --package="${PKG_NAME}" \
+        --newversion "${REVISION}" \
+            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" && \
+    dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -85,37 +85,41 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 --%: docker-build-%
 	@
 
+LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
+LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
+
+LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+
 # private ubuntu target
 --ubuntu%: OS := ubuntu
---ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
---debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
---debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--centos%: CONFIG_TOML_SUFFIX := rpm-yum
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
+--rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--rhel%: CONFIG_TOML_SUFFIX := rpm-yum
 --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
 
+
 # We allow the CONFIG_TOML_SUFFIX to be overridden.
 CONFIG_TOML_SUFFIX ?= $(OS)
 
@@ -128,10 +132,12 @@ docker-build-%:
 	    --progress=plain \
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg PKG_NAME="$(LIB_NAME)" \
+	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
+	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
+	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
 	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
-		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \

go.mod

@@ -1,18 +1,36 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.14
+go 1.18
 
 require (
 	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.11.6-0
-	github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
-	github.com/containers/podman/v4 v4.0.3
-	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
+	github.com/NVIDIA/go-nvml v0.12.0-0
+	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
+	github.com/fsnotify/fsnotify v1.5.4
+	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
 	github.com/pelletier/go-toml v1.9.4
-	github.com/sirupsen/logrus v1.8.1
+	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.7.0
-	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
 	github.com/urfave/cli/v2 v2.3.0
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438
 	golang.org/x/mod v0.5.0
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
+	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
+	sigs.k8s.io/yaml v1.3.0
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.10.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/config/config.go

@@ -45,9 +45,12 @@ var (
 // Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
 // Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
 type Config struct {
-	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+	AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+
+	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
+	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
 }
 
 // GetConfig sets up the config struct. Values are read from a toml file
@@ -61,7 +64,7 @@ func GetConfig() (*Config, error) {
 
 	tomlFile, err := os.Open(configFilePath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open config file %v: %v", configFilePath, err)
+		return getDefaultConfig(), nil
 	}
 	defer tomlFile.Close()
 
@@ -91,6 +94,8 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 		return cfg, nil
 	}
 
+	cfg.AcceptEnvvarUnprivileged = toml.GetDefault("accept-nvidia-visible-devices-envvar-when-unprivileged", cfg.AcceptEnvvarUnprivileged).(bool)
+
 	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
 	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
 	runtimeConfig, err := getRuntimeConfigFrom(toml)
@@ -99,12 +104,19 @@ func getConfigFrom(toml *toml.Tree) (*Config, error) {
 	}
 	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
 
+	runtimeHookConfig, err := getRuntimeHookConfigFrom(toml)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load nvidia-container-runtime-hook config: %v", err)
+	}
+	cfg.NVIDIAContainerRuntimeHookConfig = *runtimeHookConfig
+
 	return cfg, nil
 }
 
 // getDefaultConfig defines the default values for the config
 func getDefaultConfig() *Config {
 	c := Config{
+		AcceptEnvvarUnprivileged:     true,
 		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
 		NVIDIACTKConfig:              *getDefaultCTKConfig(),
 		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),

internal/config/config_test.go

@@ -57,6 +57,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "empty config is default",
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: true,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "",
 				},
@@ -69,6 +70,10 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind:        "nvidia.com/gpu",
+							AnnotationPrefixes: []string{"cdi.k8s.io/"},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{
@@ -79,6 +84,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set inline",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"nvidia-container-cli.root = \"/bar/baz\"",
 				"nvidia-container-runtime.debug = \"/foo/bar\"",
 				"nvidia-container-runtime.experimental = true",
@@ -86,10 +92,13 @@ func TestGetConfig(t *testing.T) {
 				"nvidia-container-runtime.log-level = \"debug\"",
 				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
 				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
+				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
 				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "/bar/baz",
 				},
@@ -102,6 +111,13 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{
@@ -112,6 +128,7 @@ func TestGetConfig(t *testing.T) {
 		{
 			description: "config options set in section",
 			contents: []string{
+				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
 				"[nvidia-container-cli]",
 				"root = \"/bar/baz\"",
 				"[nvidia-container-runtime]",
@@ -121,12 +138,16 @@ func TestGetConfig(t *testing.T) {
 				"log-level = \"debug\"",
 				"runtimes = [\"/some/runtime\",]",
 				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.cdi]",
+				"default-kind = \"example.vendor.com/device\"",
+				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
 				"[nvidia-container-runtime.modes.csv]",
 				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
 				"[nvidia-ctk]",
 				"path = \"/foo/bar/nvidia-ctk\"",
 			},
 			expectedConfig: &Config{
+				AcceptEnvvarUnprivileged: false,
 				NVIDIAContainerCLIConfig: ContainerCLIConfig{
 					Root: "/bar/baz",
 				},
@@ -139,6 +160,13 @@ func TestGetConfig(t *testing.T) {
 						CSV: csvModeConfig{
 							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
 						},
+						CDI: cdiModeConfig{
+							DefaultKind: "example.vendor.com/device",
+							AnnotationPrefixes: []string{
+								"cdi.k8s.io/",
+								"example.vendor.com/",
+							},
+						},
 					},
 				},
 				NVIDIACTKConfig: CTKConfig{

internal/config/engine/api.go

@@ -0,0 +1,25 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package engine
+
+// Interface defines the API for a runtime config updater.
+type Interface interface {
+	DefaultRuntime() string
+	AddRuntime(string, string, bool) error
+	RemoveRuntime(string) error
+	Save(string) (int64, error)
+}

internal/config/engine/containerd/config_v1.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// ConfigV1 represents a version 1 containerd config
+type ConfigV1 Config
+
+var _ engine.Interface = (*ConfigV1)(nil)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := *c.Tree
+
+	config.Set("version", int64(1))
+
+	switch runc := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", "runc"}).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, runc)
+	}
+
+	if config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name}) == nil {
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := (*Config)(c).getRuntimeAnnotations([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
+	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}, path)
+
+	if setAsDefault && c.UseDefaultRuntimeName {
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
+	} else if setAsDefault {
+		// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
+		if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
+			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
+		}
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
+		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c ConfigV1) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *ConfigV1) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	// If the specified runtime was set as the default runtime we need to remove the default runtime too.
+	runtimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}).(string)
+	if !ok || runtimePath == "" {
+		runtimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}).(string)
+	}
+	defaultRuntimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}).(string)
+	if !ok || defaultRuntimePath == "" {
+		defaultRuntimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}).(string)
+	}
+	if runtimePath != "" && defaultRuntimePath != "" && runtimePath == defaultRuntimePath {
+		config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime"})
+	}
+
+	config.DeletePath([]string{"plugins", "cri", "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimeConfigPath := []string{"plugins", "cri", "containerd", "runtimes", name}
+	for i := 0; i < len(runtimeConfigPath); i++ {
+		if runtimes, ok := config.GetPath(runtimeConfigPath[:len(runtimeConfigPath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimeConfigPath[:len(runtimeConfigPath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// Save wrotes the config to a file
+func (c ConfigV1) Save(path string) (int64, error) {
+	return (Config)(c).Save(path)
+}

internal/config/engine/containerd/config_v2.go

@@ -0,0 +1,161 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+// AddRuntime adds a runtime to the containerd config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil || c.Tree == nil {
+		return fmt.Errorf("config is nil")
+	}
+	config := *c.Tree
+
+	config.Set("version", int64(2))
+
+	switch runc := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc"}).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, runc)
+	}
+
+	if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
+	}
+
+	if len(c.ContainerAnnotations) > 0 {
+		annotations, err := c.getRuntimeAnnotations([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"})
+		if err != nil {
+			return err
+		}
+		annotations = append(c.ContainerAnnotations, annotations...)
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
+	}
+
+	config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
+
+	if setAsDefault {
+		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
+	if c == nil || c.Tree == nil {
+		return nil, nil
+	}
+
+	config := *c.Tree
+	if !config.HasPath(path) {
+		return nil, nil
+	}
+	annotationsI, ok := config.GetPath(path).([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("invalid annotations: %v", annotationsI)
+	}
+
+	var annotations []string
+	for _, annotation := range annotationsI {
+		a, ok := annotation.(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid annotation: %v", annotation)
+		}
+		annotations = append(annotations, a)
+	}
+
+	return annotations, nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil || c.Tree == nil {
+		return nil
+	}
+
+	config := *c.Tree
+
+	config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
+	if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+		}
+	}
+
+	runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
+	for i := 0; i < len(runtimePath); i++ {
+		if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimePath[:len(runtimePath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+
+	*c.Tree = config
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	config := c.Tree
+	output, err := config.ToTomlString()
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(output)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), err
+}

internal/config/engine/containerd/containerd.go

@@ -0,0 +1,40 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// Config represents the containerd config
+type Config struct {
+	*toml.Tree
+	RuntimeType           string
+	UseDefaultRuntimeName bool
+	ContainerAnnotations  []string
+}
+
+// New creates a containerd config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}

internal/config/engine/containerd/option.go

@@ -0,0 +1,149 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultRuntimeType = "io.containerd.runc.v2"
+)
+
+type builder struct {
+	path                 string
+	runtimeType          string
+	useLegacyConfig      bool
+	containerAnnotations []string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+// WithRuntimeType sets the runtime type for the config builder
+func WithRuntimeType(runtimeType string) Option {
+	return func(b *builder) {
+		b.runtimeType = runtimeType
+	}
+}
+
+// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
+func WithUseLegacyConfig(useLegacyConfig bool) Option {
+	return func(b *builder) {
+		b.useLegacyConfig = useLegacyConfig
+	}
+}
+
+// WithContainerAnnotations sets the container annotations for the config builder
+func WithContainerAnnotations(containerAnnotations ...string) Option {
+	return func(b *builder) {
+		b.containerAnnotations = containerAnnotations
+	}
+}
+
+func (b *builder) build() (engine.Interface, error) {
+	if b.path == "" {
+		return nil, fmt.Errorf("config path is empty")
+	}
+
+	if b.runtimeType == "" {
+		b.runtimeType = defaultRuntimeType
+	}
+
+	config, err := loadConfig(b.path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config.RuntimeType = b.runtimeType
+	config.UseDefaultRuntimeName = !b.useLegacyConfig
+	config.ContainerAnnotations = b.containerAnnotations
+
+	version, err := config.parseVersion(b.useLegacyConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config version: %v", err)
+	}
+	switch version {
+	case 1:
+		return (*ConfigV1)(config), nil
+	case 2:
+		return config, nil
+	}
+
+	return nil, fmt.Errorf("unsupported config version: %v", version)
+}
+
+// loadConfig loads the containerd config from disk
+func loadConfig(config string) (*Config, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	tomlConfig, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	cfg := Config{
+		Tree: tomlConfig,
+	}
+	return &cfg, nil
+}
+
+// parseVersion returns the version of the config
+func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
+	defaultVersion := 2
+	if useLegacyConfig {
+		defaultVersion = 1
+	}
+
+	switch v := c.Get("version").(type) {
+	case nil:
+		switch len(c.Keys()) {
+		case 0: // No config exists, or the config file is empty, use version inferred from containerd
+			return defaultVersion, nil
+		default: // A config file exists, has content, and no version is set
+			return 1, nil
+		}
+	case int64:
+		return int(v), nil
+	default:
+		return -1, fmt.Errorf("unsupported type for version field: %v", v)
+	}
+}

internal/config/engine/crio/crio.go

@@ -0,0 +1,131 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+	"github.com/pelletier/go-toml"
+)
+
+// Config represents the cri-o config
+type Config toml.Tree
+
+// New creates a cri-o config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}
+
+// AddRuntime adds a new runtime to the crio config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := (toml.Tree)(*c)
+
+	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"crio", "runtime", "runtimes", name}, runc)
+	}
+
+	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_path"}, path)
+	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_type"}, "oci")
+
+	if setAsDefault {
+		config.SetPath([]string{"crio", "runtime", "default_runtime"}, name)
+	}
+
+	*c = (Config)(config)
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the cri-o config
+func (c Config) DefaultRuntime() string {
+	config := (toml.Tree)(c)
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		return runtime
+	}
+	return ""
+}
+
+// RemoveRuntime removes a runtime from the cri-o config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil {
+		return nil
+	}
+
+	config := (toml.Tree)(*c)
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		if runtime == name {
+			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
+		}
+	}
+
+	runtimeClassPath := []string{"crio", "runtime", "runtimes", name}
+	config.DeletePath(runtimeClassPath)
+	for i := 0; i < len(runtimeClassPath); i++ {
+		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
+		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
+			if len(entry.Keys()) != 0 {
+				break
+			}
+			config.DeletePath(remainingPath)
+		}
+	}
+
+	*c = (Config)(config)
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	config := (toml.Tree)(c)
+	output, err := config.ToTomlString()
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(output)
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), err
+}

internal/config/engine/crio/option.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+type builder struct {
+	path string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+func (b *builder) build() (*Config, error) {
+	if b.path == "" {
+		empty := toml.Tree{}
+		return (*Config)(&empty), nil
+	}
+
+	return loadConfig(b.path)
+}
+
+// loadConfig loads the cri-o config from disk
+func loadConfig(config string) (*Config, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return (*Config)(cfg), nil
+}

internal/config/engine/docker/docker.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
+)
+
+const (
+	defaultDockerRuntime = "runc"
+)
+
+// Config defines a docker config file.
+// TODO: This should not be public, but we need to access it from the tests in tools/container/docker
+type Config map[string]interface{}
+
+// New creates a docker config with the specified options
+func New(opts ...Option) (engine.Interface, error) {
+	b := &builder{}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	return b.build()
+}
+
+// AddRuntime adds a new runtime to the docker config
+func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
+	if c == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	config := *c
+
+	// Read the existing runtimes
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	// Add / update the runtime definitions
+	runtimes[name] = map[string]interface{}{
+		"path": path,
+		"args": []string{},
+	}
+
+	config["runtimes"] = runtimes
+
+	if setAsDefault {
+		config["default-runtime"] = name
+	}
+
+	*c = config
+	return nil
+}
+
+// DefaultRuntime returns the default runtime for the docker config
+func (c Config) DefaultRuntime() string {
+	r, ok := c["default-runtime"].(string)
+	if !ok {
+		return ""
+	}
+	return r
+}
+
+// RemoveRuntime removes a runtime from the docker config
+func (c *Config) RemoveRuntime(name string) error {
+	if c == nil {
+		return nil
+	}
+	config := *c
+
+	if _, exists := config["default-runtime"]; exists {
+		defaultRuntime := config["default-runtime"].(string)
+		if defaultRuntime == name {
+			config["default-runtime"] = defaultDockerRuntime
+		}
+	}
+
+	if _, exists := config["runtimes"]; exists {
+		runtimes := config["runtimes"].(map[string]interface{})
+
+		delete(runtimes, name)
+
+		if len(runtimes) == 0 {
+			delete(config, "runtimes")
+		}
+	}
+
+	*c = config
+
+	return nil
+}
+
+// Save writes the config to the specified path
+func (c Config) Save(path string) (int64, error) {
+	output, err := json.MarshalIndent(c, "", "    ")
+	if err != nil {
+		return 0, fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	if len(output) == 0 {
+		err := os.Remove(path)
+		if err != nil {
+			return 0, fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		return 0, nil
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(string(output))
+	if err != nil {
+		return 0, fmt.Errorf("unable to write output: %v", err)
+	}
+
+	return int64(n), nil
+}

internal/config/engine/docker/docker_test.go

@@ -0,0 +1,215 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateConfigDefaultRuntime(t *testing.T) {
+	testCases := []struct {
+		config                     Config
+		runtimeName                string
+		setAsDefault               bool
+		expectedDefaultRuntimeName interface{}
+	}{
+		{
+			setAsDefault:               false,
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			runtimeName:                "NAME",
+			setAsDefault:               true,
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "ALREADY_SET",
+			},
+			runtimeName:                "NAME",
+			setAsDefault:               false,
+			expectedDefaultRuntimeName: "ALREADY_SET",
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "ALREADY_SET",
+			},
+			runtimeName:                "NAME",
+			setAsDefault:               true,
+			expectedDefaultRuntimeName: "NAME",
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			if tc.config == nil {
+				tc.config = make(map[string]interface{})
+			}
+			err := tc.config.AddRuntime(tc.runtimeName, "", tc.setAsDefault)
+			require.NoError(t, err)
+
+			defaultRuntimeName := tc.config["default-runtime"]
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+		})
+	}
+}
+
+func TestUpdateConfigRuntimes(t *testing.T) {
+	testCases := []struct {
+		config         Config
+		runtimes       map[string]string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config: map[string]interface{}{},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+				"runtime2": "/test/runtime/dir/runtime2",
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+					"runtime2": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime2",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "runtime1",
+						"args": []string{},
+					},
+				},
+			},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+				"runtime2": "/test/runtime/dir/runtime2",
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+					"runtime2": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime2",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+				},
+			},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			for runtimeName, runtimePath := range tc.runtimes {
+				err := tc.config.AddRuntime(runtimeName, runtimePath, false)
+				require.NoError(t, err)
+			}
+
+			configContent, err := json.MarshalIndent(tc.config, "", "    ")
+			require.NoError(t, err)
+
+			expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+			require.NoError(t, err)
+
+			require.EqualValues(t, string(expectedContent), string(configContent))
+		})
+
+	}
+}

internal/config/engine/docker/option.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type builder struct {
+	path string
+}
+
+// Option defines a function that can be used to configure the config builder
+type Option func(*builder)
+
+// WithPath sets the path for the config builder
+func WithPath(path string) Option {
+	return func(b *builder) {
+		b.path = path
+	}
+}
+
+func (b *builder) build() (*Config, error) {
+	if b.path == "" {
+		empty := make(Config)
+		return &empty, nil
+	}
+
+	return loadConfig(b.path)
+}
+
+// loadConfig loads the docker config from disk
+func loadConfig(configFilePath string) (*Config, error) {
+	log.Infof("Loading docker config from %v", configFilePath)
+
+	info, err := os.Stat(configFilePath)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(Config)
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return &cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return &cfg, nil
+}

internal/config/hook.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"fmt"
+
+	"github.com/pelletier/go-toml"
+)
+
+// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeHookConfig struct {
+	// SkipModeDetection disables the mode check for the runtime hook.
+	SkipModeDetection bool `toml:"skip-mode-detection"`
+}
+
+// dummyHookConfig allows us to unmarshal only a RuntimeHookConfig from a *toml.Tree
+type dummyHookConfig struct {
+	RuntimeHook RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
+}
+
+// getRuntimeHookConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getRuntimeHookConfigFrom(toml *toml.Tree) (*RuntimeHookConfig, error) {
+	cfg := GetDefaultRuntimeHookConfig()
+
+	if toml == nil {
+		return cfg, nil
+	}
+
+	d := dummyHookConfig{
+		RuntimeHook: *cfg,
+	}
+
+	if err := toml.Unmarshal(&d); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
+	}
+
+	return &d.RuntimeHook, nil
+}
+
+// GetDefaultRuntimeHookConfig defines the default values for the config
+func GetDefaultRuntimeHookConfig() *RuntimeHookConfig {
+	c := RuntimeHookConfig{
+		SkipModeDetection: false,
+	}
+
+	return &c
+}

internal/config/image/capabilities.go

@@ -0,0 +1,54 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+// DriverCapability represents the possible values of NVIDIA_DRIVER_CAPABILITIES
+type DriverCapability string
+
+// Constants for the supported driver capabilities
+const (
+	DriverCapabilityAll      DriverCapability = "all"
+	DriverCapabilityCompat32 DriverCapability = "compat32"
+	DriverCapabilityCompute  DriverCapability = "compute"
+	DriverCapabilityDisplay  DriverCapability = "display"
+	DriverCapabilityGraphics DriverCapability = "graphics"
+	DriverCapabilityNgx      DriverCapability = "ngx"
+	DriverCapabilityUtility  DriverCapability = "utility"
+	DriverCapabilityVideo    DriverCapability = "video"
+)
+
+// DriverCapabilities represents the NVIDIA_DRIVER_CAPABILITIES set for the specified image.
+type DriverCapabilities map[DriverCapability]bool
+
+// Has check whether the specified capability is selected.
+func (c DriverCapabilities) Has(capability DriverCapability) bool {
+	if c[DriverCapabilityAll] {
+		return true
+	}
+	return c[capability]
+}
+
+// Any checks whether any of the specified capabilites are set
+func (c DriverCapabilities) Any(capabilities ...DriverCapability) bool {
+	for _, cap := range capabilities {
+		if c.Has(cap) {
+			return true
+		}
+	}
+
+	return false
+}

internal/config/image/cuda_image.go

@@ -26,10 +26,12 @@ import (
 )
 
 const (
-	envCUDAVersion      = "CUDA_VERSION"
-	envNVRequirePrefix  = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA    = envNVRequirePrefix + "CUDA"
-	envNVDisableRequire = "NVIDIA_DISABLE_REQUIRE"
+	envCUDAVersion          = "CUDA_VERSION"
+	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
+	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
+	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
 // CUDA represents a CUDA image that can be used for GPU computing. This wraps
@@ -84,7 +86,7 @@ func (i CUDA) GetRequirements() ([]string, error) {
 	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
 	var requirements []string
 	for name, value := range i {
-		if strings.HasPrefix(name, envNVRequirePrefix) {
+		if strings.HasPrefix(name, envNVRequirePrefix) && !strings.HasPrefix(name, envNVRequireJetpack) {
 			requirements = append(requirements, value)
 		}
 	}
@@ -111,6 +113,51 @@ func (i CUDA) HasDisableRequire() bool {
 	return false
 }
 
+// DevicesFromEnvvars returns the devices requested by the image through environment variables
+func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+	// We concantenate all the devices from the specified envvars.
+	var isSet bool
+	var devices []string
+	requested := make(map[string]bool)
+	for _, envVar := range envVars {
+		if devs, ok := i[envVar]; ok {
+			isSet = true
+			for _, d := range strings.Split(devs, ",") {
+				trimmed := strings.TrimSpace(d)
+				if len(trimmed) == 0 {
+					continue
+				}
+				devices = append(devices, trimmed)
+				requested[trimmed] = true
+			}
+		}
+	}
+
+	// Environment variable unset with legacy image: default to "all".
+	if !isSet && len(devices) == 0 && i.IsLegacy() {
+		return NewVisibleDevices("all")
+	}
+
+	// Environment variable unset or empty or "void": return nil
+	if len(devices) == 0 || requested["void"] {
+		return NewVisibleDevices("void")
+	}
+
+	return NewVisibleDevices(devices...)
+}
+
+// GetDriverCapabilities returns the requested driver capabilities.
+func (i CUDA) GetDriverCapabilities() DriverCapabilities {
+	env := i[envNVDriverCapabilities]
+
+	capabilites := make(DriverCapabilities)
+	for _, c := range strings.Split(env, ",") {
+		capabilites[DriverCapability(c)] = true
+	}
+
+	return capabilites
+}
+
 func (i CUDA) legacyVersion() (string, error) {
 	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
 	if err != nil {

internal/config/image/cuda_image_test.go

@@ -69,3 +69,55 @@ func TestParseMajorMinorVersionInvalid(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRequirements(t *testing.T) {
+	testCases := []struct {
+		description  string
+		env          []string
+		requirements []string
+	}{
+		{
+			description:  "NVIDIA_REQUIRE_JETPACK is ignored",
+			env:          []string{"NVIDIA_REQUIRE_JETPACK=csv-mounts=all"},
+			requirements: nil,
+		},
+		{
+			description:  "NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS is ignored",
+			env:          []string{"NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS=base-only"},
+			requirements: nil,
+		},
+		{
+			description:  "single requirement set",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6"},
+			requirements: []string{"cuda>=11.6"},
+		},
+		{
+			description:  "requirements are concatenated requirement set",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
+			requirements: []string{"cuda>=11.6", "brand=tesla"},
+		},
+		{
+			description:  "legacy image",
+			env:          []string{"CUDA_VERSION=11.6"},
+			requirements: []string{"cuda>=11.6"},
+		},
+		{
+			description:  "legacy image with additional requirement",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
+			requirements: []string{"cuda>=11.6", "brand=tesla"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			image, err := NewCUDAImageFromEnv(tc.env)
+			require.NoError(t, err)
+
+			requirements, err := image.GetRequirements()
+			require.NoError(t, err)
+			require.ElementsMatch(t, tc.requirements, requirements)
+
+		})
+
+	}
+}

internal/config/image/devices.go

@@ -0,0 +1,127 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"strings"
+)
+
+// VisibleDevices represents the devices selected in a container image
+// through the NVIDIA_VISIBLE_DEVICES or other environment variables
+type VisibleDevices interface {
+	List() []string
+	Has(string) bool
+}
+
+var _ VisibleDevices = (*all)(nil)
+var _ VisibleDevices = (*none)(nil)
+var _ VisibleDevices = (*void)(nil)
+var _ VisibleDevices = (*devices)(nil)
+
+// NewVisibleDevices creates a VisibleDevices based on the value of the specified envvar.
+func NewVisibleDevices(envvars ...string) VisibleDevices {
+	for _, envvar := range envvars {
+		if envvar == "all" {
+			return all{}
+		}
+		if envvar == "none" {
+			return none{}
+		}
+		if envvar == "" || envvar == "void" {
+			return void{}
+		}
+	}
+
+	return newDevices(envvars...)
+}
+
+type all struct{}
+
+// List returns ["all"] for all devices
+func (a all) List() []string {
+	return []string{"all"}
+}
+
+// Has for all devices is true for any id except the empty ID
+func (a all) Has(id string) bool {
+	return id != ""
+}
+
+type none struct{}
+
+// List returns [""] for the none devices
+func (n none) List() []string {
+	return []string{""}
+}
+
+// Has for none devices is false for any id
+func (n none) Has(id string) bool {
+	return false
+}
+
+type void struct {
+	none
+}
+
+// List returns nil for the void devices
+func (v void) List() []string {
+	return nil
+}
+
+type devices struct {
+	len    int
+	lookup map[string]int
+}
+
+func newDevices(idOrCommaSeparated ...string) devices {
+	lookup := make(map[string]int)
+
+	i := 0
+	for _, commaSeparated := range idOrCommaSeparated {
+		for _, id := range strings.Split(commaSeparated, ",") {
+			lookup[id] = i
+			i++
+		}
+	}
+
+	d := devices{
+		len:    i,
+		lookup: lookup,
+	}
+	return d
+}
+
+// List returns the list of requested devices
+func (d devices) List() []string {
+	list := make([]string, d.len)
+
+	for id, i := range d.lookup {
+		list[i] = id
+	}
+
+	return list
+}
+
+// Has checks whether the specified ID is in the set of requested devices
+func (d devices) Has(id string) bool {
+	if id == "" {
+		return false
+	}
+
+	_, exist := d.lookup[id]
+	return exist
+}

internal/config/image/privileged.go

@@ -0,0 +1,43 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged NVIDIA capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

@@ -19,6 +19,7 @@ package config
 import (
 	"fmt"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 )
@@ -44,6 +45,16 @@ type RuntimeConfig struct {
 // modesConfig defines (optional) per-mode configs
 type modesConfig struct {
 	CSV csvModeConfig `toml:"csv"`
+	CDI cdiModeConfig `toml:"cdi"`
+}
+
+type cdiModeConfig struct {
+	// SpecDirs allows for the default spec dirs for CDI to be overridden
+	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
 }
 
 type csvModeConfig struct {
@@ -88,6 +99,12 @@ func GetDefaultRuntimeConfig() *RuntimeConfig {
 			CSV: csvModeConfig{
 				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
 			},
+			CDI: cdiModeConfig{
+				DefaultKind: "nvidia.com/gpu",
+				AnnotationPrefixes: []string{
+					cdi.AnnotationPrefix,
+				},
+			},
 		},
 	}
 

internal/discover/char_devices.go

@@ -0,0 +1,69 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
+func NewCharDeviceDiscoverer(logger *logrus.Logger, devices []string, root string) Discover {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(root),
+	)
+
+	return NewDeviceDiscoverer(logger, locator, root, devices)
+}
+
+// NewDeviceDiscoverer creates a discoverer which locates the specified set of device nodes using the specified locator.
+func NewDeviceDiscoverer(logger *logrus.Logger, locator lookup.Locator, root string, devices []string) Discover {
+	m := NewMounts(logger, locator, root, devices).(*mounts)
+
+	return (*charDevices)(m)
+}
+
+// Mounts returns the discovered mounts for the charDevices.
+// Since this explicitly specifies a device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices.
+// Here the device nodes are first discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	devicesAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range devicesAsMounts {
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
+	}
+
+	return devices, nil
+}

internal/discover/char_devices_test.go

@@ -0,0 +1,83 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCharDevices(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description          string
+		input                *charDevices
+		expectedMounts       []Mount
+		expectedMountsError  error
+		expectedDevicesError error
+		expectedDevices      []Device
+	}{
+		{
+			description: "dev mounts are empty",
+			input: (*charDevices)(
+				&mounts{
+					lookup: &lookup.LocatorMock{
+						LocateFunc: func(string) ([]string, error) {
+							return []string{"located"}, nil
+						},
+					},
+					required: []string{"required"},
+				},
+			),
+			expectedDevices: []Device{{Path: "located", HostPath: "located"}},
+		},
+		{
+			description:          "dev devices returns error for nil lookup",
+			input:                &charDevices{},
+			expectedDevicesError: fmt.Errorf("no lookup defined"),
+		},
+	}
+
+	for _, tc := range testCases {
+		logHook.Reset()
+
+		t.Run(tc.description, func(t *testing.T) {
+			tc.input.logger = logger
+
+			mounts, err := tc.input.Mounts()
+			if tc.expectedMountsError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedMounts, mounts)
+
+			devices, err := tc.input.Devices()
+			if tc.expectedDevicesError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedDevices, devices)
+		})
+	}
+}

internal/discover/csv.go

@@ -24,46 +24,39 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// charDevices is a discover for a list of character devices
-type charDevices mounts
-
-var _ Discover = (*charDevices)(nil)
-
 // NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
 // The constructed discoverer is comprised of a list, with each element in the list being associated with a
 // single CSV files.
-func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
+func NewFromCSVFiles(logger *logrus.Logger, files []string, driverRoot string) (Discover, error) {
 	if len(files) == 0 {
 		logger.Warnf("No CSV files specified")
 		return None{}, nil
 	}
 
-	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
+	symlinkLocator := lookup.NewSymlinkLocator(logger, driverRoot)
 	locators := map[csv.MountSpecType]lookup.Locator{
-		csv.MountSpecDev: lookup.NewCharDeviceLocator(logger, root),
-		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
+		csv.MountSpecDev: lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(driverRoot)),
+		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, driverRoot),
 		// Libraries and symlinks are handled in the same way
 		csv.MountSpecLib: symlinkLocator,
 		csv.MountSpecSym: symlinkLocator,
 	}
 
-	var discoverers []Discover
+	var mountSpecs []*csv.MountSpec
 	for _, filename := range files {
-		d, err := NewFromCSVFile(logger, locators, filename)
+		targets, err := loadCSVFile(logger, filename)
 		if err != nil {
 			logger.Warnf("Skipping CSV file %v: %v", filename, err)
 			continue
 		}
-		discoverers = append(discoverers, d)
+		mountSpecs = append(mountSpecs, targets...)
 	}
 
-	return &list{discoverers: discoverers}, nil
+	return newFromMountSpecs(logger, locators, driverRoot, mountSpecs)
 }
 
-// NewFromCSVFile creates a discoverer for the specified CSV file. A logger is also supplied.
-// The constructed discoverer is comprised of a list, with each element in the list being associated with a particular
-// MountSpecType.
-func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, filename string) (Discover, error) {
+// loadCSVFile loads the specified CSV file and returns the list of mount specs
+func loadCSVFile(logger *logrus.Logger, filename string) ([]*csv.MountSpec, error) {
 	// Create a discoverer for each file-kind combination
 	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
 	if err != nil {
@@ -73,12 +66,12 @@ func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup
 		return nil, fmt.Errorf("CSV file is empty")
 	}
 
-	return newFromMountSpecs(logger, locators, targets)
+	return targets, nil
 }
 
 // newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
 // A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
-func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, targets []*csv.MountSpec) (Discover, error) {
+func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, driverRoot string, targets []*csv.MountSpec) (Discover, error) {
 	if len(targets) == 0 {
 		return &None{}, nil
 	}
@@ -99,41 +92,16 @@ func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]loo
 			return nil, fmt.Errorf("no locator defined for '%v'", t)
 		}
 
-		m := &mounts{
-			logger:   logger,
-			lookup:   locator,
-			required: candidatesByType[t],
-		}
-
+		var m Discover
 		switch t {
 		case csv.MountSpecDev:
-			// For device mount specs, we insert a charDevices into the list of discoverers.
-			discoverers = append(discoverers, (*charDevices)(m))
+			m = NewDeviceDiscoverer(logger, locator, driverRoot, candidatesByType[t])
 		default:
-			discoverers = append(discoverers, m)
+			m = NewMounts(logger, locator, driverRoot, candidatesByType[t])
 		}
+		discoverers = append(discoverers, m)
+
 	}
 
 	return &list{discoverers: discoverers}, nil
 }
-
-// Mounts returns the discovered mounts for the charDevices. Since this explicitly specifies a
-// device list, the mounts are nil.
-func (d *charDevices) Mounts() ([]Mount, error) {
-	return nil, nil
-}
-
-// Devices returns the discovered devices for the charDevices. Here the device nodes are first
-// discovered as mounts and these are converted to devices.
-func (d *charDevices) Devices() ([]Device, error) {
-	devicesAsMounts, err := (*mounts)(d).Mounts()
-	if err != nil {
-		return nil, err
-	}
-	var devices []Device
-	for _, mount := range devicesAsMounts {
-		devices = append(devices, Device(mount))
-	}
-
-	return devices, nil
-}

internal/discover/csv_test.go

@@ -26,63 +26,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestCharDevices(t *testing.T) {
-	logger, logHook := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description          string
-		input                *charDevices
-		expectedMounts       []Mount
-		expectedMountsError  error
-		expectedDevicesError error
-		expectedDevices      []Device
-	}{
-		{
-			description: "dev mounts are empty",
-			input: (*charDevices)(
-				&mounts{
-					lookup: &lookup.LocatorMock{
-						LocateFunc: func(string) ([]string, error) {
-							return []string{"located"}, nil
-						},
-					},
-					required: []string{"required"},
-				},
-			),
-			expectedDevices: []Device{{Path: "located"}},
-		},
-		{
-			description:          "dev devices returns error for nil lookup",
-			input:                &charDevices{},
-			expectedDevicesError: fmt.Errorf("no lookup defined"),
-		},
-	}
-
-	for _, tc := range testCases {
-		logHook.Reset()
-
-		t.Run(tc.description, func(t *testing.T) {
-			tc.input.logger = logger
-
-			mounts, err := tc.input.Mounts()
-			if tc.expectedMountsError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			require.ElementsMatch(t, tc.expectedMounts, mounts)
-
-			devices, err := tc.input.Devices()
-			if tc.expectedDevicesError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			require.ElementsMatch(t, tc.expectedDevices, devices)
-		})
-	}
-}
-
 func TestNewFromMountSpec(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 
@@ -93,6 +36,7 @@ func TestNewFromMountSpec(t *testing.T) {
 
 	testCases := []struct {
 		description        string
+		root               string
 		targets            []*csv.MountSpec
 		expectedError      error
 		expectedDiscoverer Discover
@@ -133,12 +77,50 @@ func TestNewFromMountSpec(t *testing.T) {
 						&mounts{
 							logger:   logger,
 							lookup:   locators["dev"],
+							root:     "/",
 							required: []string{"dev0", "dev1"},
 						},
 					),
 					&mounts{
 						logger:   logger,
 						lookup:   locators["lib"],
+						root:     "/",
+						required: []string{"lib0"},
+					},
+				},
+			},
+		},
+		{
+			description: "sets root",
+			targets: []*csv.MountSpec{
+				{
+					Type: "dev",
+					Path: "dev0",
+				},
+				{
+					Type: "lib",
+					Path: "lib0",
+				},
+				{
+					Type: "dev",
+					Path: "dev1",
+				},
+			},
+			root: "/some/root",
+			expectedDiscoverer: &list{
+				discoverers: []Discover{
+					(*charDevices)(
+						&mounts{
+							logger:   logger,
+							lookup:   locators["dev"],
+							root:     "/some/root",
+							required: []string{"dev0", "dev1"},
+						},
+					),
+					&mounts{
+						logger:   logger,
+						lookup:   locators["lib"],
+						root:     "/some/root",
 						required: []string{"lib0"},
 					},
 				},
@@ -148,7 +130,7 @@ func TestNewFromMountSpec(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			discoverer, err := newFromMountSpecs(logger, locators, tc.targets)
+			discoverer, err := newFromMountSpecs(logger, locators, tc.root, tc.targets)
 			if tc.expectedError != nil {
 				require.Error(t, err)
 				return

internal/discover/discover.go

@@ -18,18 +18,21 @@ package discover
 
 // Config represents the configuration options for discovery
 type Config struct {
-	Root                                    string
-	NVIDIAContainerToolkitCLIExecutablePath string
+	DriverRoot    string
+	NvidiaCTKPath string
 }
 
 // Device represents a discovered character device.
 type Device struct {
-	Path string
+	HostPath string
+	Path     string
 }
 
 // Mount represents a discovered mount.
 type Mount struct {
-	Path string
+	HostPath string
+	Path     string
+	Options  []string
 }
 
 // Hook represents a discovered hook.
@@ -39,8 +42,9 @@ type Hook struct {
 	Args      []string
 }
 
-//go:generate moq -stub -out discover_mock.go . Discover
 // Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+//
+//go:generate moq -stub -out discover_mock.go . Discover
 type Discover interface {
 	Devices() ([]Device, error)
 	Mounts() ([]Mount, error)

internal/discover/filter.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/sirupsen/logrus"
+
+// Filter defines an interface for filtering discovered entities
+type Filter interface {
+	DeviceIsSelected(device Device) bool
+}
+
+// filtered represents a filtered discoverer
+type filtered struct {
+	Discover
+	logger *logrus.Logger
+	filter Filter
+}
+
+// newFilteredDisoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
+func newFilteredDisoverer(logger *logrus.Logger, applyTo Discover, filter Filter) Discover {
+	return filtered{
+		Discover: applyTo,
+		logger:   logger,
+		filter:   filter,
+	}
+}
+
+// Devices returns a filtered list of devices based on the specified filter.
+func (d filtered) Devices() ([]Device, error) {
+	devices, err := d.Discover.Devices()
+	if err != nil {
+		return nil, err
+	}
+
+	if d.filter == nil {
+		return devices, nil
+	}
+
+	var selected []Device
+	for _, device := range devices {
+		if d.filter.DeviceIsSelected(device) {
+			selected = append(selected, device)
+		}
+		d.logger.Debugf("skipping device %v", device)
+	}
+
+	return selected, nil
+}

internal/discover/gds.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type gdsDeviceDiscoverer struct {
+	None
+	logger  *logrus.Logger
+	devices Discover
+	mounts  Discover
+}
+
+// NewGDSDiscoverer creates a discoverer for GPUDirect Storage devices and mounts.
+func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{"/dev/nvidia-fs*"},
+		root,
+	)
+
+	udev := NewMounts(
+		logger,
+		lookup.NewDirectoryLocator(logger, root),
+		root,
+		[]string{"/run/udev"},
+	)
+
+	cufile := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{"/etc/cufile.json"},
+	)
+
+	d := gdsDeviceDiscoverer{
+		logger:  logger,
+		devices: devices,
+		mounts:  Merge(udev, cufile),
+	}
+
+	return &d, nil
+}
+
+// Devices discovers the nvidia-fs device nodes for use with GPUDirect Storage
+func (d *gdsDeviceDiscoverer) Devices() ([]Device, error) {
+	return d.devices.Devices()
+}
+
+// Mounts discovers the required mounts for GPUDirect Storage.
+// If no devices are discovered the discovered mounts are empty
+func (d *gdsDeviceDiscoverer) Mounts() ([]Mount, error) {
+	devices, err := d.Devices()
+	if err != nil || len(devices) == 0 {
+		d.logger.Debugf("No nvidia-fs devices detected; skipping detection of mounts")
+		return nil, nil
+	}
+
+	return d.mounts.Mounts()
+}

internal/discover/graphics.go

@@ -0,0 +1,386 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/cuda"
+	"github.com/sirupsen/logrus"
+)
+
+// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
+func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, cfg *Config) (Discover, error) {
+	driverRoot := cfg.DriverRoot
+
+	mounts, err := NewGraphicsMountsDiscoverer(logger, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
+	}
+
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
+
+	xorg := optionalXorgDiscoverer(logger, driverRoot, cfg.NvidiaCTKPath)
+
+	discover := Merge(
+		Merge(drmDeviceNodes, drmByPathSymlinks),
+		mounts,
+		xorg,
+	)
+
+	return discover, nil
+}
+
+// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
+func NewGraphicsMountsDiscoverer(logger *logrus.Logger, driverRoot string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct library locator: %v", err)
+	}
+	libraries := NewMounts(
+		logger,
+		locator,
+		driverRoot,
+		[]string{
+			"libnvidia-egl-gbm.so",
+		},
+	)
+
+	jsonMounts := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/etc", "/usr/share"),
+		),
+		driverRoot,
+		[]string{
+			"glvnd/egl_vendor.d/10_nvidia.json",
+			"vulkan/icd.d/nvidia_icd.json",
+			"vulkan/implicit_layer.d/nvidia_layers.json",
+			"egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"egl/egl_external_platform.d/10_nvidia_wayland.json",
+		},
+	)
+
+	discover := Merge(
+		libraries,
+		jsonMounts,
+	)
+
+	return discover, nil
+}
+
+type drmDevicesByPath struct {
+	None
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	driverRoot    string
+	devicesFrom   Discover
+}
+
+// newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
+func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover {
+	d := drmDevicesByPath{
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		driverRoot:    cfg.DriverRoot,
+		devicesFrom:   devices,
+	}
+
+	return &d
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d drmDevicesByPath) Hooks() ([]Hook, error) {
+	devices, err := d.devicesFrom.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices for by-path symlinks: %v", err)
+	}
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	links, err := d.getSpecificLinkArgs(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []Hook{hook}, nil
+}
+
+// getSpecificLinkArgs returns the required specic links that need to be created
+func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error) {
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[filepath.Base(d.HostPath)] = true
+	}
+
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.driverRoot),
+	)
+	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
+	if err != nil {
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
+	}
+
+	var links []string
+	for _, c := range candidates {
+		device, err := os.Readlink(c)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", c)
+			continue
+		}
+
+		if selectedDevices[filepath.Base(device)] {
+			d.logger.Debugf("adding device symlink %v -> %v", c, device)
+			links = append(links, fmt.Sprintf("%v::%v", device, c))
+		}
+	}
+
+	return links, nil
+}
+
+// newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
+func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, driverRoot string) (Discover, error) {
+	allDevices := NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+		),
+		driverRoot,
+		[]string{
+			"/dev/dri/card*",
+			"/dev/dri/renderD*",
+		},
+	)
+
+	filter, err := newDRMDeviceFilter(logger, devices, driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
+	}
+
+	// We return a discoverer that applies the DRM device filter created above to all discovered DRM device nodes.
+	d := newFilteredDisoverer(
+		logger,
+		allDevices,
+		filter,
+	)
+
+	return d, err
+}
+
+// newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
+func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, driverRoot string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(driverRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read GPU information: %v", err)
+	}
+
+	var selectedBusIds []string
+	for _, f := range gpuInformationPaths {
+		info, err := proc.ParseGPUInformationFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %v: %v", f, err)
+		}
+		uuid := info[proc.GPUInfoGPUUUID]
+		busID := info[proc.GPUInfoBusLocation]
+		minor := info[proc.GPUInfoDeviceMinor]
+
+		if devices.Has(minor) || devices.Has(uuid) || devices.Has(busID) {
+			selectedBusIds = append(selectedBusIds, busID)
+		}
+	}
+
+	filter := make(selectDeviceByPath)
+	for _, busID := range selectedBusIds {
+		drmDeviceNodes, err := drm.GetDeviceNodesByBusID(busID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", busID, err)
+		}
+		for _, drmDeviceNode := range drmDeviceNodes {
+			filter[filepath.Join(drmDeviceNode)] = true
+		}
+	}
+
+	return filter, nil
+}
+
+type xorgHooks struct {
+	libraries     Discover
+	driverVersion string
+	nvidiaCTKPath string
+}
+
+var _ Discover = (*xorgHooks)(nil)
+
+// optionalXorgDiscoverer creates a discoverer for Xorg libraries.
+// If the creation of the discoverer fails, a None discoverer is returned.
+func optionalXorgDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driverRoot, nvidiaCTKPath)
+	if err != nil {
+		logger.Warnf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
+		return None{}
+	}
+	return xorg
+}
+
+func newXorgDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) (Discover, error) {
+	libCudaPaths, err := cuda.New(
+		cuda.WithLogger(logger),
+		cuda.WithDriverRoot(driverRoot),
+	).Locate(".*.*.*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+	libcudaPath := libCudaPaths[0]
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return nil, fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	libRoot := filepath.Dir(libcudaPath)
+	xorgLibs := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths(libRoot, "/usr/lib/x86_64-linux-gnu"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"nvidia/xorg/nvidia_drv.so",
+			fmt.Sprintf("nvidia/xorg/libglxserver_nvidia.so.%s", version),
+		},
+	)
+	xorgHooks := xorgHooks{
+		libraries:     xorgLibs,
+		driverVersion: version,
+		nvidiaCTKPath: FindNvidiaCTK(logger, nvidiaCTKPath),
+	}
+
+	xorgConfg := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/usr/share"),
+		),
+		driverRoot,
+		[]string{"X11/xorg.conf.d/10-nvidia.conf"},
+	)
+
+	d := Merge(
+		xorgLibs,
+		xorgConfg,
+		xorgHooks,
+	)
+
+	return d, nil
+}
+
+// Devices returns no devices for Xorg
+func (m xorgHooks) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a hook to create symlinks for Xorg libraries
+func (m xorgHooks) Hooks() ([]Hook, error) {
+	mounts, err := m.libraries.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get mounts: %v", err)
+	}
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+
+	var target string
+	for _, mount := range mounts {
+		filename := filepath.Base(mount.HostPath)
+		if filename == "libglxserver_nvidia.so."+m.driverVersion {
+			target = mount.Path
+		}
+	}
+
+	if target == "" {
+		return nil, nil
+	}
+
+	link := strings.TrimSuffix(target, "."+m.driverVersion)
+	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
+	symlinkHook := CreateCreateSymlinkHook(
+		m.nvidiaCTKPath,
+		links,
+	)
+
+	return symlinkHook.Hooks()
+}
+
+// Mounts returns the libraries required for Xorg
+func (m xorgHooks) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// selectDeviceByPath is a filter that allows devices to be selected by the path
+type selectDeviceByPath map[string]bool
+
+var _ Filter = (*selectDeviceByPath)(nil)
+
+// DeviceIsSelected determines whether the device's path has been selected
+func (s selectDeviceByPath) DeviceIsSelected(device Device) bool {
+	return s[device.Path]
+}
+
+// MountIsSelected is always true
+func (s selectDeviceByPath) MountIsSelected(Mount) bool {
+	return true
+}
+
+// HookIsSelected is always true
+func (s selectDeviceByPath) HookIsSelected(Hook) bool {
+	return true
+}

internal/discover/hooks.go

@@ -0,0 +1,103 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+)
+
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+// CreateCreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateCreateSymlinkHook(nvidiaCTKPath string, links []string) Discover {
+	if len(links) == 0 {
+		return None{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateNvidiaCTKHook(
+		nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+}
+
+// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
+func CreateNvidiaCTKHook(nvidiaCTKPath string, hookName string, additionalArgs ...string) Hook {
+	return Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      nvidiaCTKPath,
+		Args:      append([]string{filepath.Base(nvidiaCTKPath), "hook", hookName}, additionalArgs...),
+	}
+}
+
+// FindNvidiaCTK locates the nvidia-ctk executable to be used in hooks.
+// If an nvidia-ctk path is specified as an absolute path, it is used directly
+// without checking for existence of an executable at that path.
+func FindNvidiaCTK(logger *logrus.Logger, nvidiaCTKPath string) string {
+	if filepath.IsAbs(nvidiaCTKPath) {
+		logger.Debugf("Using specified NVIDIA Container Toolkit CLI path %v", nvidiaCTKPath)
+		return nvidiaCTKPath
+	}
+
+	if nvidiaCTKPath == "" {
+		nvidiaCTKPath = nvidiaCTKExecutable
+	}
+	logger.Debugf("Locating NVIDIA Container Toolkit CLI as %v", nvidiaCTKPath)
+	lookup := lookup.NewExecutableLocator(logger, "")
+	hookPath := nvidiaCTKDefaultFilePath
+	targets, err := lookup.Locate(nvidiaCTKPath)
+	if err != nil {
+		logger.Warnf("Failed to locate %v: %v", nvidiaCTKPath, err)
+	} else if len(targets) == 0 {
+		logger.Warnf("%v not found", nvidiaCTKPath)
+	} else {
+		logger.Debugf("Found %v candidates: %v", nvidiaCTKPath, targets)
+		hookPath = targets[0]
+	}
+	logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
+
+	return hookPath
+}

internal/discover/icp_test.go

@@ -0,0 +1,60 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIPCMounts(t *testing.T) {
+	l := ipcMounts(
+		mounts{
+			logger: logrus.New(),
+			lookup: &lookup.LocatorMock{
+				LocateFunc: func(path string) ([]string, error) {
+					return []string{"/host/path"}, nil
+				},
+			},
+			required: []string{"target"},
+		},
+	)
+
+	mounts, err := l.Mounts()
+	require.NoError(t, err)
+
+	require.EqualValues(
+		t,
+		[]Mount{
+			{
+				HostPath: "/host/path",
+				Path:     "/host/path",
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+					"noexec",
+				},
+			},
+		},
+		mounts,
+	)
+}

internal/discover/ipc.go

@@ -0,0 +1,78 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type ipcMounts mounts
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger *logrus.Logger, driverRoot string) (Discover, error) {
+	sockets := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithSearchPaths("/run", "/var/run"),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/nvidia-persistenced/socket",
+			"/nvidia-fabricmanager/socket",
+		},
+	)
+
+	mps := newMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(driverRoot),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		[]string{
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	d := Merge(
+		(*ipcMounts)(sockets),
+		(*ipcMounts)(mps),
+	)
+	return d, nil
+}
+
+// Mounts returns the discovered mounts with "noexec" added to the mount options.
+func (d *ipcMounts) Mounts() ([]Mount, error) {
+	mounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+
+	var modifiedMounts []Mount
+	for _, m := range mounts {
+		mount := m
+		mount.Options = append(m.Options, "noexec")
+		modifiedMounts = append(modifiedMounts, mount)
+	}
+
+	return modifiedMounts, nil
+}

internal/discover/ldconfig.go

@@ -19,36 +19,27 @@ package discover
 import (
 	"fmt"
 	"path/filepath"
-	"sort"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/sirupsen/logrus"
 )
 
 // NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
 func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
 	d := ldconfig{
-		logger:                  logger,
-		mountsFrom:              mounts,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		mountsFrom:    mounts,
 	}
 
 	return &d, nil
 }
 
-const (
-	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
-)
-
 type ldconfig struct {
 	None
-	logger                  *logrus.Logger
-	mountsFrom              Discover
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	mountsFrom    Discover
 }
 
 // Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
@@ -57,70 +48,75 @@ func (d ldconfig) Hooks() ([]Hook, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
 	}
-
-	libDirs := getLibDirs(mounts)
-
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{hookPath, "hook", "update-ldcache"}
-	for _, f := range libDirs {
-		args = append(args, "--folder", f)
-	}
-	h := Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
-
+	h := CreateLDCacheUpdateHook(
+		d.nvidiaCTKPath,
+		getLibraryPaths(mounts),
+	)
 	return []Hook{h}, nil
 }
 
-// getLibDirs extracts the library dirs from the specified mounts
-func getLibDirs(mounts []Mount) []string {
-	var paths []string
-	checked := make(map[string]bool)
-
-	for _, m := range mounts {
-		dir := filepath.Dir(m.Path)
-		if dir == "" {
-			continue
-		}
-
-		_, exists := checked[dir]
-		if exists {
-			continue
-		}
-		checked[dir] = isLibName(filepath.Base(m.Path))
-
-		if checked[dir] {
-			paths = append(paths, dir)
-		}
+// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
+func CreateLDCacheUpdateHook(executable string, libraries []string) Hook {
+	var args []string
+	for _, f := range uniqueFolders(libraries) {
+		args = append(args, "--folder", f)
 	}
 
-	sort.Strings(paths)
+	hook := CreateNvidiaCTKHook(
+		executable,
+		"update-ldcache",
+		args...,
+	)
 
+	return hook
+
+}
+
+// getLibraryPaths extracts the library dirs from the specified mounts
+func getLibraryPaths(mounts []Mount) []string {
+	var paths []string
+	for _, m := range mounts {
+		if !isLibName(m.Path) {
+			continue
+		}
+		paths = append(paths, m.Path)
+	}
 	return paths
 }
 
 // isLibName checks if the specified filename is a library (i.e. ends in `.so*`)
 func isLibName(filename string) bool {
-	parts := strings.Split(filename, ".")
 
-	for _, p := range parts {
-		if p == "so" {
-			return true
-		}
+	base := filepath.Base(filename)
+
+	isLib, err := filepath.Match("lib?*.so*", base)
+	if !isLib || err != nil {
+		return false
 	}
 
-	return false
+	parts := strings.Split(base, ".so")
+	if len(parts) == 1 {
+		return true
+	}
+
+	return parts[len(parts)-1] == "" || strings.HasPrefix(parts[len(parts)-1], ".")
+}
+
+// uniqueFolders returns the unique set of folders for the specified files
+func uniqueFolders(libraries []string) []string {
+	var paths []string
+	checked := make(map[string]bool)
+
+	for _, l := range libraries {
+		dir := filepath.Dir(l)
+		if dir == "" {
+			continue
+		}
+		if checked[dir] {
+			continue
+		}
+		checked[dir] = true
+		paths = append(paths, dir)
+	}
+	return paths
 }

internal/discover/ldconfig_test.go

@@ -0,0 +1,168 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testNvidiaCTKPath = "/foo/bar/nvidia-ctk"
+)
+
+func TestLDCacheUpdateHook(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	cfg := Config{
+		DriverRoot:    "/",
+		NvidiaCTKPath: testNvidiaCTKPath,
+	}
+
+	testCases := []struct {
+		description   string
+		mounts        []Mount
+		mountError    error
+		expectedError error
+		expectedArgs  []string
+	}{
+		{
+			description:  "empty mounts",
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache"},
+		},
+		{
+			description:   "mount error",
+			mountError:    fmt.Errorf("mountError"),
+			expectedError: fmt.Errorf("mountError"),
+		},
+		{
+			description: "library folders are added to args",
+			mounts: []Mount{
+				{
+					Path: "/usr/local/lib/libfoo.so",
+				},
+				{
+					Path: "/usr/bin/notlib",
+				},
+				{
+					Path: "/usr/local/libother/libfoo.so",
+				},
+				{
+					Path: "/usr/local/lib/libbar.so",
+				},
+			},
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
+		},
+		{
+			description: "host paths are ignored",
+			mounts: []Mount{
+				{
+					HostPath: "/usr/local/other/libfoo.so",
+					Path:     "/usr/local/lib/libfoo.so",
+				},
+			},
+			expectedArgs: []string{"nvidia-ctk", "hook", "update-ldcache", "--folder", "/usr/local/lib"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mountMock := &DiscoverMock{
+				MountsFunc: func() ([]Mount, error) {
+					return tc.mounts, tc.mountError
+				},
+			}
+			expectedHook := Hook{
+				Path:      testNvidiaCTKPath,
+				Args:      tc.expectedArgs,
+				Lifecycle: "createContainer",
+			}
+
+			d, err := NewLDCacheUpdateHook(logger, mountMock, &cfg)
+			require.NoError(t, err)
+
+			hooks, err := d.Hooks()
+			require.Len(t, mountMock.MountsCalls(), 1)
+			require.Len(t, mountMock.DevicesCalls(), 0)
+			require.Len(t, mountMock.HooksCalls(), 0)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Len(t, hooks, 1)
+
+			require.EqualValues(t, hooks[0], expectedHook)
+
+			devices, err := d.Devices()
+			require.NoError(t, err)
+			require.Empty(t, devices)
+
+			mounts, err := d.Mounts()
+			require.NoError(t, err)
+			require.Empty(t, mounts)
+
+		})
+	}
+
+}
+
+func TestIsLibName(t *testing.T) {
+	testCases := []struct {
+		name  string
+		isLib bool
+	}{
+		{
+			name:  "",
+			isLib: false,
+		},
+		{
+			name:  "lib/not/.so",
+			isLib: false,
+		},
+		{
+			name:  "lib.so",
+			isLib: false,
+		},
+		{
+			name:  "notlibcuda.so",
+			isLib: false,
+		},
+		{
+			name:  "libcuda.so",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.so.1",
+			isLib: true,
+		},
+		{
+			name:  "libcuda.soNOT",
+			isLib: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.isLib, isLibName(tc.name))
+		})
+	}
+}

internal/discover/list.go

@@ -27,8 +27,8 @@ type list struct {
 
 var _ Discover = (*list)(nil)
 
-// NewList creates a discoverer that is the composite of a list of discoveres.
-func NewList(d ...Discover) Discover {
+// Merge creates a discoverer that is the composite of a list of discoveres.
+func Merge(d ...Discover) Discover {
 	l := list{
 		discoverers: d,
 	}

internal/discover/mofed.go

@@ -0,0 +1,35 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+// NewMOFEDDiscoverer creates a discoverer for MOFED devices.
+func NewMOFEDDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{
+			"/dev/infiniband/uverbs*",
+			"/dev/infiniband/rdma_cm",
+		},
+		root,
+	)
+
+	return devices, nil
+}

internal/discover/mounts.go

@@ -18,6 +18,8 @@ package discover
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
@@ -31,6 +33,7 @@ type mounts struct {
 	None
 	logger   *logrus.Logger
 	lookup   lookup.Locator
+	root     string
 	required []string
 	sync.Mutex
 	cache []Mount
@@ -38,6 +41,21 @@ type mounts struct {
 
 var _ Discover = (*mounts)(nil)
 
+// NewMounts creates a discoverer for the required mounts using the specified locator.
+func NewMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) Discover {
+	return newMounts(logger, lookup, root, required)
+}
+
+// newMounts creates a discoverer for the required mounts using the specified locator.
+func newMounts(logger *logrus.Logger, lookup lookup.Locator, root string, required []string) *mounts {
+	return &mounts{
+		logger:   logger,
+		lookup:   lookup,
+		root:     filepath.Join("/", root),
+		required: required,
+	}
+}
+
 func (d *mounts) Mounts() ([]Mount, error) {
 	if d.lookup == nil {
 		return nil, fmt.Errorf("no lookup defined")
@@ -51,7 +69,7 @@ func (d *mounts) Mounts() ([]Mount, error) {
 	d.Lock()
 	defer d.Unlock()
 
-	paths := make(map[string]bool)
+	uniqueMounts := make(map[string]Mount)
 
 	for _, candidate := range d.required {
 		d.logger.Debugf("Locating %v", candidate)
@@ -66,20 +84,45 @@ func (d *mounts) Mounts() ([]Mount, error) {
 		}
 		d.logger.Debugf("Located %v as %v", candidate, located)
 		for _, p := range located {
-			paths[p] = true
+			if _, ok := uniqueMounts[p]; ok {
+				d.logger.Debugf("Skipping duplicate mount %v", p)
+				continue
+			}
+
+			r := d.relativeTo(p)
+			if r == "" {
+				r = p
+			}
+
+			d.logger.Infof("Selecting %v as %v", p, r)
+			uniqueMounts[p] = Mount{
+				HostPath: p,
+				Path:     r,
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+				},
+			}
 		}
 	}
 
 	var mounts []Mount
-	for path := range paths {
-		d.logger.Infof("Selecting %v", path)
-		mount := Mount{
-			Path: path,
-		}
-		mounts = append(mounts, mount)
+	for _, m := range uniqueMounts {
+		mounts = append(mounts, m)
 	}
 
 	d.cache = mounts
 
-	return mounts, nil
+	return d.cache, nil
+}
+
+// relativeTo returns the path relative to the root for the file locator
+func (d *mounts) relativeTo(path string) string {
+	if d.root == "/" {
+		return path
+	}
+
+	return strings.TrimPrefix(path, d.root)
 }

internal/discover/mounts_test.go

@@ -35,6 +35,14 @@ func TestMountsReturnsEmptyDevices(t *testing.T) {
 }
 
 func TestMounts(t *testing.T) {
+
+	mountOptions := []string{
+		"ro",
+		"nosuid",
+		"nodev",
+		"bind",
+	}
+
 	logger, logHook := testlog.NewNullLogger()
 
 	testCases := []struct {
@@ -70,7 +78,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required"},
 			},
-			expectedMounts: []Mount{{Path: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description:   "mounts removes located duplicates",
@@ -83,7 +91,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "located"}},
+			expectedMounts: []Mount{{Path: "located", HostPath: "located", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips located errors",
@@ -98,7 +106,7 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "error", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0"}, {Path: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
 			description: "mounts skips unlocated",
@@ -113,10 +121,10 @@ func TestMounts(t *testing.T) {
 				},
 				required: []string{"required0", "empty", "required1"},
 			},
-			expectedMounts: []Mount{{Path: "required0"}, {Path: "required1"}},
+			expectedMounts: []Mount{{Path: "required0", HostPath: "required0", Options: mountOptions}, {Path: "required1", HostPath: "required1", Options: mountOptions}},
 		},
 		{
-			description: "mounts skips unlocated",
+			description: "mounts adds multiple",
 			input: &mounts{
 				lookup: &lookup.LocatorMock{
 					LocateFunc: func(s string) ([]string, error) {
@@ -129,10 +137,25 @@ func TestMounts(t *testing.T) {
 				required: []string{"required0", "multiple", "required1"},
 			},
 			expectedMounts: []Mount{
-				{Path: "required0"},
-				{Path: "multiple0"},
-				{Path: "multiple1"},
-				{Path: "required1"},
+				{Path: "required0", HostPath: "required0", Options: mountOptions},
+				{Path: "multiple0", HostPath: "multiple0", Options: mountOptions},
+				{Path: "multiple1", HostPath: "multiple1", Options: mountOptions},
+				{Path: "required1", HostPath: "required1", Options: mountOptions},
+			},
+		},
+		{
+			description: "mounts uses relative path",
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(s string) ([]string, error) {
+						return []string{"/some/root/located"}, nil
+					},
+				},
+				root:     "/some/root",
+				required: []string{"required0", "multiple", "required1"},
+			},
+			expectedMounts: []Mount{
+				{Path: "/located", HostPath: "/some/root/located", Options: mountOptions},
 			},
 		},
 	}

internal/discover/symlinks.go

@@ -21,28 +21,24 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/sirupsen/logrus"
 )
 
 type symlinks struct {
 	None
-	logger                  *logrus.Logger
-	lookup                  lookup.Locator
-	nvidiaCTKExecutablePath string
-	csvFiles                []string
-	mountsFrom              Discover
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	csvFiles      []string
+	mountsFrom    Discover
 }
 
 // NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container
 func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Discover, cfg *Config) (Discover, error) {
 	d := symlinks{
-		logger:                  logger,
-		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
-		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
-		csvFiles:                csvFiles,
-		mountsFrom:              mounts,
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		csvFiles:      csvFiles,
+		mountsFrom:    mounts,
 	}
 
 	return &d, nil
@@ -50,19 +46,7 @@ func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, mounts Disc
 
 // Hooks returns a hook to create the symlinks from the required CSV files
 func (d symlinks) Hooks() ([]Hook, error) {
-	hookPath := nvidiaCTKDefaultFilePath
-	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
-	if err != nil {
-		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
-	} else if len(targets) == 0 {
-		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
-	} else {
-		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
-		hookPath = targets[0]
-	}
-	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
-
-	args := []string{hookPath, "hook", "create-symlinks"}
+	var args []string
 	for _, f := range d.csvFiles {
 		args = append(args, "--csv-filename", f)
 	}
@@ -73,13 +57,13 @@ func (d symlinks) Hooks() ([]Hook, error) {
 	}
 	args = append(args, links...)
 
-	h := Hook{
-		Lifecycle: cdi.CreateContainerHook,
-		Path:      hookPath,
-		Args:      args,
-	}
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
 
-	return []Hook{h}, nil
+	return []Hook{hook}, nil
 }
 
 // getSpecificLinkArgs returns the required specic links that need to be created
@@ -117,7 +101,7 @@ func (d symlinks) getSpecificLinkArgs() ([]string, error) {
 		}
 
 		linkPath := filepath.Join(filepath.Dir(m.Path), link)
-		links = append(links, "--link", fmt.Sprintf("%v:%v", target, linkPath))
+		links = append(links, "--link", fmt.Sprintf("%v::%v", target, linkPath))
 		linkProcessed[link] = true
 	}
 

internal/dxcore/api.go

@@ -0,0 +1,64 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+import (
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+const (
+	libraryName      = "libdxcore.so"
+	libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+// dxcore stores a reference the dxcore dynamic library
+var dxcore *context
+
+// Init initializes the dxcore dynamic library
+func Init() error {
+	c, err := initContext()
+	if err != nil {
+		return err
+	}
+	dxcore = c
+	return nil
+}
+
+// Shutdown closes the dxcore dynamic library
+func Shutdown() error {
+	if dxcore != nil && dxcore.initialized != 0 {
+		dxcore.deinitContext()
+	}
+	return nil
+}
+
+// GetDriverStorePaths returns the list of driver store paths
+func GetDriverStorePaths() []string {
+	var paths []string
+	selected := make(map[string]bool)
+
+	for i := 0; i < dxcore.getAdapterCount(); i++ {
+		path := dxcore.getAdapter(i).getDriverStorePath()
+		if selected[path] {
+			continue
+		}
+		selected[path] = true
+		paths = append(paths, path)
+	}
+
+	return paths
+}

internal/dxcore/dxcore.c

@@ -0,0 +1,334 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include "dxcore.h"
+
+// We define log_write as an empty macro to allow dxcore to remain unchanged.
+#define log_write(...)
+
+// We define the following macros to allow dxcore to remain largely unchanged.
+#define log_info(msg) log_write('I', __FILE__, __LINE__, msg)
+#define log_warn(msg) log_write('W', __FILE__, __LINE__, msg)
+#define log_err(msg)  log_write('E', __FILE__, __LINE__, msg)
+#define log_infof(fmt, ...) log_write('I', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_warnf(fmt, ...) log_write('W', __FILE__, __LINE__, fmt, __VA_ARGS__)
+#define log_errf(fmt, ...)  log_write('E', __FILE__, __LINE__, fmt, __VA_ARGS__)
+
+
+#define DXCORE_MAX_PATH 260
+
+/*
+ * List of components we expect to find in the driver store that we need to mount
+ */
+static const char * const dxcore_nvidia_driver_store_components[] = {
+        "libcuda.so.1.1",                   /* Core library for cuda support */
+        "libcuda_loader.so",                /* Core library for cuda support on WSL */
+        "libnvidia-ptxjitcompiler.so.1",    /* Core library for PTX Jit support */
+        "libnvidia-ml.so.1",                /* Core library for nvml */
+        "libnvidia-ml_loader.so",           /* Core library for nvml on WSL */
+        "nvidia-smi",                       /* nvidia-smi binary*/
+        "nvcubins.bin",                     /* Binary containing GPU code for cuda */
+};
+
+
+/*
+ * List of functions and structures we need to communicate with libdxcore.
+ * Documentation on these functions can be found on docs.microsoft.com in d3dkmthk.
+ */
+
+struct dxcore_enumAdapters2;
+struct dxcore_queryAdapterInfo;
+
+typedef int(*pfnDxcoreEnumAdapters2)(struct dxcore_enumAdapters2* pParams);
+typedef int(*pfnDxcoreQueryAdapterInfo)(struct dxcore_queryAdapterInfo* pParams);
+
+struct dxcore_lib {
+        void* hDxcoreLib;
+        pfnDxcoreEnumAdapters2 pDxcoreEnumAdapters2;
+        pfnDxcoreQueryAdapterInfo pDxcoreQueryAdapterInfo;
+};
+
+struct dxcore_adapterInfo
+{
+        unsigned int              hAdapter;
+        struct dxcore_luid        AdapterLuid;
+        unsigned int              NumOfSources;
+        unsigned int              bPresentMoveRegionsPreferred;
+};
+
+struct dxcore_enumAdapters2
+{
+        unsigned int                   NumAdapters;
+        struct dxcore_adapterInfo     *pAdapters;
+};
+
+enum dxcore_kmtqueryAdapterInfoType
+{
+        DXCORE_QUERYDRIVERVERSION = 13,
+        DXCORE_QUERYREGISTRY = 48,
+};
+
+enum dxcore_queryregistry_type {
+        DXCORE_QUERYREGISTRY_DRIVERSTOREPATH = 2,
+        DXCORE_QUERYREGISTRY_DRIVERIMAGEPATH = 3,
+};
+
+enum dxcore_queryregistry_status {
+        DXCORE_QUERYREGISTRY_STATUS_SUCCESS = 0,
+        DXCORE_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW = 1,
+        DXCORE_QUERYREGISTRY_STATUS_FAIL = 2,
+};
+
+struct dxcore_queryregistry_info {
+        enum dxcore_queryregistry_type        QueryType;
+        unsigned int                          QueryFlags;
+        wchar_t                               ValueName[DXCORE_MAX_PATH];
+        unsigned int                          ValueType;
+        unsigned int                          PhysicalAdapterIndex;
+        unsigned int                          OutputValueSize;
+        enum dxcore_queryregistry_status      Status;
+        union {
+                unsigned long long                    OutputQword;
+                wchar_t                               Output;
+        };
+};
+
+struct dxcore_queryAdapterInfo
+{
+        unsigned int                           hAdapter;
+        enum dxcore_kmtqueryAdapterInfoType    Type;
+        void                                   *pPrivateDriverData;
+        unsigned int                           PrivateDriverDataSize;
+};
+
+static int dxcore_query_adapter_info_helper(struct dxcore_lib* pLib,
+                                            unsigned int hAdapter,
+                                            enum dxcore_kmtqueryAdapterInfoType type,
+                                            void* pPrivateDriverDate,
+                                            unsigned int privateDriverDataSize)
+{
+        struct dxcore_queryAdapterInfo queryAdapterInfo = { 0 };
+
+        queryAdapterInfo.hAdapter = hAdapter;
+        queryAdapterInfo.Type = type;
+        queryAdapterInfo.pPrivateDriverData = pPrivateDriverDate;
+        queryAdapterInfo.PrivateDriverDataSize = privateDriverDataSize;
+
+        return pLib->pDxcoreQueryAdapterInfo(&queryAdapterInfo);
+}
+
+static int dxcore_query_adapter_wddm_version(struct dxcore_lib* pLib, unsigned int hAdapter, unsigned int* version)
+{
+        return dxcore_query_adapter_info_helper(pLib,
+                                                hAdapter,
+                                                DXCORE_QUERYDRIVERVERSION,
+                                                (void*)version,
+                                                sizeof(*version));
+}
+
+static int dxcore_query_adapter_driverstore(struct dxcore_lib* pLib, unsigned int hAdapter, char** ppDriverStorePath)
+{
+        struct dxcore_queryregistry_info params = {0};
+        struct dxcore_queryregistry_info* pValue = NULL;
+        wchar_t* pOutput;
+        size_t outputSizeInBytes;
+        size_t outputSize;
+
+        params.QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                             hAdapter,
+                                             DXCORE_QUERYREGISTRY,
+                                             (void*)&params,
+                                             sizeof(params)))
+        {
+                log_err("Failed to query driver store path size for the WDDM Adapter");
+                return (-1);
+        }
+
+        if (params.OutputValueSize > DXCORE_MAX_PATH * sizeof(wchar_t)) {
+                log_err("The driver store path size returned by dxcore is not valid");
+                return (-1);
+        }
+
+        outputSizeInBytes = (size_t)params.OutputValueSize;
+        outputSize = outputSizeInBytes / sizeof(wchar_t);
+
+        pValue = calloc(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes + sizeof(wchar_t), 1);
+        if (!pValue) {
+                log_err("Out of memory while allocating temp buffer to query adapter info");
+                return (-1);
+        }
+
+        pValue->QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH;
+        pValue->OutputValueSize = (unsigned int)outputSizeInBytes;
+
+        if (dxcore_query_adapter_info_helper(pLib,
+                                            hAdapter,
+                                            DXCORE_QUERYREGISTRY,
+                                            (void*)pValue,
+                                            (unsigned int)(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes)))
+        {
+                log_err("Failed to query driver store path data for the WDDM Adapter");
+                free(pValue);
+                return (-1);
+        }
+        pOutput = (wchar_t*)(&pValue->Output);
+
+        // Make sure no matter what happened the wchar_t string is null terminated
+        pOutput[outputSize] = L'\0';
+
+        // Convert the output into a regular c string
+        *ppDriverStorePath = (char*)calloc(outputSize + 1, sizeof(char));
+        if (!*ppDriverStorePath) {
+                log_err("Out of memory while allocating the buffer for the driver store path");
+                free(pValue);
+                return (-1);
+        }
+        wcstombs(*ppDriverStorePath, pOutput, outputSize);
+
+        free(pValue);
+
+    return 0;
+}
+
+static void dxcore_add_adapter(struct dxcore_context* pCtx, struct dxcore_lib* pLib, struct dxcore_adapterInfo *pAdapterInfo)
+{
+        unsigned int wddmVersion = 0;
+        char* driverStorePath = NULL;
+
+        log_infof("Creating a new WDDM Adapter for hAdapter:%x luid:%llx", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid));
+
+        if (dxcore_query_adapter_wddm_version(pLib, pAdapterInfo->hAdapter, &wddmVersion)) {
+                log_err("Failed to query the WDDM version for the specified adapter. Skipping it.");
+                return;
+        }
+
+        if (wddmVersion < 2700) {
+                log_err("Found a WDDM adapter running a driver with pre-WDDM 2.7 . Skipping it.");
+                return;
+        }
+
+        if (dxcore_query_adapter_driverstore(pLib, pAdapterInfo->hAdapter, &driverStorePath)) {
+                log_err("Failed to query driver store path for the WDDM Adapter . Skipping it.");
+                return;
+        }
+
+        // We got all the info we needed. Adding it to the tracking structure.
+        {
+                struct dxcore_adapter* newList;
+                newList = realloc(pCtx->adapterList, sizeof(struct dxcore_adapter) * (pCtx->adapterCount + 1));
+                if (!newList) {
+                        log_err("Out of memory when trying to add a new WDDM Adapter to the list of valid adapters");
+                        free(driverStorePath);
+                        return;
+                }
+
+                pCtx->adapterList = newList;
+
+                pCtx->adapterList[pCtx->adapterCount].hAdapter = pAdapterInfo->hAdapter;
+                pCtx->adapterList[pCtx->adapterCount].pDriverStorePath = driverStorePath;
+                pCtx->adapterList[pCtx->adapterCount].wddmVersion = wddmVersion;
+                pCtx->adapterCount++;
+        }
+
+        log_infof("Adding new adapter via dxcore hAdapter:%x luid:%llx wddm version:%d", pAdapterInfo->hAdapter, *((unsigned long long*)&pAdapterInfo->AdapterLuid), wddmVersion);
+}
+
+static void dxcore_enum_adapters(struct dxcore_context* pCtx, struct dxcore_lib* pLib)
+{
+        struct dxcore_enumAdapters2 params = {0};
+        unsigned int adapterIndex = 0;
+
+        params.NumAdapters = 0;
+        params.pAdapters = NULL;
+
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        params.pAdapters = malloc(sizeof(struct dxcore_adapterInfo) * params.NumAdapters);
+        if (pLib->pDxcoreEnumAdapters2(&params)) {
+                free(params.pAdapters);
+                log_err("Failed to enumerate adapters via dxcore");
+                return;
+        }
+
+        for (adapterIndex = 0; adapterIndex < params.NumAdapters; adapterIndex++) {
+                dxcore_add_adapter(pCtx, pLib, &params.pAdapters[adapterIndex]);
+        }
+
+        free(params.pAdapters);
+}
+
+int dxcore_init_context(struct dxcore_context* pCtx)
+{
+        struct dxcore_lib lib = {0};
+
+        pCtx->initialized = 0;
+        pCtx->adapterCount = 0;
+        pCtx->adapterList = NULL;
+
+        lib.hDxcoreLib = dlopen("libdxcore.so", RTLD_LAZY);
+        if (!lib.hDxcoreLib) {
+                goto error;
+        }
+
+        lib.pDxcoreEnumAdapters2 = (pfnDxcoreEnumAdapters2)dlsym(lib.hDxcoreLib, "D3DKMTEnumAdapters2");
+        if (!lib.pDxcoreEnumAdapters2) {
+                log_err("dxcore library is present but the symbol D3DKMTEnumAdapters2 is missing");
+                goto error;
+        }
+
+        lib.pDxcoreQueryAdapterInfo = (pfnDxcoreQueryAdapterInfo)dlsym(lib.hDxcoreLib, "D3DKMTQueryAdapterInfo");
+        if (!lib.pDxcoreQueryAdapterInfo) {
+                log_err("dxcore library is present but the symbol D3DKMTQueryAdapterInfo is missing");
+                goto error;
+        }
+
+        dxcore_enum_adapters(pCtx, &lib);
+
+        log_info("dxcore layer initialized successfully");
+        pCtx->initialized = 1;
+
+        dlclose(lib.hDxcoreLib);
+
+        return 0;
+
+error:
+        dxcore_deinit_context(pCtx);
+
+        if (lib.hDxcoreLib)
+                dlclose(lib.hDxcoreLib);
+
+        return (-1);
+}
+
+static void dxcore_deinit_adapter(struct dxcore_adapter* pAdapter)
+{
+        if (!pAdapter)
+            return;
+
+        free(pAdapter->pDriverStorePath);
+}
+
+void dxcore_deinit_context(struct dxcore_context* pCtx)
+{
+        unsigned int adapterIndex = 0;
+
+        if (!pCtx)
+                return;
+
+        for (adapterIndex = 0; adapterIndex < pCtx->adapterCount; adapterIndex++) {
+                dxcore_deinit_adapter(&pCtx->adapterList[adapterIndex]);
+        }
+
+        free(pCtx->adapterList);
+
+        pCtx->initialized = 0;
+}

internal/dxcore/dxcore.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dxcore
+
+/*
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
+#include <dxcore.h>
+*/
+import "C"
+import (
+	"fmt"
+	"unsafe"
+)
+
+type context C.struct_dxcore_context
+type adapter C.struct_dxcore_adapter
+
+// initContext initializes the dxcore context and populates the list of adapters.
+func initContext() (*context, error) {
+	cContext := C.struct_dxcore_context{}
+	if C.dxcore_init_context(&cContext) != 0 {
+		return nil, fmt.Errorf("failed to initialize dxcore context")
+	}
+	c := (*context)(&cContext)
+	return c, nil
+}
+
+// deinitContext deinitializes the dxcore context and frees the list of adapters.
+func (c context) deinitContext() {
+	cContext := C.struct_dxcore_context(c)
+	C.dxcore_deinit_context(&cContext)
+}
+
+func (c context) getAdapterCount() int {
+	return int(c.adapterCount)
+}
+
+func (c context) getAdapter(index int) adapter {
+	arrayPointer := (*[1 << 30]C.struct_dxcore_adapter)(unsafe.Pointer(c.adapterList))
+	return adapter(arrayPointer[index])
+}
+
+func (a adapter) getDriverStorePath() string {
+	return C.GoString(a.pDriverStorePath)
+}