[no-relnote] Refactor driver library discovery

This change aligns the driver file discovery with device discovery
and allows other sources such as nvsandboxutils to be added.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,17 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker:stable
+  image: docker
   services:
-    - name: docker:stable-dind
+    - name: docker:dind
       command: ["--experimental"]
 
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
-  BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
+  - trigger
   - image
   - lint
   - go-checks
@@ -33,52 +33,70 @@ stages:
   - test
   - scan
   - release
+  - sign
+
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# The main or manual job is used to filter out distributions or architectures that are not required on
+# every build.
+.main-or-manual:
+  rules:
+    - !reference [.pipeline-trigger-rules, rules]
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: manual
+
+# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
+trigger-pipeline:
+  stage: trigger
+  script:
+    - echo "starting pipeline"
+  rules:
+    - !reference [.main-or-manual, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: false
+    - when: always
 
 # Define the distribution targets
-.dist-amazonlinux2:
-  variables:
-    DIST: amazonlinux2
-
 .dist-centos7:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: centos7
-    CVE_UPDATES: "cyrus-sasl-lib"
 
 .dist-centos8:
   variables:
     DIST: centos8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-debian10:
-  variables:
-    DIST: debian10
-
-.dist-debian9:
-  variables:
-    DIST: debian9
-
-.dist-opensuse-leap15.1:
-  variables:
-    DIST: opensuse-leap15.1
 
 .dist-ubi8:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubi8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-ubuntu16.04:
-  variables:
-    DIST: ubuntu16.04
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
 
 .dist-ubuntu20.04:
   variables:
     DIST: ubuntu20.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
 
 .dist-packaging:
   variables:
@@ -98,6 +116,8 @@ stages:
     ARCH: arm64
 
 .arch-ppc64le:
+  rules:
+    - !reference [.main-or-manual, rules]
   variables:
     ARCH: ppc64le
 
@@ -125,7 +145,7 @@ stages:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
   script:
-    - make -f build/container/Makefile test-${DIST}
+    - make -f deployments/container/Makefile test-${DIST}
 
 # Define the test targets
 test-packaging:
@@ -138,7 +158,7 @@ test-packaging:
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.3.10
+    - export REGCTL_VERSION=v0.4.5
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
@@ -175,7 +195,7 @@ test-packaging:
 
     # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
     # Target
-    - make -f build/container/Makefile push-${DIST}
+    - make -f deployments/container/Makefile push-${DIST}
 
 # Define a staging release step that pushes an image to an internal "staging" repository
 # This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
@@ -194,6 +214,8 @@ test-packaging:
 .release:external:
   extends:
     - .release
+  variables:
+    FORCE_PUBLISH_IMAGES: "yes"
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -203,13 +225,6 @@ test-packaging:
         OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
 
 # Define the release jobs
-release:staging-centos7:
-  extends:
-    - .release:staging
-    - .dist-centos7
-  needs:
-    - image-centos7
-
 release:staging-ubi8:
   extends:
     - .release:staging
@@ -217,16 +232,6 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
 release:staging-ubuntu20.04:
   extends:
     - .release:staging

.github/copy-pr-bot.yaml

@@ -0,0 +1,3 @@
+# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/#configuration
+
+enabled: true

.github/dependabot.yml

@@ -0,0 +1,120 @@
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+# main branch
+  - package-ecosystem: "gomod"
+    target-branch: main
+    directories:
+    - "/"
+    - "deployments/devel"
+    - "tests"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+    groups:
+      k8sio:
+        patterns:
+        - k8s.io/*
+        exclude-patterns:
+        - k8s.io/klog/*
+
+  - package-ecosystem: "docker"
+    target-branch: main
+    directories:
+    # CUDA image
+    - "/deployments/container"
+    # Golang version
+    - "/deployments/devel"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+
+  - package-ecosystem: "github-actions"
+    target-branch: main
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+
+  # Allow dependabot to update the libnvidia-container submodule.
+  - package-ecosystem: "gitsubmodule"
+    target-branch: main
+    directory: "/"
+    allow:
+    - dependency-name: "third_party/libnvidia-container"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+    - libnvidia-container
+
+# The release branch(es):
+  - package-ecosystem: "gomod"
+    target-branch: release-1.17
+    directories:
+    - "/"
+    # We don't update development or test dependencies on release branches
+    # - "deployments/devel"
+    # - "tests"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    labels:
+    - dependencies
+    - maintenance
+    ignore:
+    # For release branches we only consider patch updates.
+    - dependency-name: "*"
+      update-types:
+      - version-update:semver-major
+      - version-update:semver-minor
+    groups:
+      k8sio:
+        patterns:
+        - k8s.io/*
+        exclude-patterns:
+        - k8s.io/klog/*
+
+  - package-ecosystem: "docker"
+    target-branch: release-1.17
+    directories:
+    # CUDA image
+    - "/deployments/container"
+    # Golang version
+    - "/deployments/devel"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    ignore:
+    # For release branches we only apply patch updates to the golang version.
+    - dependency-name: "*golang*"
+      update-types:
+      - version-update:semver-major
+      - version-update:semver-minor
+    labels:
+    - dependencies
+    - maintenance
+
+  - package-ecosystem: "github-actions"
+    target-branch: release-1.17
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    labels:
+    - dependencies
+    - maintenance
+
+  # Github actions need to be gh-pages branches.
+  - package-ecosystem: "github-actions"
+    target-branch: gh-pages
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies

.github/workflows/ci.yaml

@@ -0,0 +1,53 @@
+# Copyright 2025 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI Pipeline
+
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+      - main
+      - release-*
+
+jobs:
+  code-scanning:
+    uses: ./.github/workflows/code_scanning.yaml
+
+  variables:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Generate Commit Short SHA
+        id: version
+        run: echo "version=$(echo $GITHUB_SHA | cut -c1-8)" >> "$GITHUB_OUTPUT"
+
+  golang:
+    uses: ./.github/workflows/golang.yaml
+
+  image:
+    uses: ./.github/workflows/image.yaml
+    needs: [variables, golang, code-scanning]
+    secrets: inherit
+    with:
+      version: ${{ needs.variables.outputs.version }}
+      build_multi_arch_images: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release-') }}
+
+  e2e-test:
+    needs: [image, variables]
+    secrets: inherit
+    uses: ./.github/workflows/e2e.yaml
+    with:
+      version: ${{ needs.variables.outputs.version }}

.github/workflows/code_scanning.yaml

@@ -0,0 +1,49 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "CodeQL"
+
+on:
+  workflow_call: {}
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - main
+      - release-*
+
+jobs:
+  analyze:
+    name: Analyze Go code with CodeQL
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      security-events: write
+      packages: read
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: go
+        build-mode: manual
+    - shell: bash
+      run: |
+        make build
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:go"

.github/workflows/e2e.yaml

@@ -0,0 +1,98 @@
+# Copyright 2025 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: End-to-end Tests
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      AWS_SSH_KEY:
+        required: true
+      E2E_SSH_USER:
+        required: true
+      SLACK_BOT_TOKEN:
+        required: true
+      SLACK_CHANNEL_ID:
+        required: true
+
+jobs:
+  e2e-tests:
+    runs-on: linux-amd64-cpu4
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Calculate build vars
+        id: vars
+        run: |
+          echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
+          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV
+          GOLANG_VERSION=$(./hack/golang-version.sh)
+          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION := }" >> $GITHUB_ENV
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+
+      - name: Set up Holodeck
+        uses: NVIDIA/holodeck@v0.2.6
+        with:
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
+          holodeck_config: "tests/e2e/infra/aws.yaml"
+
+      - name: Get public dns name
+        id: holodeck_public_dns_name
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
+
+      - name: Run e2e tests
+        env:
+          IMAGE_NAME: ghcr.io/nvidia/container-toolkit
+          VERSION: ${{ inputs.version }}
+          SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          E2E_SSH_USER: ${{ secrets.E2E_SSH_USER }}
+          E2E_SSH_HOST: ${{ steps.holodeck_public_dns_name.outputs.result }}
+          E2E_INSTALL_CTK: "true"
+        run: |
+          e2e_ssh_key=$(mktemp)
+          echo "$SSH_KEY" > "$e2e_ssh_key"
+          chmod 600 "$e2e_ssh_key"
+          export E2E_SSH_KEY="$e2e_ssh_key"
+
+          make -f tests/e2e/Makefile test
+
+      - name: Send Slack alert notification
+        if: ${{ failure() }}
+        uses: slackapi/slack-github-action@v2.0.0
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          payload: |
+            channel: ${{ secrets.SLACK_CHANNEL_ID }}
+            text: |
+              :x: On repository ${{ github.repository }}, the Workflow *${{ github.workflow }}* has failed.
+      
+              Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/golang.yaml

@@ -0,0 +1,83 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Golang
+
+on:
+  workflow_call: {}
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - main
+      - release-*
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout code
+    - name: Get Golang version
+      id: vars
+      run: |
+        GOLANG_VERSION=$(./hack/golang-version.sh)
+        echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION := }" >> $GITHUB_ENV
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GOLANG_VERSION }}
+    - name: Lint
+      uses: golangci/golangci-lint-action@v6
+      with:
+        version: latest
+        args: -v --timeout 5m
+        skip-cache: true
+    - name: Check golang modules
+      run: | 
+        make check-vendor
+        make -C deployments/devel check-modules
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Get Golang version
+        id: vars
+        run: |
+          GOLANG_VERSION=$(./hack/golang-version.sh)
+          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION := }" >> $GITHUB_ENV
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+      - run: make test
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Get Golang version
+        id: vars
+        run: |
+          GOLANG_VERSION=$(./hack/golang-version.sh)
+          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> $GITHUB_ENV
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+      - run: make build

.github/workflows/image.yaml

@@ -0,0 +1,126 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run this workflow on pull requests
+name: image
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      build_multi_arch_images:
+        required: true
+        type: string
+
+jobs:
+  packages:
+    runs-on: linux-amd64-cpu4
+    strategy:
+      matrix:
+        target:
+          - ubuntu18.04-arm64
+          - ubuntu18.04-amd64
+          - ubuntu18.04-ppc64le
+          - centos7-aarch64
+          - centos7-x86_64
+          - centos8-ppc64le
+        ispr:
+          - ${{ github.ref_name != 'main' && !startsWith( github.ref_name, 'release-' ) }}
+        exclude:
+          - ispr: true
+            target: ubuntu18.04-arm64
+          - ispr: true
+            target: ubuntu18.04-ppc64le
+          - ispr: true
+            target: centos7-aarch64
+          - ispr: true
+            target: centos8-ppc64le
+      fail-fast: false
+      
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out code
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:master
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: build ${{ matrix.target }} packages
+        run: |
+          sudo apt-get install -y coreutils build-essential sed git bash make
+          echo "Building packages"
+          ./scripts/build-packages.sh ${{ matrix.target }}
+
+      - name: 'Upload Artifacts'
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: toolkit-container-${{ matrix.target }}-${{ github.run_id }}
+          path: ${{ github.workspace }}/dist/*
+
+  image:
+    runs-on: linux-amd64-cpu4
+    strategy:
+      matrix:
+        dist:
+          - ubuntu20.04
+          - ubi8
+          - packaging
+        ispr:
+          - ${{ github.ref_name != 'main' && !startsWith( github.ref_name, 'release-' ) }}
+        exclude:
+          - ispr: true
+            dist: ubi8
+    needs: packages
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out code
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:master
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Get built packages
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ github.workspace }}/dist/
+          pattern: toolkit-container-*-${{ github.run_id }}
+          merge-multiple: true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image
+        env:
+          IMAGE_NAME: ghcr.io/nvidia/container-toolkit
+          VERSION: ${{ inputs.version }}
+          PUSH_ON_BUILD: "true"
+          BUILD_MULTI_ARCH_IMAGES: ${{ inputs.build_multi_arch_images }}
+        run: |
+          echo "${VERSION}"
+          make -f deployments/container/Makefile build-${{ matrix.dist }}

.github/workflows/release.yaml

@@ -0,0 +1,38 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run this workflow on new tags
+name: Release
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out code
+
+      - name: Prepare Artifacts
+        run: |
+          ./hack/prepare-artifacts.sh ${{ github.ref_name }}
+
+      - name: Create Draft Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          ./hack/create-release.sh ${{ github.ref_name }}

.gitignore

@@ -1,9 +1,13 @@
-dist
+/dist
+/artifacts
 *.swp
 *.swo
 /coverage.out*
-/test/output/
+/tests/output/
 /nvidia-container-runtime
+/nvidia-container-runtime.*
+/nvidia-container-runtime-hook
 /nvidia-container-toolkit
 /nvidia-ctk
 /shared-*
+/release-*

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,68 +15,6 @@
 include:
   - .common-ci.yml
 
-build-dev-image:
-  stage: image
-  script:
-    - apk --no-cache add make bash
-    - make .build-image
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - make .push-build-image
-
-.requires-build-image:
-  image: "${BUILDIMAGE}"
-
-.go-check:
-  extends:
-    - .requires-build-image
-  stage: go-checks
-
-fmt:
-  extends:
-    - .go-check
-  script:
-    - make assert-fmt
-
-vet:
-  extends:
-    - .go-check
-  script:
-    - make vet
-
-lint:
-  extends:
-    - .go-check
-  script:
-    - make lint
-  allow_failure: true
-
-ineffassign:
-  extends:
-    - .go-check
-  script:
-    - make ineffassign
-  allow_failure: true
-
-misspell:
-  extends:
-    - .go-check
-  script:
-    - make misspell
-
-go-build:
-  extends:
-    - .requires-build-image
-  stage: go-build
-  script:
-    - make build
-
-unit-tests:
-  extends:
-    - .requires-build-image
-  stage: unit-tests
-  script:
-    - make coverage
-
 # Define the package build helpers
 .multi-arch-build:
   before_script:
@@ -94,7 +32,7 @@ unit-tests:
     - .multi-arch-build
     - .package-artifacts
   stage: package-build
-  timeout: 2h 30m
+  timeout: 3h
   script:
     - ./scripts/build-packages.sh ${DIST}-${ARCH}
 
@@ -102,25 +40,35 @@ unit-tests:
     name: ${ARTIFACTS_NAME}
     paths:
       - ${ARTIFACTS_ROOT}
+  needs:
+    - job: package-meta-packages
+      artifacts: true
 
 # Define the package build targets
-package-amazonlinux2-aarch64:
+package-meta-packages:
   extends:
-    - .package-build
+    - .package-artifacts
-    - .dist-amazonlinux2
+  stage: package-build
-    - .arch-aarch64
+  variables:
+    SKIP_LIBNVIDIA_CONTAINER: "yes"
+    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
+  parallel:
+    matrix:
+      - PACKAGING: [deb, rpm]
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+  script:
+    - ./scripts/build-packages.sh ${PACKAGING}
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
 
-package-amazonlinux2-x86_64:
+package-centos7-aarch64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-package-centos7-ppc64le:
   extends:
     - .package-build
     - .dist-centos7
-    - .arch-ppc64le
+    - .arch-aarch64
 
 package-centos7-x86_64:
   extends:
@@ -128,54 +76,12 @@ package-centos7-x86_64:
     - .dist-centos7
     - .arch-x86_64
 
-package-centos8-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-aarch64
-
 package-centos8-ppc64le:
   extends:
     - .package-build
     - .dist-centos8
     - .arch-ppc64le
 
-package-centos8-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-x86_64
-
-package-debian10-amd64:
-  extends:
-    - .package-build
-    - .dist-debian10
-    - .arch-amd64
-
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-opensuse-leap15.1-x86_64:
-  extends:
-    - .package-build
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -216,20 +122,11 @@ package-ubuntu18.04-ppc64le:
   before_script:
     - !reference [.buildx-setup, before_script]
 
-    - apk add --no-cache bash make
+    - apk add --no-cache bash make git
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
-    - make -f build/container/Makefile build-${DIST}
+    - make -f deployments/container/Makefile build-${DIST}
-
-image-centos7:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos7
-  needs:
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
 
 image-ubi8:
   extends:
@@ -237,20 +134,9 @@ image-ubi8:
     - .package-artifacts
     - .dist-ubi8
   needs:
-    # Note: The ubi8 image uses the centos8 packages
+    # Note: The ubi8 image uses the centos7 packages
-    - package-centos8-aarch64
+    - package-centos7-aarch64
-    - package-centos8-x86_64
+    - package-centos7-x86_64
-    - package-centos8-ppc64le
-
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
 
 image-ubuntu20.04:
   extends:
@@ -260,7 +146,8 @@ image-ubuntu20.04:
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -269,29 +156,26 @@ image-packaging:
     - .package-artifacts
     - .dist-packaging
   needs:
-    - package-amazonlinux2-aarch64
+    - job: package-ubuntu18.04-amd64
-    - package-amazonlinux2-x86_64
+    - job: package-ubuntu18.04-arm64
-    - package-centos7-ppc64le
+    - job: package-amazonlinux2-aarch64
-    - package-centos7-x86_64
+      optional: true
-    - package-centos8-aarch64
+    - job: package-amazonlinux2-x86_64
-    - package-centos8-ppc64le
+      optional: true
-    - package-centos8-x86_64
+    - job: package-centos7-aarch64
-    - package-debian10-amd64
+      optional: true
-    - package-debian9-amd64
+    - job: package-centos7-x86_64
-    - package-opensuse-leap15.1-x86_64
+      optional: true
-    - package-ubuntu16.04-amd64
+    - job: package-centos8-ppc64le
-    - package-ubuntu16.04-ppc64le
+      optional: true
-    - package-ubuntu18.04-amd64
+    - job: package-debian10-amd64
-    - package-ubuntu18.04-arm64
+      optional: true
-    - package-ubuntu18.04-ppc64le
+    - job: package-opensuse-leap15.1-x86_64
+      optional: true
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
 
 # Define publish test helpers
-.test:toolkit:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "toolkit"
-
 .test:docker:
   extends:
     - .integration
@@ -315,34 +199,6 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu18.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-containerd-ubuntu18.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-crio-ubuntu18.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-docker-ubuntu18.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
 test-toolkit-ubuntu20.04:
   extends:
     - .test:toolkit
@@ -370,4 +226,3 @@ test-docker-ubuntu20.04:
     - .dist-ubuntu20.04
   needs:
     - image-ubuntu20.04
-

.gitmodules

@@ -1,10 +1,4 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
-	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
+	url = https://github.com/NVIDIA/libnvidia-container.git
 	branch = main
-[submodule "third_party/nvidia-container-runtime"]
-	path = third_party/nvidia-container-runtime
-	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
-[submodule "third_party/nvidia-docker"]
-	path = third_party/nvidia-docker
-	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git

.golangci.yml

@@ -0,0 +1,43 @@
+run:
+  timeout: 10m
+
+linters:
+  enable:
+    - contextcheck
+    - gocritic
+    - gofmt
+    - goimports
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - staticcheck
+    - unconvert
+
+linters-settings:
+  goimports:
+    local-prefixes: github.com/NVIDIA/nvidia-container-toolkit
+
+issues:
+  exclude:
+  # The legacy hook relies on spec.Hooks.Prestart, which is deprecated as of the v1.2.0 OCI runtime spec.
+  - "SA1019:(.+).Prestart is deprecated(.+)"
+  # TODO: We should address each of the following integer overflows.
+  - "G115: integer overflow conversion(.+)"
+  exclude-rules:
+  # Exclude the gocritic dupSubExpr issue for cgo files.
+  - path: internal/dxcore/dxcore.go
+    linters:
+    - gocritic
+    text: dupSubExpr
+  # Exclude the checks for usage of returns to config.Delete(Path) in the crio and containerd config packages.
+  - path: pkg/config/engine/
+    linters:
+    - errcheck
+    text: config.Delete
+  # RENDERD refers to the Render Device and not the past tense of render.
+  - path: .*.go
+    linters:
+    - misspell
+    text: "`RENDERD` is a misspelling of `RENDERED`"

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,8 +33,11 @@ variables:
   # On the multi-arch builder we don't need the qemu setup.
   SKIP_QEMU_SETUP: "1"
   # Define the public staging registry
-  STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
+  STAGING_REGISTRY: ghcr.io/nvidia
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
+  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
 .image-pull:
   stage: image-build
@@ -48,8 +51,9 @@ variables:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PUSH_MULTIPLE_TAGS: "false"
   # We delay the job start to allow the public pipeline to generate the required images.
-  when: delayed
+  rules:
-  start_in: 30 minutes
+    - when: delayed
+      start_in: 30 minutes
   timeout: 30 minutes
   retry:
     max: 2
@@ -63,33 +67,23 @@ variables:
       regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
   script:
     - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
-    - make -f build/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
+    - make -f deployments/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
-
-image-centos7:
-  extends:
-    - .image-pull
-    - .dist-centos7
 
 image-ubi8:
   extends:
-    - .image-pull
     - .dist-ubi8
-
-image-ubuntu18.04:
-  extends:
     - .image-pull
-    - .dist-ubuntu18.04
 
 image-ubuntu20.04:
   extends:
-    - .image-pull
     - .dist-ubuntu20.04
+    - .image-pull
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
   extends:
-    - .image-pull
     - .dist-packaging
+    - .image-pull
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -106,10 +100,10 @@ image-packaging:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
+    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
-  except:
+  rules:
-    variables:
+    - if: $SKIP_SCANS != "yes"
-      - $SKIP_SCANS && $SKIP_SCANS == "yes"
+    - when: manual
   before_script:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     # TODO: We should specify the architecture here and scan all architectures
@@ -121,6 +115,7 @@ image-packaging:
     - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
   script:
     - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+    - rm -f "${IMAGE_ARCHIVE}"
   artifacts:
     when: always
     expire_in: 1 week
@@ -132,65 +127,47 @@ image-packaging:
       - policy_evaluation.json
 
 # Define the scan targets
-scan-centos7-amd64:
-  extends:
-    - .scan
-    - .dist-centos7
-    - .platform-amd64
-  needs:
-    - image-centos7
-
-scan-centos7-arm64:
-  extends:
-    - .scan
-    - .dist-centos7
-    - .platform-arm64
-  needs:
-    - image-centos7
-    - scan-centos7-amd64
-
-scan-ubuntu18.04-amd64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-amd64
-  needs:
-    - image-ubuntu18.04
-
 scan-ubuntu20.04-amd64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-amd64
+    - .scan
   needs:
     - image-ubuntu20.04
 
 scan-ubuntu20.04-arm64:
   extends:
-    - .scan
     - .dist-ubuntu20.04
     - .platform-arm64
+    - .scan
   needs:
     - image-ubuntu20.04
     - scan-ubuntu20.04-amd64
 
 scan-ubi8-amd64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-amd64
+    - .scan
   needs:
     - image-ubi8
 
 scan-ubi8-arm64:
   extends:
-    - .scan
     - .dist-ubi8
     - .platform-arm64
+    - .scan
   needs:
     - image-ubi8
     - scan-ubi8-amd64
 
+scan-packaging:
+  extends:
+    - .dist-packaging
+    - .scan
+  needs:
+    - image-packaging
+
 # Define external release helpers
 .release:ngc:
   extends:
@@ -201,12 +178,48 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
-release:staging-ubuntu18.04:
+.release:packages:
-  extends:
+  stage: release
-    - .release:staging
-    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu18.04
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+    ARTIFACTS_DIR: "${CI_PROJECT_DIR}/artifacts"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash git
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+    - rm -rf ${ARTIFACTS_DIR}
+
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
+
+release:archive:
+  extends:
+    - .release:external
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}"
+  script:
+    - apk add --no-cache bash git
+    - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}"
 
 release:staging-ubuntu20.04:
   extends:
@@ -217,22 +230,76 @@ release:staging-ubuntu20.04:
 
 # Define the external release targets
 # Release to NGC
-release:ngc-centos7:
-  extends:
-    - .release:ngc
-    - .dist-centos7
-
-release:ngc-ubuntu18.04:
-  extends:
-    - .release:ngc
-    - .dist-ubuntu18.04
-
 release:ngc-ubuntu20.04:
   extends:
-    - .release:ngc
     - .dist-ubuntu20.04
+    - .release:ngc
 
 release:ngc-ubi8:
   extends:
-    - .release:ngc
     - .dist-ubi8
+    - .release:ngc
+
+release:ngc-packaging:
+  extends:
+    - .dist-packaging
+    - .release:ngc
+
+# Define the external image signing steps for NGC
+# Download the ngc cli binary for use in the sign steps
+.ngccli-setup:
+  before_script:
+    - apt-get update && apt-get install -y curl unzip jq
+    - |
+      if [ -z "${NGCCLI_VERSION}" ]; then
+        NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions"
+        # Extract the latest version from the JSON data using jq
+        export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr')
+      fi
+      echo "NGCCLI_VERSION ${NGCCLI_VERSION}"
+    - curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip
+    - unzip ngccli_linux.zip
+    - chmod u+x ngc-cli/ngc
+
+# .sign forms the base of the deployment jobs which signs images in the CI registry.
+# This is extended with the image name and version to be deployed.
+.sign:ngc:
+  image: ubuntu:latest
+  stage: sign
+  rules:
+    - if: $CI_COMMIT_TAG
+  variables:
+    NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
+    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    IMAGE_TAG: "${CI_COMMIT_TAG}-${DIST}"
+  retry:
+    max: 2
+  before_script:
+    - !reference [.ngccli-setup, before_script]
+    # We ensure that the IMAGE_NAME and IMAGE_TAG is set
+    - 'echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1'
+    - 'echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1'
+  script:
+    - 'echo "Signing the image ${IMAGE_NAME}:${IMAGE_TAG}"'
+    - ngc-cli/ngc registry image publish --source ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${IMAGE_TAG} --public --discoverable --allow-guest --sign --org nvidia
+
+sign:ngc-ubuntu20.04:
+  extends:
+    - .dist-ubuntu20.04
+    - .sign:ngc
+  needs:
+    - release:ngc-ubuntu20.04
+
+sign:ngc-ubi8:
+  extends:
+    - .dist-ubi8
+    - .sign:ngc
+  needs:
+    - release:ngc-ubi8
+
+sign:ngc-packaging:
+  extends:
+    - .dist-packaging
+    - .sign:ngc
+  needs:
+    - release:ngc-packaging

CHANGELOG.md

@@ -1,5 +1,376 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.17.4
+- Disable mounting of compat libs from container by default
+- Add allow-cuda-compat-libs-from-container feature flag
+- Skip graphics modifier in CSV mode
+- Properly pass configSearchPaths to a Driver constructor
+- Add support for containerd version 3 config
+- Add string TOML source
+
+### Changes in libnvidia-container
+- Add no-cntlibs CLI option to nvidia-container-cli
+
+### Changes in the Toolkit Container
+- Bump CUDA base image version to 12.6.3
+
+## v1.17.3
+- Only allow host-relative LDConfig paths by default.
+### Changes in libnvidia-container
+- Create virtual copy of host ldconfig binary before calling fexecve()
+
+## v1.17.2
+- Fixed a bug where legacy images would set imex channels as `all`.
+
+## v1.17.1
+- Fixed a bug where specific symlinks existing in a container image could cause a container to fail to start.
+- Fixed a bug on Tegra-based systems where a container would fail to start.
+- Fixed a bug where the default container runtime config path was not properly set.
+
+### Changes in the Toolkit Container
+- Fallback to using a config file if the current runtime config can not be determined from the command line.
+
+## v1.17.0
+- Promote v1.17.0-rc.2 to v1.17.0
+- Fix bug when using just-in-time CDI spec generation
+- Check for valid paths in create-symlinks hook
+
+## v1.17.0-rc.2
+- Fix bug in locating libcuda.so from ldcache
+- Fix bug in sorting of symlink chain
+- Remove unsupported print-ldcache command
+- Remove csv-filename support from create-symlinks
+
+### Changes in the Toolkit Container
+- Fallback to `crio-status` if `crio status` does not work when configuring the crio runtime
+
+## v1.17.0-rc.1
+- Allow IMEX channels to be requested as volume mounts
+- Fix typo in error message
+- Add disable-imex-channel-creation feature flag
+- Add -z,lazy to LDFLAGS
+- Add imex channels to management CDI spec
+- Add support to fetch current container runtime config from the command line.
+- Add creation of select driver symlinks to CDI spec generation.
+- Remove support for config overrides when configuring runtimes.
+- Skip explicit creation of libnvidia-allocator.so.1 symlink
+- Add vdpau as as a driver library search path.
+- Add support for using libnvsandboxutils to generate CDI specifications.
+
+### Changes in the Toolkit Container
+
+- Allow opt-in features to be selected when deploying the toolkit-container.
+- Bump CUDA base image version to 12.6.2
+- Remove support for config overrides when configuring runtimes.
+
+### Changes in libnvidia-container
+
+- Add no-create-imex-channels command line option.
+
+## v1.16.2
+- Exclude libnvidia-allocator from graphics mounts. This fixes a bug that leaks mounts when a container is started with bi-directional mount propagation.
+- Use empty string for default runtime-config-override. This removes a redundant warning for runtimes (e.g. Docker) where this is not applicable.
+
+### Changes in the Toolkit Container
+- Bump CUDA base image version to 12.6.0
+
+### Changes in libnvidia-container
+- Add no-gsp-firmware command line option
+- Add no-fabricmanager command line option
+- Add no-persistenced command line option
+- Skip directories and symlinks when mounting libraries.
+
+## v1.16.1
+- Fix bug with processing errors during CDI spec generation for MIG devices
+
+## v1.16.0
+- Promote v1.16.0-rc.2 to v1.16.0
+
+### Changes in the Toolkit Container
+- Bump CUDA base image version to 12.5.1
+
+## v1.16.0-rc.2
+- Use relative path to locate driver libraries
+- Add RelativeToRoot function to Driver
+- Inject additional libraries for full X11 functionality
+- Extract options from default runtime if runc does not exist
+- Avoid using map pointers as maps are always passed by reference
+- Reduce logging for the NVIDIA Container runtime
+- Fix bug in argument parsing for logger creation
+
+## v1.16.0-rc.1
+
+- Support vulkan ICD files directly in a driver root. This allows for the discovery of vulkan files in GKE driver installations.
+- Increase priority of ld.so.conf.d config file injected into container. This ensures that injected libraries are preferred over libraries present in the container.
+- Set default CDI spec permissions to 644. This fixes permission issues when using the `nvidia-ctk cdi transform` functions.
+- Add `dev-root` option to `nvidia-ctk system create-device-nodes` command.
+- Fix location of `libnvidia-ml.so.1` when a non-standard driver root is used. This enabled CDI spec generation when using the driver container on a host.
+- Recalculate minimum required CDI spec version on save.
+- Move `nvidia-ctk hook` commands to a separate `nvidia-cdi-hook` binary. The same subcommands are supported.
+- Use `:` as an `nvidia-ctk config --set` list separator. This fixes a bug when trying to set config options that are lists.
+
+- [toolkit-container] Bump CUDA base image version to 12.5.0
+- [toolkit-container] Allow the path to `toolkit.pid` to be specified directly.
+- [toolkit-container] Remove provenance information from image manifests.
+- [toolkit-container] Add `dev-root` option when configuring the toolkit. This adds support for GKE driver installations.
+
+## v1.15.0
+
+* Remove `nvidia-container-runtime` and `nvidia-docker2` packages.
+* Use `XDG_DATA_DIRS` environment variable when locating config files such as graphics config files.
+* Add support for v0.7.0 Container Device Interface (CDI) specification.
+* Add `--config-search-path` option to `nvidia-ctk cdi generate` command. These paths are used when locating driver files such as graphics config files.
+* Use D3DKMTEnumAdapters3 to enumerate adpaters on WSL2 if available.
+* Add support for v1.2.0 OCI Runtime specification.
+* Explicitly set `NVIDIA_VISIBLE_DEVICES=void` in generated CDI specifications. This prevents the NVIDIA Container Runtime from making additional modifications.
+
+* [libnvidia-container] Use D3DKMTEnumAdapters3 to enumerate adpaters on WSL2 if available.
+
+* [toolkit-container] Bump CUDA base image version to 12.4.1
+
+## v1.15.0-rc.4
+* Add a `--spec-dir` option to the `nvidia-ctk cdi generate` command. This allows specs outside of `/etc/cdi` and `/var/run/cdi` to be processed.
+* Add support for extracting device major number from `/proc/devices` if `nvidia` is used as a device name over `nvidia-frontend`.
+* Allow multiple device naming strategies for `nvidia-ctk cdi generate` command. This allows a single
+  CDI spec to be generated that includes GPUs by index and UUID.
+* Set the default `--device-name-strategy` for the `nvidia-ctk cdi generate` command to `[index, uuid]`.
+* Remove `libnvidia-container0` jetpack dependency included for legacy Tegra-based systems.
+* Add `NVIDIA_VISIBLE_DEVICES=void` to generated CDI specifications.
+
+* [toolkit-container] Remove centos7 image. The ubi8 image can be used on all RPM-based platforms.
+* [toolkit-container] Bump CUDA base image version to 12.3.2
+
+## v1.15.0-rc.3
+* Fix bug in `nvidia-ctk hook update-ldcache` where default `--ldconfig-path` value was not applied.
+
+## v1.15.0-rc.2
+* Extend the `runtime.nvidia.com/gpu` CDI kind to support full-GPUs and MIG devices specified by index or UUID.
+* Fix bug when specifying `--dev-root` for Tegra-based systems.
+* Log explicitly requested runtime mode.
+* Remove package dependency on libseccomp.
+* Added detection of libnvdxgdmal.so.1 on WSL2
+* Use devRoot to resolve MIG device nodes.
+* Fix bug in determining default nvidia-container-runtime.user config value on SUSE-based systems.
+* Add `crun` to the list of configured low-level runtimes.
+* Added support for `--ldconfig-path` to `nvidia-ctk cdi generate` command.
+* Fix `nvidia-ctk runtime configure --cdi.enabled` for Docker.
+* Add discovery of the GDRCopy device (`gdrdrv`) if the `NVIDIA_GDRCOPY` environment variable of the container is set to `enabled`
+
+* [toolkit-container] Bump CUDA base image version to 12.3.1.
+
+## v1.15.0-rc.1
+* Skip update of ldcache in containers without ldconfig. The .so.SONAME symlinks are still created.
+* Normalize ldconfig path on use. This automatically adjust the ldconfig setting applied to ldconfig.real on systems where this exists.
+* Include `nvidia/nvoptix.bin` in list of graphics mounts.
+* Include `vulkan/icd.d/nvidia_layers.json` in list of graphics mounts.
+* Add support for `--library-search-paths` to `nvidia-ctk cdi generate` command.
+* Add support for injecting /dev/nvidia-nvswitch* devices if the NVIDIA_NVSWITCH=enabled envvar is specified.
+* Added support for `nvidia-ctk runtime configure --enable-cdi` for the `docker` runtime. Note that this requires Docker >= 25.
+* Fixed bug in `nvidia-ctk config` command when using `--set`. The types of applied config options are now applied correctly.
+* Add `--relative-to` option to `nvidia-ctk transform root` command. This controls whether the root transformation is applied to host or container paths.
+* Added automatic CDI spec generation when the `runtime.nvidia.com/gpu=all` device is requested by a container.
+
+* [libnvidia-container] Fix device permission check when using cgroupv2 (fixes #227)
+
+## v1.14.3
+* [toolkit-container] Bump CUDA base image version to 12.2.2.
+
+## v1.14.2
+* Fix bug on Tegra-based systems where symlinks were not created in containers.
+* Add --csv.ignore-pattern command line option to nvidia-ctk cdi generate command.
+
+## v1.14.1
+* Fixed bug where contents of `/etc/nvidia-container-runtime/config.toml` is ignored by the NVIDIA Container Runtime Hook.
+
+* [libnvidia-container] Use libelf.so on RPM-based systems due to removed mageia repositories hosting pmake and bmake.
+
+## v1.14.0
+* Promote v1.14.0-rc.3 to v1.14.0
+
+## v1.14.0-rc.3
+* Added support for generating OCI hook JSON file to `nvidia-ctk runtime configure` command.
+* Remove installation of OCI hook JSON from RPM package.
+* Refactored config for `nvidia-container-runtime-hook`.
+* Added a `nvidia-ctk config` command which supports setting config options using a `--set` flag.
+* Added `--library-search-path` option to `nvidia-ctk cdi generate` command in `csv` mode. This allows folders where
+  libraries are located to be specified explicitly.
+* Updated go-nvlib to support devices which are not present in the PCI device database. This allows the creation of dev/char symlinks on systems with such devices installed.
+* Added `UsesNVGPUModule` info function for more robust platform detection. This is required on Tegra-based systems where libnvidia-ml.so is also supported.
+
+* [toolkit-container] Set `NVIDIA_VISIBLE_DEVICES=void` to prevent injection of NVIDIA devices and drivers into the NVIDIA Container Toolkit container.
+
+## v1.14.0-rc.2
+* Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
+* Remove dependency on coreutils when installing package on RPM-based systems.
+* Create output folders if required when running `nvidia-ctk runtime configure`
+* Generate default config as post-install step.
+* Added support for detecting GSP firmware at custom paths when generating CDI specifications.
+* Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.
+
+* [libnvidia-container] Include Shared Compiler Library (libnvidia-gpucomp.so) in the list of compute libaries.
+
+* [toolkit-container] Ensure that common envvars have higher priority when configuring the container engines.
+* [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
+
+## v1.14.0-rc.1
+
+* Add support for updating containerd configs to the `nvidia-ctk runtime configure` command.
+* Create file in `etc/ld.so.conf.d` with permissions `644` to support non-root containers.
+* Generate CDI specification files with `644` permissions to allow rootless applications (e.g. podman)
+* Add `nvidia-ctk cdi list` command to show the known CDI devices.
+* Add support for generating merged devices (e.g. `all` device) to the nvcdi API.
+* Use *.* pattern to locate libcuda.so when generating a CDI specification to support platforms where a patch version is not specified.
+* Update go-nvlib to skip devices that are not MIG capable when generating CDI specifications.
+* Add `nvidia-container-runtime-hook.path` config option to specify NVIDIA Container Runtime Hook path explicitly.
+* Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
+* Add option to load kernel modules when creating device nodes
+* Add option to create device nodes when creating `/dev/char` symlinks
+
+* [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
+
+* [toolkit-container] Allow same envars for all runtime configs
+
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
+## v1.12.0-rc.1
+
+* Add support for multiple Docker Swarm resources
+* Improve injection of Vulkan configurations and libraries
+* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
+* [libnvidia-container] Include NVVM compiler library in compute libs
+
+## v1.11.0
+
+* Promote v1.11.0-rc.3 to v1.11.0
+
+## v1.11.0-rc.3
+
+* Build fedora35 packages
+* Introduce an `nvidia-container-toolkit-base` package for better dependency management
+* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
+* Inject platform files into container on Tegra-based systems
+* [toolkit container] Update CUDA base images to 11.7.1
+* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
+
+## v1.11.0-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.0-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
 ## v1.10.0
 
 * Promote v1.10.0-rc.3 to v1.10.0

DEVELOPMENT.md

@@ -19,7 +19,7 @@ where `TARGET` is a make target that is valid for each of the sub-components.
 
 These include:
 * `ubuntu18.04-amd64`
-* `centos8-x86_64`
+* `centos7-x86_64`
 
 If no `TARGET` is specified, all valid release targets are built.
 
@@ -34,7 +34,7 @@ environment variables.
 
 ## Testing packages locally
 
-The [test/release](./test/release/) folder contains documentation on how the installation of local or staged packages can be tested.
+The [tests/release](./tests/release/) folder contains documentation on how the installation of local or staged packages can be tested.
 
 
 ## Releasing

Jenkinsfile

@@ -1,142 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-podTemplate (cloud:'sw-gpu-cloudnative',
-    containers: [
-    containerTemplate(name: 'docker', image: 'docker:dind', ttyEnabled: true, privileged: true),
-    containerTemplate(name: 'golang', image: 'golang:1.16.3', ttyEnabled: true)
-  ]) {
-    node(POD_LABEL) {
-        def scmInfo
-
-        stage('checkout') {
-            scmInfo = checkout(scm)
-        }
-
-        stage('dependencies') {
-            container('golang') {
-                sh 'GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell'
-                sh 'GO111MODULE=off go get -u github.com/gordonklaus/ineffassign'
-                sh 'GO111MODULE=off go get -u golang.org/x/lint/golint'
-            }
-            container('docker') {
-                sh 'apk add --no-cache make bash git'
-            }
-        }
-        stage('check') {
-            parallel (
-                getGolangStages(["assert-fmt", "lint", "vet", "ineffassign", "misspell"])
-            )
-        }
-        stage('test') {
-            parallel (
-                getGolangStages(["test"])
-            )
-        }
-
-        def versionInfo
-        stage('version') {
-            container('docker') {
-                versionInfo = getVersionInfo(scmInfo)
-                println "versionInfo=${versionInfo}"
-            }
-        }
-
-        def dist = 'ubuntu20.04'
-        def arch = 'amd64'
-        def stageLabel = "${dist}-${arch}"
-
-        stage('build-one') {
-            container('docker') {
-                stage (stageLabel) {
-                    sh "make ${dist}-${arch}"
-                }
-            }
-        }
-
-        stage('release') {
-            container('docker') {
-                stage (stageLabel) {
-
-                    def component = 'main'
-                    def repository = 'sw-gpu-cloudnative-debian-local/pool/main/'
-
-                    def uploadSpec = """{
-                                        "files":
-                                        [  {
-                                                "pattern": "./dist/${dist}/${arch}/*.deb",
-                                                "target": "${repository}",
-                                                "props": "deb.distribution=${dist};deb.component=${component};deb.architecture=${arch}"
-                                            }
-                                        ]
-                                    }"""
-
-                    sh "echo starting release with versionInfo=${versionInfo}"
-                    if (versionInfo.isTag) {
-                        // upload to artifactory repository
-                        def server = Artifactory.server 'sw-gpu-artifactory'
-                        server.upload spec: uploadSpec
-                    } else {
-                        sh "echo skipping release for non-tagged build"
-                    }
-                }
-            }
-        }
-    }
-}
-
-def getGolangStages(def targets) {
-    stages = [:]
-
-    for (t in targets) {
-        stages[t] = getLintClosure(t)
-    }
-
-    return stages
-}
-
-def getLintClosure(def target) {
-    return {
-        container('golang') {
-            stage(target) {
-                sh "make ${target}"
-            }
-        }
-    }
-}
-
-// getVersionInfo returns a hash of version info
-def getVersionInfo(def scmInfo) {
-    def versionInfo = [
-        isTag: isTag(scmInfo.GIT_BRANCH)
-    ]
-
-    scmInfo.each { k, v -> versionInfo[k] = v }
-    return versionInfo
-}
-
-def isTag(def branch) {
-    if (!branch.startsWith('v')) {
-        return false
-    }
-
-    def version = shOutput('git describe --all --exact-match --always')
-    return version == "tags/${branch}"
-}
-
-def shOuptut(def script) {
-    return sh(script: script, returnStdout: true).trim()
-}

Makefile

@@ -38,8 +38,8 @@ EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES))
 CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
-CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
+CHECK_TARGETS := lint
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+MAKE_TARGETS := binaries build check fmt test examples cmds coverage generate licenses vendor check-vendor $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
@@ -51,23 +51,28 @@ CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 else
 CLI_VERSION = $(VERSION)
 endif
-
+CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
-GOOS ?= linux
 
 binaries: cmds
 ifneq ($(PREFIX),)
 cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
+
+ifneq ($(shell uname),Darwin)
+EXTLDFLAGS = -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files -Wl,-z,lazy
+else
+EXTLDFLAGS = -Wl,-undefined,dynamic_lookup
+endif
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.gitCommit=$(GIT_COMMIT) -X github.com/NVIDIA/nvidia-container-toolkit/internal/info.version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	go build -ldflags "-s -w '-extldflags=$(EXTLDFLAGS)' -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
-	GOOS=$(GOOS) go build ./...
+	go build ./...
 
 examples: $(EXAMPLE_TARGETS)
 $(EXAMPLE_TARGETS): example-%:
-	GOOS=$(GOOS) go build ./examples/$(*)
+	go build ./examples/$(*)
 
 all: check test build binary
 check: $(CHECK_TARGETS)
@@ -77,34 +82,47 @@ fmt:
 	go list -f '{{.Dir}}' $(MODULE)/... \
 		| xargs gofmt -s -l -w
 
-assert-fmt:
+# Apply goimports -local github.com/NVIDIA/container-toolkit to the codebase
-	go list -f '{{.Dir}}' $(MODULE)/... \
+goimports:
-		| xargs gofmt -s -l > fmt.out
+	go list -f {{.Dir}} $(MODULE)/... \
-	@if [ -s fmt.out ]; then \
+		| xargs goimports -local $(MODULE) -w
-		echo "\nERROR: The following files are not formatted:\n"; \
-		cat fmt.out; \
-		rm fmt.out; \
-		exit 1; \
-	else \
-		rm fmt.out; \
-	fi
-
-ineffassign:
-	ineffassign $(MODULE)/...
 
 lint:
-# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
+	golangci-lint run ./...
-	go list -f '{{.Dir}}' $(MODULE)/... | xargs golint -set_exit_status
 
-misspell:
+vendor:  | mod-tidy mod-vendor mod-verify
-	misspell $(MODULE)/...
 
-vet:
+mod-tidy:
-	go vet $(MODULE)/...
+	@for mod in $$(find . -name go.mod -not -path "./testdata/*" -not -path "./third_party/*"); do \
+	    echo "Tidying $$mod..."; ( \
+	        cd $$(dirname $$mod) && go mod tidy \
+            ) || exit 1; \
+	done
+
+mod-vendor:
+	@for mod in $$(find . -name go.mod -not -path "./testdata/*" -not -path "./third_party/*" -not -path "./deployments/*"); do \
+		echo "Vendoring $$mod..."; ( \
+			cd $$(dirname $$mod) && go mod vendor \
+			) || exit 1; \
+	done
+
+mod-verify:
+	@for mod in $$(find . -name go.mod -not -path "./testdata/*" -not -path "./third_party/*"); do \
+	    echo "Verifying $$mod..."; ( \
+	        cd $$(dirname $$mod) && go mod verify | sed 's/^/  /g' \
+	    ) || exit 1; \
+	done
+
+
+check-vendor: vendor
+	git diff --quiet HEAD -- go.mod go.sum vendor
+
+licenses:
+	go-licenses csv $(MODULE)/...
 
 COVERAGE_FILE := coverage.out
 test: build cmds
-	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...
+	go test -coverprofile=$(COVERAGE_FILE) $(MODULE)/...
 
 coverage: test
 	cat $(COVERAGE_FILE) | grep -v "_mock.go" > $(COVERAGE_FILE).no-mocks
@@ -115,30 +133,24 @@ generate:
 
 # Generate an image for containerized builds
 # Note: This image is local only
-.PHONY: .build-image .pull-build-image .push-build-image
+.PHONY: .build-image
-.build-image: docker/Dockerfile.devel
+.build-image:
-	if [ x"$(SKIP_IMAGE_BUILD)" = x"" ]; then \
+	make -f deployments/devel/Makefile .build-image
-		$(DOCKER) build \
-			--progress=plain \
-			--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-			--tag $(BUILDIMAGE) \
-			-f $(^) \
-			docker; \
-	fi
 
-.pull-build-image:
+ifeq ($(BUILD_DEVEL_IMAGE),yes)
-	$(DOCKER) pull $(BUILDIMAGE)
+$(DOCKER_TARGETS): .build-image
+.shell: .build-image
+endif
 
-.push-build-image:
+$(DOCKER_TARGETS): docker-%:
-	$(DOCKER) push $(BUILDIMAGE)
+	@echo "Running 'make $(*)' in container image $(BUILDIMAGE)"
-
-$(DOCKER_TARGETS): docker-%: .build-image
-	@echo "Running 'make $(*)' in docker container $(BUILDIMAGE)"
 	$(DOCKER) run \
 		--rm \
-		-e GOCACHE=/tmp/.cache \
+		-e GOCACHE=/tmp/.cache/go \
-		-v $(PWD):$(PWD) \
+		-e GOMODCACHE=/tmp/.cache/gomod \
-		-w $(PWD) \
+		-e GOLANGCI_LINT_CACHE=/tmp/.cache/golangci-lint \
+		-v $(PWD):/work \
+		-w /work \
 		--user $$(id -u):$$(id -g) \
 		$(BUILDIMAGE) \
 			make $(*)
@@ -149,8 +161,10 @@ PHONY: .shell
 	$(DOCKER) run \
 		--rm \
 		-ti \
-		-e GOCACHE=/tmp/.cache \
+		-e GOCACHE=/tmp/.cache/go \
-		-v $(PWD):$(PWD) \
+		-e GOMODCACHE=/tmp/.cache/gomod \
-		-w $(PWD) \
+		-e GOLANGCI_LINT_CACHE=/tmp/.cache/golangci-lint \
+		-v $(PWD):/work \
+		-w /work \
 		--user $$(id -u):$$(id -g) \
 		$(BUILDIMAGE)

RELEASE.md

@@ -0,0 +1,36 @@
+# Release Process
+
+The NVIDIA Container Toolkit consists of the following artifacts:
+- The NVIDIA Container Toolkit container
+- Packages for debian-based systems
+- Packages for rpm-based systems
+
+# Release Process Checklist:
+- [ ] Create a release PR:
+    - [ ] Run the `./hack/prepare-release.sh` script to update the version in all the needed files. This also creates a [release issue](https://github.com/NVIDIA/cloud-native-team/issues?q=is%3Aissue+is%3Aopen+label%3Arelease)
+    - [ ] Run the `./hack/generate-changelog.sh` script to generate the a draft changelog and update `CHANGELOG.md` with the changes.
+    - [ ] Create a PR from the created `bump-release-{{ .VERSION }}` branch.
+- [ ] Merge the release PR
+- [ ] Tag the release and push the tag to the `internal` mirror:
+    - [ ] Image release pipeline: https://gitlab-master.nvidia.com/dl/container-dev/container-toolkit/-/pipelines/16466098
+- [ ] Wait for the image release to complete.
+- [ ] Push the tag to the the upstream GitHub repo.
+- [ ] Wait for the [`Release`](https://github.com/NVIDIA/k8s-device-plugin/actions/workflows/release.yaml) GitHub Action to complete
+- [ ] Publish the [draft release](https://github.com/NVIDIA/k8s-device-plugin/releases) created by the GitHub Action
+- [ ] Publish the packages to the gh-pages branch of the libnvidia-container repo
+- [ ] Create a KitPick
+
+## Troubleshooting
+
+*Note*: This assumes that we have the release tag checked out locally.
+
+- If the `Release` GitHub Action fails:
+    - Check the logs for the error first.
+    - Create the helm packages locally by running:
+      ```bash
+      ./hack/prepare-artifacts.sh {{ .VERSION }}
+      ```
+    - Create the draft release by running:
+      ```bash
+      ./hack/create-release.sh {{ .VERSION }}
+      ```

build/container/Dockerfile.centos

@@ -1,97 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG BASE_DIST
-ARG CUDA_VERSION
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
-
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
-
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
-
-WORKDIR /build
-COPY . .
-
-# NOTE: Until the config utilities are properly integrated into the
-# nvidia-container-toolkit repository, these are built from the `tools` folder
-# and not `cmd`.
-RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
-
-
-FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
-
-ARG BASE_DIST
-# See https://www.centos.org/centos-linux-eol/
-# and https://stackoverflow.com/a/70930049 for move to vault.centos.org
-# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud
-RUN [[ "${BASE_DIST}" != "centos8" ]] || \
-    ( \
-      sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
-      sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \
-    )
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH ${TARGETARCH}
-RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
-    yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
-
-WORKDIR /work
-
-COPY --from=build /artifacts/bin /work
-
-ENV PATH=/work:$PATH
-
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        yum update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
-
-ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.ubuntu

@@ -1,105 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG BASE_DIST
-ARG CUDA_VERSION
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
-
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
-
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
-
-WORKDIR /build
-COPY . .
-
-# NOTE: Until the config utilities are properly integrated into the
-# nvidia-container-toolkit repository, these are built from the `tools` folder
-# and not `cmd`.
-RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
-
-
-FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
-
-# Remove the CUDA repository configurations to avoid issues with rotated GPG keys
-RUN rm -f /etc/apt/sources.list.d/cuda.list
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libcap2 \
-    curl \
-        && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH ${TARGETARCH}
-
-ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
-ARG LIBNVIDIA_CONTAINER0_VERSION
-RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
-        curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
-            --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \
-        dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \
-    fi
-
-RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
-
-WORKDIR /work
-
-COPY --from=build /artifacts/bin /work/
-
-ENV PATH=/work:$PATH
-
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
-
-ENTRYPOINT ["/work/nvidia-toolkit"]

cmd/nvidia-cdi-hook/README.md

@@ -0,0 +1,31 @@
+# NVIDIA CDI Hook
+
+The CLI `nvidia-cdi-hook` provides container device runtime hook capabilities when
+called by a container runtime, as specific in a
+[Container Device Interface](https://tags.cncf.io/container-device-interface/blob/main/SPEC.md)
+file.
+
+## Generating a CDI
+
+The CDI itself is created for an NVIDIA-capable device using the
+[`nvidia-ctk cdi generate`](../nvidia-ctk/) command.
+
+When `nvidia-ctk cdi generate` is run, the CDI specification is generated as a yaml file.
+The CDI specification provides instructions for a container runtime to set up devices, files and
+other resources for the container prior to starting it. Those instructions
+may include executing command-line tools to prepare the filesystem. The execution
+of such command-line tools is called a hook.
+
+`nvidia-cdi-hook` is the CLI tool that is expected to be called by the container runtime,
+when specified by the CDI file.
+
+See the [`nvidia-ctk` documentation](../nvidia-ctk/README.md) for more information
+on generating a CDI file.
+
+## Functionality
+
+The `nvidia-cdi-hook` CLI provides the following functionality:
+
+* `chmod` - Change the permissions of a file or directory inside the directory path to be mounted into a container.
+* `create-symlinks` - Create symlinks inside the directory path to be mounted into a container.
+* `update-ldcache` - Update the dynamic linker cache inside the directory path to be mounted into a container.

cmd/nvidia-cdi-hook/chmod/chmod.go

@@ -0,0 +1,160 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	paths         cli.StringSlice
+	modeStr       string
+	mode          fs.FileMode
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specify a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.modeStr,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.modeStr) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	modeInt, err := strconv.ParseUint(cfg.modeStr, 8, 32)
+	if err != nil {
+		return fmt.Errorf("failed to parse mode as octal: %v", err)
+	}
+	cfg.mode = fs.FileMode(modeInt)
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value(), cfg.mode)
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	for _, path := range paths {
+		err = os.Chmod(path, cfg.mode)
+		// in some cases this is not an issue (e.g. whole /dev mounted), see #143
+		if errors.Is(err, fs.ErrPermission) {
+			m.logger.Debugf("Ignoring permission error with chmod: %v", err)
+			err = nil
+		}
+	}
+
+	return err
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string, desiredMode fs.FileMode) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		path := filepath.Join(root, f)
+		stat, err := os.Stat(path)
+		if err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		if (stat.Mode()&(fs.ModePerm|fs.ModeSetuid|fs.ModeSetgid|fs.ModeSticky))^desiredMode == 0 {
+			m.logger.Debugf("Skipping path %q: already desired mode", path)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-cdi-hook/commands/commands.go

@@ -0,0 +1,36 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package commands
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
+	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+// New creates the commands associated with supported CDI hooks.
+// These are shared by the nvidia-cdi-hook and nvidia-ctk hook commands.
+func New(logger logger.Interface) []*cli.Command {
+	return []*cli.Command{
+		ldcache.NewCommand(logger),
+		symlinks.NewCommand(logger),
+		chmod.NewCommand(logger),
+	}
+}

cmd/nvidia-cdi-hook/create-symlinks/create-symlinks.go

@@ -0,0 +1,172 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package symlinks
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/moby/sys/symlink"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	links         cli.StringSlice
+	containerSpec string
+}
+
+// NewCommand constructs a hook command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the create-symlink command.
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	c := cli.Command{
+		Name:  "create-symlinks",
+		Usage: "A hook to create symlinks in the container.",
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "link",
+			Usage:       "Specify a specific link to create. The link is specified as target::link. If the link exists in the container root, it is removed.",
+			Destination: &cfg.links,
+		},
+		// The following flags are testing-only flags.
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN. This is only intended for testing.",
+			Destination: &cfg.containerSpec,
+			Hidden:      true,
+		},
+	}
+
+	return &c
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	created := make(map[string]bool)
+	for _, l := range cfg.links.Value() {
+		if created[l] {
+			m.logger.Debugf("Link %v already processed", l)
+			continue
+		}
+		parts := strings.Split(l, "::")
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid symlink specification %v", l)
+		}
+
+		err := m.createLink(containerRoot, parts[0], parts[1])
+		if err != nil {
+			return fmt.Errorf("failed to create link %v: %w", parts, err)
+		}
+		created[l] = true
+	}
+	return nil
+}
+
+// createLink creates a symbolic link in the specified container root.
+// This is equivalent to:
+//
+//	chroot {{ .containerRoot }} ln -f -s {{ .target }} {{ .link }}
+//
+// If the specified link already exists and points to the same target, this
+// operation is a no-op.
+// If a file exists at the link path or the link points to a different target
+// this file is removed before creating the link.
+//
+// Note that if the link path resolves to an absolute path oudside of the
+// specified root, this is treated as an absolute path in this root.
+func (m command) createLink(containerRoot string, targetPath string, link string) error {
+	linkPath := filepath.Join(containerRoot, link)
+
+	exists, err := linkExists(targetPath, linkPath)
+	if err != nil {
+		return fmt.Errorf("failed to check if link exists: %w", err)
+	}
+	if exists {
+		m.logger.Debugf("Link %s already exists", linkPath)
+		return nil
+	}
+
+	// We resolve the parent of the symlink that we're creating in the container root.
+	// If we resolve the full link path, an existing link at the location itself
+	// is also resolved here and we are unable to force create the link.
+	resolvedLinkParent, err := symlink.FollowSymlinkInScope(filepath.Dir(linkPath), containerRoot)
+	if err != nil {
+		return fmt.Errorf("failed to follow path for link %v relative to %v: %w", link, containerRoot, err)
+	}
+	resolvedLinkPath := filepath.Join(resolvedLinkParent, filepath.Base(linkPath))
+
+	m.logger.Infof("Symlinking %v to %v", resolvedLinkPath, targetPath)
+	err = os.MkdirAll(filepath.Dir(resolvedLinkPath), 0755)
+	if err != nil {
+		return fmt.Errorf("failed to create directory: %v", err)
+	}
+	err = symlinks.ForceCreate(targetPath, resolvedLinkPath)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+
+	return nil
+}
+
+// linkExists checks whether the specified link exists.
+// A link exists if the path exists, is a symlink, and points to the specified target.
+func linkExists(target string, link string) (bool, error) {
+	currentTarget, err := symlinks.Resolve(link)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("failed to resolve existing symlink %s: %w", link, err)
+	}
+	if currentTarget == target {
+		return true, nil
+	}
+	return false, nil
+}

cmd/nvidia-cdi-hook/create-symlinks/create-symlinks_test.go

@@ -0,0 +1,297 @@
+package symlinks
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+)
+
+func TestLinkExist(t *testing.T) {
+	tmpDir := t.TempDir()
+	require.NoError(
+		t,
+		makeFs(tmpDir,
+			dirOrLink{path: "/a/b/c", target: "d"},
+			dirOrLink{path: "/a/b/e", target: "/a/b/f"},
+		),
+	)
+
+	exists, err := linkExists("d", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.True(t, exists)
+
+	exists, err = linkExists("/a/b/f", filepath.Join(tmpDir, "/a/b/e"))
+	require.NoError(t, err)
+	require.True(t, exists)
+
+	exists, err = linkExists("different-target", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.False(t, exists)
+
+	exists, err = linkExists("/a/b/d", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.False(t, exists)
+
+	exists, err = linkExists("foo", filepath.Join(tmpDir, "/a/b/does-not-exist"))
+	require.NoError(t, err)
+	require.False(t, exists)
+}
+
+func TestCreateLink(t *testing.T) {
+	type link struct {
+		path   string
+		target string
+	}
+	type expectedLink struct {
+		link
+		err error
+	}
+
+	testCases := []struct {
+		description         string
+		containerContents   []dirOrLink
+		link                link
+		expectedCreateError error
+		expectedLinks       []expectedLink
+	}{
+		{
+			description: "link to / resolves to container root",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "libfoo.so.1",
+					},
+				},
+			},
+		},
+		{
+			description: "link to / resolves to container root; parent relative link",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "../libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "../libfoo.so.1",
+					},
+				},
+			},
+		},
+		{
+			description: "link to / resolves to container root; absolute link",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "/a-path-in-container/foo/libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "/a-path-in-container/foo/libfoo.so.1",
+					},
+				},
+				{
+					// We also check that the target is NOT created.
+					link: link{
+						path: "{{ .containerRoot }}/a-path-in-container/foo/libfoo.so.1",
+					},
+					err: os.ErrNotExist,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			hostRoot := filepath.Join(tmpDir, "/host-root/")
+			containerRoot := filepath.Join(tmpDir, "/container-root")
+
+			require.NoError(t, makeFs(hostRoot))
+			require.NoError(t, makeFs(containerRoot, tc.containerContents...))
+
+			// nvidia-cdi-hook create-symlinks --link linkSpec
+			err := getTestCommand().createLink(containerRoot, tc.link.target, tc.link.path)
+			// TODO: We may be able to replace this with require.ErrorIs.
+			if tc.expectedCreateError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			for _, expectedLink := range tc.expectedLinks {
+				path := strings.ReplaceAll(expectedLink.path, "{{ .containerRoot }}", containerRoot)
+				path = strings.ReplaceAll(path, "{{ .hostRoot }}", hostRoot)
+				if expectedLink.target != "" {
+					target, err := symlinks.Resolve(path)
+					require.ErrorIs(t, err, expectedLink.err)
+					require.Equal(t, expectedLink.target, target)
+				} else {
+					_, err := os.Stat(path)
+					require.ErrorIs(t, err, expectedLink.err)
+				}
+			}
+		})
+	}
+}
+
+func TestCreateLinkRelativePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root/")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t, makeFs(hostRoot))
+	require.NoError(t, makeFs(containerRoot, dirOrLink{path: "/lib/"}))
+
+	// nvidia-cdi-hook create-symlinks --link libfoo.so.1::/lib/libfoo.so
+	err := getTestCommand().createLink(containerRoot, "libfoo.so.1", "/lib/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "libfoo.so.1", target)
+}
+
+func TestCreateLinkAbsolutePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root/")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t, makeFs(hostRoot))
+	require.NoError(t, makeFs(containerRoot, dirOrLink{path: "/lib/"}))
+
+	// nvidia-cdi-hook create-symlinks --link /lib/libfoo.so.1::/lib/libfoo.so
+	err := getTestCommand().createLink(containerRoot, "/lib/libfoo.so.1", "/lib/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "/lib/libfoo.so.1", target)
+}
+
+func TestCreateLinkAlreadyExists(t *testing.T) {
+	testCases := []struct {
+		description       string
+		containerContents []dirOrLink
+		shouldExist       []string
+	}{
+		{
+			description:       "link already exists with correct target",
+			containerContents: []dirOrLink{{path: "/lib/libfoo.so", target: "libfoo.so.1"}},
+			shouldExist:       []string{},
+		},
+		{
+			description:       "link already exists with different target",
+			containerContents: []dirOrLink{{path: "/lib/libfoo.so", target: "different-target"}, {path: "different-target"}},
+			shouldExist:       []string{"{{ .containerRoot }}/different-target"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			hostRoot := filepath.Join(tmpDir, "/host-root/")
+			containerRoot := filepath.Join(tmpDir, "/container-root")
+			require.NoError(t, makeFs(hostRoot))
+			require.NoError(t, makeFs(containerRoot, tc.containerContents...))
+
+			// nvidia-cdi-hook create-symlinks --link libfoo.so.1::/lib/libfoo.so
+			err := getTestCommand().createLink(containerRoot, "libfoo.so.1", "/lib/libfoo.so")
+			require.NoError(t, err)
+			target, err := symlinks.Resolve(filepath.Join(containerRoot, "lib/libfoo.so"))
+			require.NoError(t, err)
+			require.Equal(t, "libfoo.so.1", target)
+
+			for _, p := range tc.shouldExist {
+				require.DirExists(t, strings.ReplaceAll(p, "{{ .containerRoot }}", containerRoot))
+			}
+		})
+	}
+}
+
+func TestCreateLinkOutOfBounds(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t,
+		makeFs(hostRoot,
+			dirOrLink{path: "libfoo.so"},
+		),
+	)
+	require.NoError(t,
+		makeFs(containerRoot,
+			dirOrLink{path: "/lib"},
+			dirOrLink{path: "/lib/foo", target: hostRoot},
+		),
+	)
+
+	path, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/foo"))
+	require.NoError(t, err)
+	require.Equal(t, hostRoot, path)
+
+	// nvidia-cdi-hook create-symlinks --link ../libfoo.so.1::/lib/foo/libfoo.so
+	_ = getTestCommand().createLink(containerRoot, "../libfoo.so.1", "/lib/foo/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, hostRoot, "libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "../libfoo.so.1", target)
+
+	require.DirExists(t, filepath.Join(hostRoot, "libfoo.so"))
+}
+
+type dirOrLink struct {
+	path   string
+	target string
+}
+
+func makeFs(tmpdir string, fs ...dirOrLink) error {
+	if err := os.MkdirAll(tmpdir, 0o755); err != nil {
+		return err
+	}
+	for _, s := range fs {
+		s.path = filepath.Join(tmpdir, s.path)
+		if s.target == "" {
+			_ = os.MkdirAll(s.path, 0o755)
+			continue
+		}
+		if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
+			return err
+		}
+		if err := os.Symlink(s.target, s.path); err != nil && !os.IsExist(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// getTestCommand creates a command for running tests against.
+func getTestCommand() *command {
+	logger, _ := testlog.NewNullLogger()
+	return &command{
+		logger: logger,
+	}
+}

cmd/nvidia-cdi-hook/main.go

@@ -0,0 +1,93 @@
+/**
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/commands"
+)
+
+// options defines the options that can be set for the CLI through config files,
+// environment variables, or command line flags
+type options struct {
+	// Debug indicates whether the CLI is started in "debug" mode
+	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
+}
+
+func main() {
+	logger := logrus.New()
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "NVIDIA CDI Hook"
+	c.UseShortOptionHandling = true
+	c.EnableBashCompletion = true
+	c.Usage = "Command to structure files for usage inside a container, called as hooks from a container runtime, defined in a CDI yaml file"
+	c.Version = info.GetVersionString()
+
+	// Setup the flags for this command
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "debug",
+			Aliases:     []string{"d"},
+			Usage:       "Enable debug-level logging",
+			Destination: &opts.Debug,
+			EnvVars:     []string{"NVIDIA_CDI_DEBUG"},
+		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CDI_QUIET"},
+		},
+	}
+
+	// Set log-level for all subcommands
+	c.Before = func(c *cli.Context) error {
+		logLevel := logrus.InfoLevel
+		if opts.Debug {
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
+		}
+		logger.SetLevel(logLevel)
+		return nil
+	}
+
+	// Define the subcommands
+	c.Commands = commands.New(logger)
+
+	// Run the CLI
+	err := c.Run(os.Args)
+	if err != nil {
+		logger.Errorf("%v", err)
+		os.Exit(1)
+	}
+}

cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go

@@ -0,0 +1,197 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package ldcache
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	folders       cli.StringSlice
+	ldconfigPath  string
+	containerSpec string
+}
+
+// NewCommand constructs an update-ldcache command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the update-ldcache command
+func (m command) build() *cli.Command {
+	cfg := options{}
+
+	// Create the 'update-ldcache' command
+	c := cli.Command{
+		Name:  "update-ldcache",
+		Usage: "Update ldcache in a container by running ldconfig",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "folder",
+			Usage:       "Specify a folder to add to /etc/ld.so.conf before updating the ld cache",
+			Destination: &cfg.folders,
+		},
+		&cli.StringFlag{
+			Name:        "ldconfig-path",
+			Usage:       "Specify the path to the ldconfig program",
+			Destination: &cfg.ldconfigPath,
+			Value:       "/sbin/ldconfig",
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *options) error {
+	if cfg.ldconfigPath == "" {
+		return errors.New("ldconfig-path must be specified")
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *options) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	ldconfigPath := m.resolveLDConfigPath(cfg.ldconfigPath)
+	args := []string{filepath.Base(ldconfigPath)}
+	if containerRoot != "" {
+		args = append(args, "-r", containerRoot)
+	}
+
+	if root(containerRoot).hasPath("/etc/ld.so.cache") {
+		args = append(args, "-C", "/etc/ld.so.cache")
+	} else {
+		m.logger.Debugf("No ld.so.cache found, skipping update")
+		args = append(args, "-N")
+	}
+
+	folders := cfg.folders.Value()
+	if root(containerRoot).hasPath("/etc/ld.so.conf.d") {
+		err := m.createConfig(containerRoot, folders)
+		if err != nil {
+			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
+		}
+	} else {
+		args = append(args, folders...)
+	}
+
+	// Explicitly specify using /etc/ld.so.conf since the host's ldconfig may
+	// be configured to use a different config file by default.
+	args = append(args, "-f", "/etc/ld.so.conf")
+
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
+	return syscall.Exec(ldconfigPath, args, nil)
+}
+
+type root string
+
+func (r root) hasPath(path string) bool {
+	_, err := os.Stat(filepath.Join(string(r), path))
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+// resolveLDConfigPath determines the LDConfig path to use for the system.
+// On systems such as Ubuntu where `/sbin/ldconfig` is a wrapper around
+// /sbin/ldconfig.real, the latter is returned.
+func (m command) resolveLDConfigPath(path string) string {
+	return strings.TrimPrefix(config.NormalizeLDConfigPath("@"+path), "@")
+}
+
+// createConfig creates (or updates) /etc/ld.so.conf.d/00-nvcr-<RANDOM_STRING>.conf in the container
+// to include the required paths.
+// Note that the 00-nvcr prefix is chosen to ensure that these libraries have
+// a higher precedence than other libraries on the system but are applied AFTER
+// 00-cuda-compat.conf.
+func (m command) createConfig(root string, folders []string) error {
+	if len(folders) == 0 {
+		m.logger.Debugf("No folders to add to /etc/ld.so.conf")
+		return nil
+	}
+
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
+	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "00-nvcr-*.conf")
+	if err != nil {
+		return fmt.Errorf("failed to create config file: %v", err)
+	}
+	defer configFile.Close()
+
+	m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name())
+
+	configured := make(map[string]bool)
+	for _, folder := range folders {
+		if configured[folder] {
+			continue
+		}
+		_, err = configFile.WriteString(fmt.Sprintf("%s\n", folder))
+		if err != nil {
+			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
+		}
+		configured[folder] = true
+	}
+
+	// The created file needs to be world readable for the cases where the container is run as a non-root user.
+	if err := os.Chmod(configFile.Name(), 0644); err != nil {
+		return fmt.Errorf("failed to chmod config file: %v", err)
+	}
+
+	return nil
+}

cmd/nvidia-container-runtime-hook/capabilities.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"log"
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -6,47 +6,33 @@ import (
 	"log"
 	"os"
 	"path"
-	"path/filepath"
+
-	"strings"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"golang.org/x/mod/semver"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"golang.org/x/mod/semver"
-)
-
-var envSwarmGPU *string
-
-const (
-	envCUDAVersion          = "CUDA_VERSION"
-	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
-	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
-	envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
-	envNVMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
-	envNVMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
-	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
 const (
 	capSysAdmin = "CAP_SYS_ADMIN"
 )
 
-const (
-	deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
-)
-
 type nvidiaConfig struct {
-	Devices            string
+	Devices            []string
 	MigConfigDevices   string
 	MigMonitorDevices  string
+	ImexChannels       []string
 	DriverCapabilities string
-	Requirements       []string
+	// Requirements defines the requirements DSL for the container to run.
-	DisableRequire     bool
+	// This is empty if no specific requirements are needed, or if requirements are
+	// explicitly disabled.
+	Requirements []string
 }
 
 type containerConfig struct {
 	Pid    int
 	Rootfs string
-	Env    map[string]string
+	Image  image.CUDA
 	Nvidia *nvidiaConfig
 }
 
@@ -73,23 +59,14 @@ type LinuxCapabilities struct {
 	Ambient     []string `json:"ambient,omitempty" platform:"linux"`
 }
 
-// Mount from OCI runtime spec
-// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L103
-type Mount struct {
-	Destination string   `json:"destination"`
-	Type        string   `json:"type,omitempty" platform:"linux,solaris"`
-	Source      string   `json:"source,omitempty"`
-	Options     []string `json:"options,omitempty"`
-}
-
 // Spec from OCI runtime spec
 // We use pointers to structs, similarly to the latest version of runtime-spec:
 // https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L5-L28
 type Spec struct {
-	Version *string  `json:"ociVersion"`
+	Version *string       `json:"ociVersion"`
-	Process *Process `json:"process,omitempty"`
+	Process *Process      `json:"process,omitempty"`
-	Root    *Root    `json:"root,omitempty"`
+	Root    *Root         `json:"root,omitempty"`
-	Mounts  []Mount  `json:"mounts,omitempty"`
+	Mounts  []specs.Mount `json:"mounts,omitempty"`
 }
 
 // HookState holds state information about the hook
@@ -132,7 +109,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -141,118 +118,57 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// Otherwise, parse s.Process.Capabilities as:
+		for _, c := range caps {
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+			if c == capSysAdmin {
-	} else {
+				return true
-		var lc LinuxCapabilities
+			}
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		// We only make sure that the bounding capabibility set has
+		return false
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
 	}
 
-	for _, c := range caps {
+	// Otherwise, parse s.Process.Capabilities as:
-		if c == capSysAdmin {
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-			return true
+	process := specs.Process{
-		}
+		Env: s.Process.Env,
 	}
 
-	return false
+	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	fullSpec := specs.Spec{
+		Version: *s.Version,
+		Process: &process,
+	}
+
+	return image.IsPrivileged(&fullSpec)
 }
 
-func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
+func getDevicesFromEnvvar(containerImage image.CUDA, swarmResourceEnvvars []string) []string {
-	// Build a list of envvars to consider.
+	// We check if the image has at least one of the Swarm resource envvars defined and use this
-	envVars := []string{envNVVisibleDevices}
+	// if specified.
-	if envSwarmGPU != nil {
+	for _, envvar := range swarmResourceEnvvars {
-		// The Swarm envvar has higher precedence.
+		if containerImage.HasEnvvar(envvar) {
-		envVars = append([]string{*envSwarmGPU}, envVars...)
+			return containerImage.DevicesFromEnvvars(swarmResourceEnvvars...).List()
-	}
-
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
-	for _, envVar := range envVars {
-		if devs, ok := env[envVar]; ok {
-			devices = &devs
-			break
 		}
 	}
 
-	// Environment variable unset with legacy image: default to "all".
+	return containerImage.VisibleDevicesFromEnvVar()
-	if devices == nil && legacyImage {
-		all := "all"
-		return &all
-	}
-
-	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
-		return nil
-	}
-
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		empty := ""
-		return &empty
-	}
-
-	// Any other value.
-	return devices
 }
 
-func getDevicesFromMounts(mounts []Mount) *string {
+func (hookConfig *hookConfig) getDevices(image image.CUDA, privileged bool) []string {
-	var devices []string
-	for _, m := range mounts {
-		root := filepath.Clean(deviceListAsVolumeMountsRoot)
-		source := filepath.Clean(m.Source)
-		destination := filepath.Clean(m.Destination)
-
-		// Only consider mounts who's host volume is /dev/null
-		if source != "/dev/null" {
-			continue
-		}
-		// Only consider container mount points that begin with 'root'
-		if len(destination) < len(root) {
-			continue
-		}
-		if destination[:len(root)] != root {
-			continue
-		}
-		// Grab the full path beyond 'root' and add it to the list of devices
-		device := destination[len(root):]
-		if len(device) > 0 && device[0] == '/' {
-			device = device[1:]
-		}
-		if len(device) == 0 {
-			continue
-		}
-		devices = append(devices, device)
-	}
-
-	if devices == nil {
-		return nil
-	}
-
-	ret := strings.Join(devices, ",")
-	return &ret
-}
-
-func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
-		devices := getDevicesFromMounts(mounts)
+		devices := image.VisibleDevicesFromMounts()
-		if devices != nil {
+		if len(devices) > 0 {
 			return devices
 		}
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(env, legacyImage)
+	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
-	if devices == nil {
+	if len(devices) == 0 {
 		return nil
 	}
 	if privileged || hookConfig.AcceptEnvvarUnprivileged {
@@ -265,26 +181,51 @@ func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, p
 	return nil
 }
 
-func getMigConfigDevices(env map[string]string) *string {
+func getMigConfigDevices(i image.CUDA) *string {
-	if devices, ok := env[envNVMigConfigDevices]; ok {
+	return getMigDevices(i, image.EnvVarNvidiaMigConfigDevices)
-		return &devices
+}
+
+func getMigMonitorDevices(i image.CUDA) *string {
+	return getMigDevices(i, image.EnvVarNvidiaMigMonitorDevices)
+}
+
+func getMigDevices(image image.CUDA, envvar string) *string {
+	if !image.HasEnvvar(envvar) {
+		return nil
 	}
+	devices := image.Getenv(envvar)
+	return &devices
+}
+
+func (hookConfig *hookConfig) getImexChannels(image image.CUDA, privileged bool) []string {
+	// If enabled, try and get the device list from volume mounts first
+	if hookConfig.AcceptDeviceListAsVolumeMounts {
+		devices := image.ImexChannelsFromMounts()
+		if len(devices) > 0 {
+			return devices
+		}
+	}
+	devices := image.ImexChannelsFromEnvVar()
+	if len(devices) == 0 {
+		return nil
+	}
+
+	if privileged || hookConfig.AcceptEnvvarUnprivileged {
+		return devices
+	}
+
 	return nil
 }
 
-func getMigMonitorDevices(env map[string]string) *string {
+func (hookConfig *hookConfig) getDriverCapabilities(cudaImage image.CUDA, legacyImage bool) image.DriverCapabilities {
-	if devices, ok := env[envNVMigMonitorDevices]; ok {
-		return &devices
-	}
-	return nil
-}
-
-func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
 	// We use the default driver capabilities by default. This is filtered to only include the
 	// supported capabilities
-	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
+	supportedDriverCapabilities := image.NewDriverCapabilities(hookConfig.SupportedDriverCapabilities)
 
-	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
+	capabilities := supportedDriverCapabilities.Intersection(image.DefaultDriverCapabilities)
+
+	capsEnvSpecified := cudaImage.HasEnvvar(image.EnvVarNvidiaDriverCapabilities)
+	capsEnv := cudaImage.Getenv(image.EnvVarNvidiaDriverCapabilities)
 
 	if !capsEnvSpecified && legacyImage {
 		// Environment variable unset with legacy image: set all capabilities.
@@ -293,9 +234,9 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr
 
 	if capsEnvSpecified && len(capsEnv) > 0 {
 		// If the envvironment variable is specified and is non-empty, use the capabilities value
-		envCapabilities := DriverCapabilities(capsEnv)
+		envCapabilities := image.NewDriverCapabilities(capsEnv)
 		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
-		if envCapabilities != all && capabilities != envCapabilities {
+		if !envCapabilities.IsAll() && len(capabilities) != len(envCapabilities) {
 			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
 		}
 	}
@@ -303,14 +244,12 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr
 	return capabilities
 }
 
-func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
+func (hookConfig *hookConfig) getNvidiaConfig(image image.CUDA, privileged bool) *nvidiaConfig {
 	legacyImage := image.IsLegacy()
 
-	var devices string
+	devices := hookConfig.getDevices(image, privileged)
-	if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil {
+	if len(devices) == 0 {
-		devices = *d
+		// empty devices means this is not a GPU container.
-	} else {
-		// 'nil' devices means this is not a GPU container.
 		return nil
 	}
 
@@ -330,26 +269,26 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+	imexChannels := hookConfig.getImexChannels(image, privileged)
+
+	driverCapabilities := hookConfig.getDriverCapabilities(image, legacyImage).String()
 
 	requirements, err := image.GetRequirements()
 	if err != nil {
 		log.Panicln("failed to get requirements", err)
 	}
 
-	disableRequire := image.HasDisableRequire()
-
 	return &nvidiaConfig{
 		Devices:            devices,
 		MigConfigDevices:   migConfigDevices,
 		MigMonitorDevices:  migMonitorDevices,
+		ImexChannels:       imexChannels,
 		DriverCapabilities: driverCapabilities,
 		Requirements:       requirements,
-		DisableRequire:     disableRequire,
 	}
 }
 
-func getContainerConfig(hook HookConfig) (config containerConfig) {
+func (hookConfig *hookConfig) getContainerConfig() (config containerConfig) {
 	var h HookState
 	d := json.NewDecoder(os.Stdin)
 	if err := d.Decode(&h); err != nil {
@@ -363,17 +302,20 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	image, err := image.New(
+		image.WithEnv(s.Process.Env),
+		image.WithMounts(s.Mounts),
+		image.WithDisableRequire(hookConfig.DisableRequire),
+	)
 	if err != nil {
 		log.Panicln(err)
 	}
 
 	privileged := isPrivileged(s)
-	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    image,
+		Image:  image,
-		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
+		Nvidia: hookConfig.getNvidiaConfig(image, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -0,0 +1,977 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
+
+func TestGetNvidiaConfig(t *testing.T) {
+	var tests = []struct {
+		description    string
+		env            map[string]string
+		privileged     bool
+		hookConfig     *hookConfig
+		expectedConfig *nvidiaConfig
+		expectedPanic  bool
+	}{
+		{
+			description:    "No environment, unprivileged",
+			env:            map[string]string{},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description:    "No environment, privileged",
+			env:            map[string]string{},
+			privileged:     true,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, no devices, no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion: "9.0",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices 'all', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices 'empty', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, devices 'void', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "void",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, devices 'none', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "none",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{""},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "gpu0,gpu1",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities 'empty', no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities 'all', no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, requirements set",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, requirements set, disable requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+				image.EnvVarNvidiaDisableRequire:     "true",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
+			},
+		},
+		{
+			description: "Modern image, no devices, no capabilities, no requirements, no image.EnvVarCudaVersion",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda: "cuda>=9.0",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, no devices, no capabilities, no requirement, image.EnvVarCudaVersion set",
+			env: map[string]string{
+				image.EnvVarCudaVersion:       "9.0",
+				image.EnvVarNvidiaRequireCuda: "cuda>=9.0",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'all', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices 'empty', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'void', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "void",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'none', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "none",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{""},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "gpu0,gpu1",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities 'empty', no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities 'all', no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, requirements set",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, requirements set, disable requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+				image.EnvVarNvidiaDisableRequire:     "true",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
+			},
+		},
+		{
+			description: "No cuda envs, devices 'all'",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: false,
+
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{},
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migConfig set, privileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:      "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:   "all",
+				image.EnvVarNvidiaMigConfigDevices: "mig0,mig1",
+			},
+			privileged: true,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				MigConfigDevices:   "mig0,mig1",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migConfig set, unprivileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:      "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:   "all",
+				image.EnvVarNvidiaMigConfigDevices: "mig0,mig1",
+			},
+			privileged:    false,
+			expectedPanic: true,
+		},
+		{
+			description: "Modern image, devices 'all', migMonitor set, privileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:       "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:    "all",
+				image.EnvVarNvidiaMigMonitorDevices: "mig0,mig1",
+			},
+			privileged: true,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				MigMonitorDevices:  "mig0,mig1",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migMonitor set, unprivileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:       "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:    "all",
+				image.EnvVarNvidiaMigMonitorDevices: "mig0,mig1",
+			},
+			privileged:    false,
+			expectedPanic: true,
+		},
+		{
+			description: "Hook config set as driver-capabilities-all",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:     "all",
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: "video,display",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: "display,video",
+			},
+		},
+		{
+			description: "Hook config set, envvar sets driver-capabilities",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:     "all",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: "video,display,compute,utility",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: "display,video",
+			},
+		},
+		{
+			description: "Hook config set, envvar unset sets default driver-capabilities",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: "video,display,utility,compute",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, swarmResource overrides device selection",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+				"DOCKER_SWARM_RESOURCE":          "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SwarmResource:               "DOCKER_SWARM_RESOURCE",
+					SupportedDriverCapabilities: "video,display,utility,compute",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"GPU1", "GPU2"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, comma separated swarmResource is split and overrides device selection",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+				"DOCKER_SWARM_RESOURCE":          "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SwarmResource:               "NOT_DOCKER_SWARM_RESOURCE,DOCKER_SWARM_RESOURCE",
+					SupportedDriverCapabilities: "video,display,utility,compute",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"GPU1", "GPU2"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			image, _ := image.New(
+				image.WithEnvMap(tc.env),
+			)
+			// Wrap the call to getNvidiaConfig() in a closure.
+			var cfg *nvidiaConfig
+			getConfig := func() {
+				hookCfg := tc.hookConfig
+				if hookCfg == nil {
+					defaultConfig, _ := config.GetDefault()
+					hookCfg = &hookConfig{defaultConfig}
+				}
+				cfg = hookCfg.getNvidiaConfig(image, tc.privileged)
+			}
+
+			// For any tests that are expected to panic, make sure they do.
+			if tc.expectedPanic {
+				require.Panics(t, getConfig)
+				return
+			}
+
+			// For all other tests, just grab the config
+			getConfig()
+
+			// And start comparing the test results to the expected results.
+			if tc.expectedConfig == nil {
+				require.Nil(t, cfg, tc.description)
+				return
+			}
+
+			require.NotNil(t, cfg, tc.description)
+
+			require.Equal(t, tc.expectedConfig.Devices, cfg.Devices)
+			require.Equal(t, tc.expectedConfig.MigConfigDevices, cfg.MigConfigDevices)
+			require.Equal(t, tc.expectedConfig.MigMonitorDevices, cfg.MigMonitorDevices)
+			require.Equal(t, tc.expectedConfig.DriverCapabilities, cfg.DriverCapabilities)
+
+			require.ElementsMatch(t, tc.expectedConfig.Requirements, cfg.Requirements)
+		})
+	}
+}
+
+func TestDeviceListSourcePriority(t *testing.T) {
+	var tests = []struct {
+		description        string
+		mountDevices       []specs.Mount
+		envvarDevices      string
+		privileged         bool
+		acceptUnprivileged bool
+		acceptMounts       bool
+		expectedDevices    []string
+	}{
+		{
+			description: "Mount devices, unprivileged, no accept unprivileged",
+			mountDevices: []specs.Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(image.DeviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(image.DeviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			envvarDevices:      "GPU2,GPU3",
+			privileged:         false,
+			acceptUnprivileged: false,
+			acceptMounts:       true,
+			expectedDevices:    []string{"GPU0", "GPU1"},
+		},
+		{
+			description:        "No mount devices, unprivileged, no accept unprivileged",
+			mountDevices:       nil,
+			envvarDevices:      "GPU0,GPU1",
+			privileged:         false,
+			acceptUnprivileged: false,
+			acceptMounts:       true,
+			expectedDevices:    nil,
+		},
+		{
+			description:        "No mount devices, privileged, no accept unprivileged",
+			mountDevices:       nil,
+			envvarDevices:      "GPU0,GPU1",
+			privileged:         true,
+			acceptUnprivileged: false,
+			acceptMounts:       true,
+			expectedDevices:    []string{"GPU0", "GPU1"},
+		},
+		{
+			description:        "No mount devices, unprivileged, accept unprivileged",
+			mountDevices:       nil,
+			envvarDevices:      "GPU0,GPU1",
+			privileged:         false,
+			acceptUnprivileged: true,
+			acceptMounts:       true,
+			expectedDevices:    []string{"GPU0", "GPU1"},
+		},
+		{
+			description: "Mount devices, unprivileged, accept unprivileged, no accept mounts",
+			mountDevices: []specs.Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(image.DeviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(image.DeviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			envvarDevices:      "GPU2,GPU3",
+			privileged:         false,
+			acceptUnprivileged: true,
+			acceptMounts:       false,
+			expectedDevices:    []string{"GPU2", "GPU3"},
+		},
+		{
+			description: "Mount devices, unprivileged, no accept unprivileged, no accept mounts",
+			mountDevices: []specs.Mount{
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(image.DeviceListAsVolumeMountsRoot, "GPU0"),
+				},
+				{
+					Source:      "/dev/null",
+					Destination: filepath.Join(image.DeviceListAsVolumeMountsRoot, "GPU1"),
+				},
+			},
+			envvarDevices:      "GPU2,GPU3",
+			privileged:         false,
+			acceptUnprivileged: false,
+			acceptMounts:       false,
+			expectedDevices:    nil,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			// Wrap the call to getDevices() in a closure.
+			var devices []string
+			getDevices := func() {
+				image, _ := image.New(
+					image.WithEnvMap(
+						map[string]string{
+							image.EnvVarNvidiaVisibleDevices: tc.envvarDevices,
+						},
+					),
+					image.WithMounts(tc.mountDevices),
+				)
+				defaultConfig, _ := config.GetDefault()
+				cfg := &hookConfig{defaultConfig}
+				cfg.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
+				cfg.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
+				devices = cfg.getDevices(image, tc.privileged)
+			}
+
+			// For all other tests, just grab the devices and check the results
+			getDevices()
+
+			require.Equal(t, tc.expectedDevices, devices)
+		})
+	}
+}
+
+func TestGetDevicesFromEnvvar(t *testing.T) {
+	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
+	gpuID := "GPU-12345"
+	anotherGPUID := "GPU-67890"
+	thirdGPUID := "MIG-12345"
+
+	var tests = []struct {
+		description          string
+		swarmResourceEnvvars []string
+		env                  map[string]string
+		expectedDevices      []string
+	}{
+		{
+			description: "empty env returns nil for non-legacy image",
+		},
+		{
+			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "",
+			},
+		},
+		{
+			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "void",
+			},
+		},
+		{
+			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "none",
+			},
+			expectedDevices: []string{""},
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: gpuID,
+			},
+			expectedDevices: []string{gpuID},
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: gpuID,
+				image.EnvVarCudaVersion:          "legacy",
+			},
+			expectedDevices: []string{gpuID},
+		},
+		{
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				image.EnvVarCudaVersion: "legacy",
+			},
+			expectedDevices: []string{"all"},
+		},
+		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
+		// not enabled
+		{
+			description: "missing NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "",
+				envDockerResourceGPUs:            anotherGPUID,
+			},
+		},
+		{
+			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "void",
+				envDockerResourceGPUs:            anotherGPUID,
+			},
+		},
+		{
+			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "none",
+				envDockerResourceGPUs:            anotherGPUID,
+			},
+			expectedDevices: []string{""},
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: gpuID,
+				envDockerResourceGPUs:            anotherGPUID,
+			},
+			expectedDevices: []string{gpuID},
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: gpuID,
+				envDockerResourceGPUs:            anotherGPUID,
+				image.EnvVarCudaVersion:          "legacy",
+			},
+			expectedDevices: []string{gpuID},
+		},
+		{
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envDockerResourceGPUs:   anotherGPUID,
+				image.EnvVarCudaVersion: "legacy",
+			},
+			expectedDevices: []string{"all"},
+		},
+		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
+		// enabled
+		{
+			description:          "empty env returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+		},
+		{
+			description:          "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				envDockerResourceGPUs: "",
+			},
+		},
+		{
+			description:          "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				envDockerResourceGPUs: "void",
+			},
+		},
+		{
+			description:          "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				envDockerResourceGPUs: "none",
+			},
+			expectedDevices: []string{""},
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				envDockerResourceGPUs: gpuID,
+			},
+			expectedDevices: []string{gpuID},
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				envDockerResourceGPUs:   gpuID,
+				image.EnvVarCudaVersion: "legacy",
+			},
+			expectedDevices: []string{gpuID},
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS is selected if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: []string{anotherGPUID},
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: gpuID,
+				envDockerResourceGPUs:            anotherGPUID,
+			},
+			expectedDevices: []string{anotherGPUID},
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL overrides NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:  gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: []string{anotherGPUID},
+		},
+		{
+			description:          "All available swarm resource envvars are selected and override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:  gpuID,
+				"DOCKER_RESOURCE_GPUS":            thirdGPUID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: []string{thirdGPUID, anotherGPUID},
+		},
+		{
+			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS override NVIDIA_VISIBLE_DEVICES if present",
+			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:  gpuID,
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
+			},
+			expectedDevices: []string{anotherGPUID},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			image, _ := image.New(
+				image.WithEnvMap(tc.env),
+			)
+			devices := getDevicesFromEnvvar(image, tc.swarmResourceEnvvars)
+			require.EqualValues(t, tc.expectedDevices, devices)
+		})
+	}
+}
+
+func TestGetDriverCapabilities(t *testing.T) {
+
+	supportedCapabilities := "compute,display,utility,video"
+
+	testCases := []struct {
+		description           string
+		env                   map[string]string
+		legacyImage           bool
+		supportedCapabilities string
+		expectedPanic         bool
+		expectedCapabilities  string
+	}{
+		{
+			description: "Env is set for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "display,video",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description: "Env is all for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
+		},
+		{
+			description:           "Env unset for legacy image is 'all'",
+			env:                   map[string]string{},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is set for modern image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "display,video",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description:           "Env unset for modern image is default",
+			env:                   map[string]string{},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
+		},
+		{
+			description: "Env is all for modern image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for modern image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
+		},
+		{
+			description: "Invalid capabilities panic",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "compute,utility",
+			},
+			supportedCapabilities: "not-compute,not-utility",
+			expectedPanic:         true,
+		},
+		{
+			description:           "Default is restricted for modern image",
+			legacyImage:           false,
+			supportedCapabilities: "compute",
+			expectedCapabilities:  "compute",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			var capabilities string
+
+			c := hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: tc.supportedCapabilities,
+				},
+			}
+
+			image, _ := image.New(
+				image.WithEnvMap(tc.env),
+			)
+			getDriverCapabilities := func() {
+				capabilities = c.getDriverCapabilities(image, tc.legacyImage).String()
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getDriverCapabilities)
+				return
+			}
+
+			getDriverCapabilities()
+			require.EqualValues(t, tc.expectedCapabilities, capabilities)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
+
+const (
+	configPath = "/etc/nvidia-container-runtime/config.toml"
+	driverPath = "/run/nvidia/driver"
+)
+
+// hookConfig wraps the toolkit config.
+// This allows for functions to be defined on the local type.
+type hookConfig struct {
+	*config.Config
+}
+
+// loadConfig loads the required paths for the hook config.
+func loadConfig() (*config.Config, error) {
+	var configPaths []string
+	var required bool
+	if len(*configflag) != 0 {
+		configPaths = append(configPaths, *configflag)
+		required = true
+	} else {
+		configPaths = append(configPaths, path.Join(driverPath, configPath), configPath)
+	}
+
+	for _, p := range configPaths {
+		cfg, err := config.New(
+			config.WithConfigFile(p),
+			config.WithRequired(true),
+		)
+		if err == nil {
+			return cfg.Config()
+		} else if os.IsNotExist(err) && !required {
+			continue
+		}
+		return nil, fmt.Errorf("couldn't open required configuration file: %v", err)
+	}
+
+	return config.GetDefault()
+}
+
+func getHookConfig() (*hookConfig, error) {
+	cfg, err := loadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config := &hookConfig{cfg}
+
+	allSupportedDriverCapabilities := image.SupportedDriverCapabilities
+	if config.SupportedDriverCapabilities == "all" {
+		config.SupportedDriverCapabilities = allSupportedDriverCapabilities.String()
+	}
+	configuredCapabilities := image.NewDriverCapabilities(config.SupportedDriverCapabilities)
+	// We ensure that the configured value is a subset of all supported capabilities
+	if !allSupportedDriverCapabilities.IsSuperset(configuredCapabilities) {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allSupportedDriverCapabilities.String())
+	}
+
+	return config, nil
+}
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c hookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}
+
+// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
+func (c *hookConfig) getSwarmResourceEnvvars() []string {
+	if c.SwarmResource == "" {
+		return nil
+	}
+
+	candidates := strings.Split(c.SwarmResource, ",")
+
+	var envvars []string
+	for _, c := range candidates {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) > 0 {
+			envvars = append(envvars, trimmed)
+		}
+	}
+
+	return envvars
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -22,22 +22,25 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 )
 
 func TestGetHookConfig(t *testing.T) {
 	testCases := []struct {
 		lines                      []string
 		expectedPanic              bool
-		expectedDriverCapabilities DriverCapabilities
+		expectedDriverCapabilities string
 	}{
 		{
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"all\"",
 			},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
@@ -47,19 +50,19 @@ func TestGetHookConfig(t *testing.T) {
 		},
 		{
 			lines:                      []string{},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"\"",
 			},
-			expectedDriverCapabilities: none,
+			expectedDriverCapabilities: "",
 		},
 		{
 			lines: []string{
-				"supported-driver-capabilities = \"utility,compute\"",
+				"supported-driver-capabilities = \"compute,utility\"",
 			},
-			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+			expectedDriverCapabilities: "compute,utility",
 		},
 	}
 
@@ -87,9 +90,10 @@ func TestGetHookConfig(t *testing.T) {
 				}
 			}
 
-			var config HookConfig
+			var cfg hookConfig
 			getHookConfig := func() {
-				config = getHookConfig()
+				c, _ := getHookConfig()
+				cfg = *c
 			}
 
 			if tc.expectedPanic {
@@ -99,7 +103,56 @@ func TestGetHookConfig(t *testing.T) {
 
 			getHookConfig()
 
-			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+			require.EqualValues(t, tc.expectedDriverCapabilities, cfg.SupportedDriverCapabilities)
+		})
+	}
+}
+
+func TestGetSwarmResourceEnvvars(t *testing.T) {
+	testCases := []struct {
+		value    string
+		expected []string
+	}{
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value:    " ",
+			expected: nil,
+		},
+		{
+			value:    "single",
+			expected: []string{"single"},
+		},
+		{
+			value:    "single ",
+			expected: []string{"single"},
+		},
+		{
+			value:    "one,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one ,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one, two",
+			expected: []string{"one", "two"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			c := &hookConfig{
+				Config: &config.Config{
+					SwarmResource: tc.value,
+				},
+			}
+
+			envvars := c.getSwarmResourceEnvvars()
+			require.EqualValues(t, tc.expected, envvars)
 		})
 	}
 }

cmd/nvidia-container-runtime-hook/main.go

@@ -13,7 +13,9 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
@@ -36,16 +38,12 @@ func exit() {
 	os.Exit(0)
 }
 
-func getCLIPath(config CLIConfig) string {
+func getCLIPath(config config.ContainerCLIConfig) string {
-	if config.Path != nil {
+	if config.Path != "" {
-		return *config.Path
+		return config.Path
 	}
 
-	var root string
+	if err := os.Setenv("PATH", lookup.GetPath(config.Root)); err != nil {
-	if config.Root != nil {
-		root = *config.Root
-	}
-	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -71,53 +69,62 @@ func doPrestart() {
 	defer exit()
 	log.SetFlags(0)
 
-	hook := getHookConfig()
+	hook, err := getHookConfig()
-	cli := hook.NvidiaContainerCLI
+	if err != nil || hook == nil {
-
+		log.Panicln("error getting hook config:", err)
-	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
 	}
+	cli := hook.NVIDIAContainerCLIConfig
 
-	container := getContainerConfig(hook)
+	container := hook.getContainerConfig()
 	nvidia := container.Nvidia
 	if nvidia == nil {
 		// Not a GPU container, nothing to do.
 		return
 	}
 
+	if !hook.NVIDIAContainerRuntimeHookConfig.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntimeConfig.Mode, container.Image) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
+	}
+
 	rootfs := getRootfsPath(container)
 
 	args := []string{getCLIPath(cli)}
-	if cli.Root != nil {
+	if cli.Root != "" {
-		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
+		args = append(args, fmt.Sprintf("--root=%s", cli.Root))
 	}
 	if cli.LoadKmods {
 		args = append(args, "--load-kmods")
 	}
+	if hook.Features.DisableImexChannelCreation.IsEnabled() {
+		args = append(args, "--no-create-imex-channels")
+	}
 	if cli.NoPivot {
 		args = append(args, "--no-pivot")
 	}
 	if *debugflag {
 		args = append(args, "--debug=/dev/stderr")
-	} else if cli.Debug != nil {
+	} else if cli.Debug != "" {
-		args = append(args, fmt.Sprintf("--debug=%s", *cli.Debug))
+		args = append(args, fmt.Sprintf("--debug=%s", cli.Debug))
 	}
-	if cli.Ldcache != nil {
+	if cli.Ldcache != "" {
-		args = append(args, fmt.Sprintf("--ldcache=%s", *cli.Ldcache))
+		args = append(args, fmt.Sprintf("--ldcache=%s", cli.Ldcache))
 	}
-	if cli.User != nil {
+	if cli.User != "" {
-		args = append(args, fmt.Sprintf("--user=%s", *cli.User))
+		args = append(args, fmt.Sprintf("--user=%s", cli.User))
 	}
 	args = append(args, "configure")
 
-	if cli.Ldconfig != nil {
+	if !hook.Features.AllowCUDACompatLibsFromContainer.IsEnabled() {
-		args = append(args, fmt.Sprintf("--ldconfig=%s", *cli.Ldconfig))
+		args = append(args, "--no-cntlibs")
+	}
+	if ldconfigPath := cli.NormalizeLDConfigPath(); ldconfigPath != "" {
+		args = append(args, fmt.Sprintf("--ldconfig=%s", ldconfigPath))
 	}
 	if cli.NoCgroups {
 		args = append(args, "--no-cgroups")
 	}
-	if len(nvidia.Devices) > 0 {
+	if devicesString := strings.Join(nvidia.Devices, ","); len(devicesString) > 0 {
-		args = append(args, fmt.Sprintf("--device=%s", nvidia.Devices))
+		args = append(args, fmt.Sprintf("--device=%s", devicesString))
 	}
 	if len(nvidia.MigConfigDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-config=%s", nvidia.MigConfigDevices))
@@ -125,6 +132,9 @@ func doPrestart() {
 	if len(nvidia.MigMonitorDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
 	}
+	if imexString := strings.Join(nvidia.ImexChannels, ","); len(imexString) > 0 {
+		args = append(args, fmt.Sprintf("--imex-channel=%s", imexString))
+	}
 
 	for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
 		if len(cap) == 0 {
@@ -133,16 +143,15 @@ func doPrestart() {
 		args = append(args, capabilityToCLI(cap))
 	}
 
-	if !hook.DisableRequire && !nvidia.DisableRequire {
+	for _, req := range nvidia.Requirements {
-		for _, req := range nvidia.Requirements {
+		args = append(args, fmt.Sprintf("--require=%s", req))
-			args = append(args, fmt.Sprintf("--require=%s", req))
-		}
 	}
 
 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
 	args = append(args, rootfs)
 
 	env := append(os.Environ(), cli.Environment...)
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection?
 	err = syscall.Exec(args[0], args, env)
 	log.Panicln("exec failed:", err)
 }
@@ -185,11 +194,11 @@ func main() {
 	}
 }
 
-// logInterceptor implements the info.Logger interface to allow for logging from this function.
+// logInterceptor implements the logger.Interface to allow for logging from executable.
-type logInterceptor struct{}
+type logInterceptor struct {
+	logger.NullLogger
+}
 
 func (l *logInterceptor) Infof(format string, args ...interface{}) {
 	log.Printf(format, args...)
 }
-
-func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/README.md

@@ -85,3 +85,126 @@ Alternatively the NVIDIA Container Runtime can be set as the default runtime for
     }
 }
 ```
+
+## Environment variables (OCI spec)
+
+Each environment variable maps to an command-line argument for `nvidia-container-cli` from [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+These variables are already set in our [official CUDA images](https://hub.docker.com/r/nvidia/cuda/).
+
+### `NVIDIA_VISIBLE_DEVICES`
+This variable controls which GPUs will be made accessible inside the container.
+
+#### Possible values
+* `0,1,2`, `GPU-fef8089b` …: a comma-separated list of GPU UUID(s) or index(es).
+* `all`: all GPUs will be accessible, this is the default value in our container images.
+* `none`: no GPU will be accessible, but driver capabilities will be enabled.
+* `void` or *empty* or *unset*: `nvidia-container-runtime` will have the same behavior as `runc`.
+
+**Note**: When running on a MIG capable device, the following values will also be available:
+* `0:0,0:1,1:0`, `MIG-GPU-fef8089b/0/1` …: a comma-separated list of MIG Device UUID(s) or index(es).
+
+Where the MIG device indices have the form `<GPU Device Index>:<MIG Device Index>` as seen in the example output:
+```
+$ nvidia-smi -L
+GPU 0: Graphics Device (UUID: GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5)
+  MIG Device 0: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/0)
+  MIG Device 1: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/1)
+  MIG Device 2: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/11/0)
+```
+
+### `NVIDIA_MIG_CONFIG_DEVICES`
+This variable controls which of the visible GPUs can have their MIG
+configuration managed from within the container. This includes enabling and
+disabling MIG mode, creating and destroying GPU Instances and Compute
+Instances, etc.
+
+#### Possible values
+* `all`: Allow all MIG-capable GPUs in the visible device list to have their
+  MIG configurations managed.
+
+**Note**:
+* This feature is only available on MIG capable devices (e.g. the A100).
+* To use this feature, the container must be started with `CAP_SYS_ADMIN` privileges.
+* When not running as `root`, the container user must have read access to the
+  `/proc/driver/nvidia/capabilities/mig/config` file on the host.
+
+### `NVIDIA_MIG_MONITOR_DEVICES`
+This variable controls which of the visible GPUs can have aggregate information
+about all of their MIG devices monitored from within the container. This
+includes inspecting the aggregate memory usage, listing the aggregate running
+processes, etc.
+
+#### Possible values
+* `all`: Allow all MIG-capable GPUs in the visible device list to have their
+  MIG devices monitored.
+
+**Note**:
+* This feature is only available on MIG capable devices (e.g. the A100).
+* To use this feature, the container must be started with `CAP_SYS_ADMIN` privileges.
+* When not running as `root`, the container user must have read access to the
+  `/proc/driver/nvidia/capabilities/mig/monitor` file on the host.
+
+### `NVIDIA_DRIVER_CAPABILITIES`
+This option controls which driver libraries/binaries will be mounted inside the container.
+
+#### Possible values
+* `compute,video`, `graphics,utility` …: a comma-separated list of driver features the container needs.
+* `all`: enable all available driver capabilities.
+* *empty* or *unset*: use default driver capability: `utility,compute`.
+
+#### Supported driver capabilities
+* `compute`: required for CUDA and OpenCL applications.
+* `compat32`: required for running 32-bit applications.
+* `graphics`: required for running OpenGL and Vulkan applications.
+* `utility`: required for using `nvidia-smi` and NVML.
+* `video`: required for using the Video Codec SDK.
+* `display`: required for leveraging X11 display.
+
+### `NVIDIA_REQUIRE_*`
+A logical expression to define constraints on the configurations supported by the container.
+
+#### Supported constraints
+* `cuda`: constraint on the CUDA driver version.
+* `driver`: constraint on the driver version.
+* `arch`: constraint on the compute architectures of the selected GPUs.
+* `brand`: constraint on the brand of the selected GPUs (e.g. GeForce, Tesla, GRID).
+
+#### Expressions
+Multiple constraints can be expressed in a single environment variable: space-separated constraints are ORed, comma-separated constraints are ANDed.
+Multiple environment variables of the form `NVIDIA_REQUIRE_*` are ANDed together.
+
+### `NVIDIA_DISABLE_REQUIRE`
+Single switch to disable all the constraints of the form `NVIDIA_REQUIRE_*`.
+
+### `NVIDIA_REQUIRE_CUDA`
+
+The version of the CUDA toolkit used by the container. It is an instance of the generic `NVIDIA_REQUIRE_*` case and it is set by official CUDA images.
+If the version of the NVIDIA driver is insufficient to run this version of CUDA, the container will not be started.
+
+#### Possible values
+* `cuda>=7.5`, `cuda>=8.0`, `cuda>=9.0` …: any valid CUDA version in the form `major.minor`.
+
+### `CUDA_VERSION`
+Similar to `NVIDIA_REQUIRE_CUDA`, for legacy CUDA images.
+In addition, if `NVIDIA_REQUIRE_CUDA` is not set, `NVIDIA_VISIBLE_DEVICES` and `NVIDIA_DRIVER_CAPABILITIES` will default to `all`.
+
+## Usage example
+
+**NOTE:** The use of the `nvidia-container-runtime` as CLI replacement for `runc` is uncommon and is only provided for completeness.
+
+Although the `nvidia-container-runtime` is typically configured as a replacement for `runc` or `crun` in various container engines, it can also be
+invoked from the command line as `runc` would. For example:
+
+```sh
+# Setup a rootfs based on Ubuntu 16.04
+cd $(mktemp -d) && mkdir rootfs
+curl -sS http://cdimage.ubuntu.com/ubuntu-base/releases/16.04/release/ubuntu-base-16.04.6-base-amd64.tar.gz | tar --exclude 'dev/*' -C rootfs -xz
+
+# Create an OCI runtime spec
+nvidia-container-runtime spec
+sed -i 's;"sh";"nvidia-smi";' config.json
+sed -i 's;\("TERM=xterm"\);\1, "NVIDIA_VISIBLE_DEVICES=0";' config.json
+
+# Run the container
+sudo nvidia-container-runtime run nvidia_smi
+```

cmd/nvidia-container-runtime/main.go

@@ -1,84 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
-// version must be set by go build's -X main.version= option in the Makefile.
-var version = "unknown"
-
-// gitCommit will be the hash that the binary was built from
-// and will be populated by the Makefile
-var gitCommit = ""
-
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	printVersion := hasVersionFlag(argv)
-	if printVersion {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
-	}
-
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	logger, err = UpdateLogger(
-		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
-		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
-		argv,
-	)
-	if err != nil {
-		return fmt.Errorf("failed to set up logger: %v", err)
-	}
-	defer logger.Reset()
-
-	logger.Debugf("Command line arguments: %v", argv)
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	if printVersion {
-		fmt.Print("\n")
-	}
-	return runtime.Exec(argv)
-}
-
-// TODO: This should be refactored / combined with parseArgs in logger.
-func hasVersionFlag(args []string) bool {
-	for i := 0; i < len(args); i++ {
-		param := args[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		trimmed := strings.TrimLeft(parts[0], "-")
-		// If this is not a flag we continue
-		if parts[0] == trimmed {
-			continue
-		}
-
-		// Check the version flag
-		if trimmed == "version" {
-			return true
-		}
-	}
-
-	return false
-}

cmd/nvidia-container-runtime/main_test.go

@@ -3,25 +3,28 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"io/ioutil"
+	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 )
 
 const (
 	nvidiaRuntime            = "nvidia-container-runtime"
 	nvidiaHook               = "nvidia-container-runtime-hook"
-	bundlePathSuffix         = "test/output/bundle/"
+	bundlePathSuffix         = "tests/output/bundle/"
 	specFile                 = "config.json"
-	unmodifiedSpecFileSuffix = "test/input/test_spec.json"
+	unmodifiedSpecFileSuffix = "tests/input/test_spec.json"
 )
 
 const (
@@ -41,10 +44,10 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		log.Fatalf("error in test setup: could not get module root: %v", err)
 	}
-	testBinPath := filepath.Join(moduleRoot, "test", "bin")
+	testBinPath := filepath.Join(moduleRoot, "tests", "bin")
-	testInputPath := filepath.Join(moduleRoot, "test", "input")
+	testInputPath := filepath.Join(moduleRoot, "tests", "input")
 
 	// Set the environment variables for the test
 	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
@@ -53,11 +56,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -77,13 +80,14 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	err = cmdCreate.Run()
@@ -91,15 +95,17 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
 		t.Fatalf("error generating runtime spec: %v", err)
 	}
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle", cfg.bundlePath(), "testcontainer")
 	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
 	output, err := cmdRun.CombinedOutput()
@@ -110,6 +116,7 @@ func TestGoodInput(t *testing.T) {
 	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
 	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	err = cmdCreate.Run()
@@ -155,6 +162,7 @@ func TestDuplicateHook(t *testing.T) {
 	}
 
 	// Test how runtime handles already existing prestart hook in config.json
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	output, err := cmdCreate.CombinedOutput()
@@ -170,7 +178,8 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	logger, _ := testlog.NewNullLogger()
+	m := modifier.NewStableRuntimeModifier(logger, nvidiaHook)
 	return m.Modify(spec)
 }
 
@@ -184,15 +193,16 @@ func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
 	}
 	defer jsonFile.Close()
 
-	jsonContent, err := ioutil.ReadAll(jsonFile)
+	jsonContent, err := io.ReadAll(jsonFile)
-	if err != nil {
+	switch {
+	case err != nil:
 		return spec, err
-	} else if json.Valid(jsonContent) {
+	case json.Valid(jsonContent):
 		err = json.Unmarshal(jsonContent, &spec)
 		if err != nil {
 			return spec, err
 		}
-	} else {
+	default:
 		err = json.NewDecoder(bytes.NewReader(jsonContent)).Decode(&spec)
 		if err != nil {
 			return spec, err
@@ -222,6 +232,7 @@ func (c testConfig) generateNewRuntimeSpec() error {
 		return err
 	}
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmd := exec.Command("cp", c.unmodifiedSpecFile(), c.specFilePath())
 	err = cmd.Run()
 	if err != nil {

cmd/nvidia-container-runtime/modifier/csv.go

@@ -1,166 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/cuda"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/requirements"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
-)
-
-// csvMode represents the modifications as performed by the csv runtime mode
-type csvMode struct {
-	logger     *logrus.Logger
-	discoverer discover.Discover
-}
-
-const (
-	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
-	visibleDevicesVoid   = "void"
-
-	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
-)
-
-// NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
-// The modifications are defined by CSV MountSpecs.
-func NewCSVModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	rawSpec, err := ociSpec.Load()
-	if err != nil {
-		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
-	// if no modification is required.
-	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
-	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
-		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
-		return nil, nil
-	}
-	logger.Infof("Constructing modifier from config: %+v", *cfg)
-
-	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
-	}
-
-	// TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the
-	// visible devices are checked.
-	image, err := image.NewCUDAImageFromSpec(rawSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := checkRequirements(logger, &image); err != nil {
-		return nil, fmt.Errorf("requirements not met: %v", err)
-	}
-
-	csvFiles, err := csv.GetFileList(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.MountSpecPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
-	}
-
-	nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
-	if nvidiaRequireJetpack != "csv-mounts=all" {
-		csvFiles = csv.BaseFilesOnly(csvFiles)
-	}
-
-	csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
-	}
-
-	ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
-	}
-
-	createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, csvDiscoverer, config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
-	}
-
-	d := discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
-
-	return newModifierFromDiscoverer(logger, d)
-}
-
-// newModifierFromDiscoverer created a modifier that aplies the discovered
-// modifications to an OCI spec if require by the runtime wrapper.
-func newModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
-	m := csvMode{
-		logger:     logger,
-		discoverer: d,
-	}
-	return &m, nil
-}
-
-// Modify applies the required modifications to the incomming OCI spec. These modifications
-// are applied in-place.
-func (m csvMode) Modify(spec *specs.Spec) error {
-	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
-	if err != nil {
-		return fmt.Errorf("failed to remove existing hooks: %v", err)
-	}
-
-	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
-	if err != nil {
-		return fmt.Errorf("failed to get required container edits: %v", err)
-	}
-
-	return specEdits.Modify(spec)
-}
-
-func checkRequirements(logger *logrus.Logger, image *image.CUDA) error {
-	if image.HasDisableRequire() {
-		// TODO: We could print the real value here instead
-		logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true)
-		return nil
-	}
-
-	imageRequirements, err := image.GetRequirements()
-	if err != nil {
-		//  TODO: Should we treat this as a failure, or just issue a warning?
-		return fmt.Errorf("failed to get image requirements: %v", err)
-	}
-
-	r := requirements.New(logger, imageRequirements)
-
-	cudaVersion, err := cuda.Version()
-	if err != nil {
-		logger.Warnf("Failed to get CUDA version: %v", err)
-	} else {
-		r.AddVersionProperty(requirements.CUDA, cudaVersion)
-	}
-
-	compteCapability, err := cuda.ComputeCapability(0)
-	if err != nil {
-		logger.Warnf("Failed to get CUDA Compute Capability: %v", err)
-	} else {
-		r.AddVersionProperty(requirements.ARCH, compteCapability)
-	}
-
-	return r.Assert()
-}

cmd/nvidia-container-runtime/modifier/csv_test.go

@@ -1,284 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewCSVModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description    string
-		cfg            *config.Config
-		spec           oci.Spec
-		visibleDevices string
-		expectedError  error
-		expectedNil    bool
-	}{
-		{
-			description: "spec load error returns error",
-			spec: &oci.SpecMock{
-				LoadFunc: func() (*specs.Spec, error) {
-					return nil, fmt.Errorf("load failed")
-				},
-			},
-			expectedError: fmt.Errorf("load failed"),
-		},
-		{
-			description:    "visible devices not set returns nil",
-			visibleDevices: "NOT_SET",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices empty returns nil",
-			visibleDevices: "",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices 'void' returns nil",
-			visibleDevices: "void",
-			expectedNil:    true,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			spec := tc.spec
-			if spec == nil {
-				spec = &oci.SpecMock{
-					LookupEnvFunc: func(s string) (string, bool) {
-						if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
-							return tc.visibleDevices, true
-						}
-						return "", false
-					},
-				}
-			}
-
-			m, err := NewCSVModifier(logger, tc.cfg, spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			if tc.expectedNil || tc.expectedError != nil {
-				require.Nil(t, m)
-			} else {
-				require.NotNil(t, m)
-			}
-		})
-	}
-}
-
-func TestExperimentalModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description   string
-		discover      *discover.DiscoverMock
-		spec          *specs.Spec
-		expectedError error
-		expectedSpec  *specs.Spec
-	}{
-		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
-		},
-		{
-			description: "failed hooks discoverer returns error",
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					return nil, fmt.Errorf("discover.Hooks error")
-				},
-			},
-			expectedError: fmt.Errorf("discover.Hooks error"),
-		},
-		{
-			description: "discovered hooks are injected into spec",
-			spec:        &specs.Spec{},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/a",
-							Args:      []string{"/hook/a", "arga"},
-						},
-						{
-							Lifecycle: "createContainer",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-					CreateContainer: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "existing hooks are maintained",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-runtime-hook",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-runtime-hook",
-							Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-toolkit",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-toolkit",
-							Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			m, err := newModifierFromDiscoverer(logger, tc.discover)
-			require.NoError(t, err)
-
-			err = m.Modify(tc.spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			require.EqualValues(t, tc.expectedSpec, tc.spec)
-		})
-	}
-}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -1,73 +0,0 @@
-/*
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
-	"github.com/sirupsen/logrus"
-)
-
-// newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger
-func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) {
-	lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, cfg.NVIDIAContainerRuntimeConfig.Runtimes)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
-	}
-
-	if !oci.HasCreateSubcommand(argv) {
-		logger.Debugf("Skipping modifier for non-create subcommand")
-		return lowLevelRuntime, nil
-	}
-
-	ociSpec, err := oci.NewSpec(logger, argv)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
-	}
-
-	specModifier, err := newSpecModifier(logger, cfg, ociSpec, argv)
-	if err != nil {
-		return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
-	}
-
-	// Create the wrapping runtime with the specified modifier
-	r := runtime.NewModifyingRuntimeWrapper(
-		logger,
-		lowLevelRuntime,
-		ociSpec,
-		specModifier,
-	)
-
-	return r, nil
-}
-
-// newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
-func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
-	switch info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode) {
-	case "legacy":
-		return modifier.NewStableRuntimeModifier(logger), nil
-	case "csv":
-		return modifier.NewCSVModifier(logger, cfg, ociSpec)
-	}
-
-	return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)
-}

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -1,129 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"encoding/json"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestFactoryMethod(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description   string
-		cfg           *config.Config
-		spec          *specs.Spec
-		expectedError bool
-	}{
-		{
-			description: "empty config raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
-			},
-			expectedError: true,
-		},
-		{
-			description: "config with runtime raises no error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Runtimes: []string{"runc"},
-					Mode:     "legacy",
-				},
-			},
-		},
-		{
-			description: "csv mode is supported",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Runtimes: []string{"runc"},
-					Mode:     "csv",
-				},
-			},
-			spec: &specs.Spec{
-				Process: &specs.Process{
-					Env: []string{
-						"NVIDIA_VISIBLE_DEVICES=all",
-					},
-				},
-			},
-		},
-		{
-			description: "non-legacy discover mode raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Runtimes: []string{"runc"},
-					Mode:     "non-legacy",
-				},
-			},
-			expectedError: true,
-		},
-		{
-			description: "legacy discover mode returns modifier",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Runtimes: []string{"runc"},
-					Mode:     "legacy",
-				},
-			},
-		},
-		{
-			description: "csv discover mode returns modifier",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Runtimes: []string{"runc"},
-					Mode:     "csv",
-				},
-			},
-		},
-		{
-			description: "empty mode raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Runtimes: []string{"runc"},
-				},
-			},
-			expectedError: true,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			bundleDir := t.TempDir()
-
-			specFile, err := os.Create(filepath.Join(bundleDir, "config.json"))
-			require.NoError(t, err)
-			require.NoError(t, json.NewEncoder(specFile).Encode(tc.spec))
-
-			argv := []string{"--bundle", bundleDir, "create"}
-
-			_, err = newNVIDIAContainerRuntime(logger, tc.cfg, argv)
-			if tc.expectedError {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-		})
-	}
-}

cmd/nvidia-container-toolkit/capabilities.go

@@ -1,83 +0,0 @@
-package main
-
-import (
-	"log"
-	"strings"
-)
-
-const (
-	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
-	defaultDriverCapabilities = DriverCapabilities("utility,compute")
-
-	none = DriverCapabilities("")
-	all  = DriverCapabilities("all")
-)
-
-func capabilityToCLI(cap string) string {
-	switch cap {
-	case "compute":
-		return "--compute"
-	case "compat32":
-		return "--compat32"
-	case "graphics":
-		return "--graphics"
-	case "utility":
-		return "--utility"
-	case "video":
-		return "--video"
-	case "display":
-		return "--display"
-	case "ngx":
-		return "--ngx"
-	default:
-		log.Panicln("unknown driver capability:", cap)
-	}
-	return ""
-}
-
-// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
-// variable. Operations include default values, filtering, and handling meta values such as "all"
-type DriverCapabilities string
-
-// Intersection returns intersection between two sets of capabilities.
-func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
-	if capabilities == all {
-		return d
-	}
-	if d == all {
-		return capabilities
-	}
-
-	lookup := make(map[string]bool)
-	for _, c := range d.list() {
-		lookup[c] = true
-	}
-	var found []string
-	for _, c := range capabilities.list() {
-		if lookup[c] {
-			found = append(found, c)
-		}
-	}
-
-	intersection := DriverCapabilities(strings.Join(found, ","))
-	return intersection
-}
-
-// String returns the string representation of the driver capabilities
-func (d DriverCapabilities) String() string {
-	return string(d)
-}
-
-// list returns the driver capabilities as a list
-func (d DriverCapabilities) list() []string {
-	var caps []string
-	for _, c := range strings.Split(string(d), ",") {
-		trimmed := strings.TrimSpace(c)
-		if len(trimmed) == 0 {
-			continue
-		}
-		caps = append(caps, trimmed)
-	}
-
-	return caps
-}

cmd/nvidia-container-toolkit/capabilities_test.go

@@ -1,134 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestDriverCapabilitiesIntersection(t *testing.T) {
-	testCases := []struct {
-		capabilities          DriverCapabilities
-		supportedCapabilities DriverCapabilities
-		expectedIntersection  DriverCapabilities
-	}{
-		{
-			capabilities:          none,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          allDriverCapabilities,
-			supportedCapabilities: all,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: all,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: DriverCapabilities("cap1"),
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("cap0,cap1"),
-			supportedCapabilities: DriverCapabilities("cap1,cap0"),
-			expectedIntersection:  DriverCapabilities("cap0,cap1"),
-		},
-		{
-			capabilities:          defaultDriverCapabilities,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  defaultDriverCapabilities,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-		{
-			capabilities:          DriverCapabilities("cap1"),
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
-			require.EqualValues(t, tc.expectedIntersection, intersection)
-		})
-	}
-}
-
-func TestDriverCapabilitiesList(t *testing.T) {
-	testCases := []struct {
-		capabilities DriverCapabilities
-		expected     []string
-	}{
-		{
-			capabilities: DriverCapabilities(""),
-		},
-		{
-			capabilities: DriverCapabilities("  "),
-		},
-		{
-			capabilities: DriverCapabilities(","),
-		},
-		{
-			capabilities: DriverCapabilities(",cap"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap,"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap0,,cap1"),
-			expected:     []string{"cap0", "cap1"},
-		},
-		{
-			capabilities: DriverCapabilities("cap1,cap0,cap3"),
-			expected:     []string{"cap1", "cap0", "cap3"},
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			require.EqualValues(t, tc.expected, tc.capabilities.list())
-		})
-	}
-}

cmd/nvidia-container-toolkit/container_config_test.go

@@ -1,989 +0,0 @@
-package main
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetNvidiaConfig(t *testing.T) {
-	var tests = []struct {
-		description    string
-		env            map[string]string
-		privileged     bool
-		hookConfig     *HookConfig
-		expectedConfig *nvidiaConfig
-		expectedPanic  bool
-	}{
-		{
-			description:    "No environment, unprivileged",
-			env:            map[string]string{},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description:    "No environment, privileged",
-			env:            map[string]string{},
-			privileged:     true,
-			expectedConfig: nil,
-		},
-		{
-			description: "Legacy image, no devices, no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion: "9.0",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices 'all', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices 'empty', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Legacy image, devices 'void', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Legacy image, devices 'none', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "none",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "gpu0,gpu1",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities 'empty', no requirements",
-			env: map[string]string{
-				envCUDAVersion:          "9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities 'all', no requirements",
-			env: map[string]string{
-				envCUDAVersion:          "9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities set, no requirements",
-			env: map[string]string{
-				envCUDAVersion:          "9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "video,display",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities set, requirements set",
-			env: map[string]string{
-				envCUDAVersion:              "9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities set, requirements set, disable requirements",
-			env: map[string]string{
-				envCUDAVersion:              "9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-				envNVDisableRequire:         "true",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
-			},
-		},
-		{
-			description: "Modern image, no devices, no capabilities, no requirements, no envCUDAVersion",
-			env: map[string]string{
-				envNVRequireCUDA: "cuda>=9.0",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, no devices, no capabilities, no requirement, envCUDAVersion set",
-			env: map[string]string{
-				envCUDAVersion:   "9.0",
-				envNVRequireCUDA: "cuda>=9.0",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, devices 'all', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'empty', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, devices 'void', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, devices 'none', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "none",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "gpu0,gpu1",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities 'empty', no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:        "cuda>=9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities 'all', no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:        "cuda>=9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities set, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:        "cuda>=9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "video,display",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities set, requirements set",
-			env: map[string]string{
-				envNVRequireCUDA:            "cuda>=9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities set, requirements set, disable requirements",
-			env: map[string]string{
-				envNVRequireCUDA:            "cuda>=9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-				envNVDisableRequire:         "true",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
-			},
-		},
-		{
-			description: "No cuda envs, devices 'all'",
-			env: map[string]string{
-				envNVVisibleDevices: "all",
-			},
-			privileged: false,
-
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'all', migConfig set, privileged",
-			env: map[string]string{
-				envNVRequireCUDA:      "cuda>=9.0",
-				envNVVisibleDevices:   "all",
-				envNVMigConfigDevices: "mig0,mig1",
-			},
-			privileged: true,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				MigConfigDevices:   "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'all', migConfig set, unprivileged",
-			env: map[string]string{
-				envNVRequireCUDA:      "cuda>=9.0",
-				envNVVisibleDevices:   "all",
-				envNVMigConfigDevices: "mig0,mig1",
-			},
-			privileged:    false,
-			expectedPanic: true,
-		},
-		{
-			description: "Modern image, devices 'all', migMonitor set, privileged",
-			env: map[string]string{
-				envNVRequireCUDA:       "cuda>=9.0",
-				envNVVisibleDevices:    "all",
-				envNVMigMonitorDevices: "mig0,mig1",
-			},
-			privileged: true,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				MigMonitorDevices:  "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'all', migMonitor set, unprivileged",
-			env: map[string]string{
-				envNVRequireCUDA:       "cuda>=9.0",
-				envNVVisibleDevices:    "all",
-				envNVMigMonitorDevices: "mig0,mig1",
-			},
-			privileged:    false,
-			expectedPanic: true,
-		},
-		{
-			description: "Hook config set as driver-capabilities-all",
-			env: map[string]string{
-				envNVVisibleDevices:     "all",
-				envNVDriverCapabilities: "all",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SupportedDriverCapabilities: "video,display",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: "video,display",
-			},
-		},
-		{
-			description: "Hook config set, envvar sets driver-capabilities",
-			env: map[string]string{
-				envNVVisibleDevices:     "all",
-				envNVDriverCapabilities: "video,display",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SupportedDriverCapabilities: "video,display,compute,utility",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: "video,display",
-			},
-		},
-		{
-			description: "Hook config set, envvar unset sets default driver-capabilities",
-			env: map[string]string{
-				envNVVisibleDevices: "all",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SupportedDriverCapabilities: "video,display,utility,compute",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-			},
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			// Wrap the call to getNvidiaConfig() in a closure.
-			var config *nvidiaConfig
-			getConfig := func() {
-				hookConfig := tc.hookConfig
-				if hookConfig == nil {
-					defaultConfig := getDefaultHookConfig()
-					hookConfig = &defaultConfig
-				}
-				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
-			}
-
-			// For any tests that are expected to panic, make sure they do.
-			if tc.expectedPanic {
-				require.Panics(t, getConfig)
-				return
-			}
-
-			// For all other tests, just grab the config
-			getConfig()
-
-			// And start comparing the test results to the expected results.
-			if tc.expectedConfig == nil {
-				require.Nil(t, config, tc.description)
-				return
-			}
-
-			require.NotNil(t, config, tc.description)
-
-			require.Equal(t, tc.expectedConfig.Devices, config.Devices)
-			require.Equal(t, tc.expectedConfig.MigConfigDevices, config.MigConfigDevices)
-			require.Equal(t, tc.expectedConfig.MigMonitorDevices, config.MigMonitorDevices)
-			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
-
-			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
-			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
-		})
-	}
-}
-
-func TestGetDevicesFromMounts(t *testing.T) {
-	var tests = []struct {
-		description     string
-		mounts          []Mount
-		expectedDevices *string
-	}{
-		{
-			description:     "No mounts",
-			mounts:          nil,
-			expectedDevices: nil,
-		},
-		{
-			description: "Host path is not /dev/null",
-			mounts: []Mount{
-				{
-					Source:      "/not/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-			},
-			expectedDevices: nil,
-		},
-		{
-			description: "Container path is not prefixed by 'root'",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join("/other/prefix", "GPU0"),
-				},
-			},
-			expectedDevices: nil,
-		},
-		{
-			description: "Container path is only 'root'",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: deviceListAsVolumeMountsRoot,
-				},
-			},
-			expectedDevices: nil,
-		},
-		{
-			description: "Discover 2 devices",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			expectedDevices: &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description: "Discover 2 devices with slashes in the name",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0-MIG0/0/1"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1-MIG0/0/1"),
-				},
-			},
-			expectedDevices: &[]string{"GPU0-MIG0/0/1,GPU1-MIG0/0/1"}[0],
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			devices := getDevicesFromMounts(tc.mounts)
-			require.Equal(t, tc.expectedDevices, devices)
-		})
-	}
-}
-
-func TestDeviceListSourcePriority(t *testing.T) {
-	var tests = []struct {
-		description        string
-		mountDevices       []Mount
-		envvarDevices      string
-		privileged         bool
-		acceptUnprivileged bool
-		acceptMounts       bool
-		expectedDevices    *string
-	}{
-		{
-			description: "Mount devices, unprivileged, no accept unprivileged",
-			mountDevices: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			envvarDevices:      "GPU2,GPU3",
-			privileged:         false,
-			acceptUnprivileged: false,
-			acceptMounts:       true,
-			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description:        "No mount devices, unprivileged, no accept unprivileged",
-			mountDevices:       nil,
-			envvarDevices:      "GPU0,GPU1",
-			privileged:         false,
-			acceptUnprivileged: false,
-			acceptMounts:       true,
-			expectedDevices:    nil,
-		},
-		{
-			description:        "No mount devices, privileged, no accept unprivileged",
-			mountDevices:       nil,
-			envvarDevices:      "GPU0,GPU1",
-			privileged:         true,
-			acceptUnprivileged: false,
-			acceptMounts:       true,
-			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description:        "No mount devices, unprivileged, accept unprivileged",
-			mountDevices:       nil,
-			envvarDevices:      "GPU0,GPU1",
-			privileged:         false,
-			acceptUnprivileged: true,
-			acceptMounts:       true,
-			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description: "Mount devices, unprivileged, accept unprivileged, no accept mounts",
-			mountDevices: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			envvarDevices:      "GPU2,GPU3",
-			privileged:         false,
-			acceptUnprivileged: true,
-			acceptMounts:       false,
-			expectedDevices:    &[]string{"GPU2,GPU3"}[0],
-		},
-		{
-			description: "Mount devices, unprivileged, no accept unprivileged, no accept mounts",
-			mountDevices: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			envvarDevices:      "GPU2,GPU3",
-			privileged:         false,
-			acceptUnprivileged: false,
-			acceptMounts:       false,
-			expectedDevices:    nil,
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			// Wrap the call to getDevices() in a closure.
-			var devices *string
-			getDevices := func() {
-				env := map[string]string{
-					envNVVisibleDevices: tc.envvarDevices,
-				}
-				hookConfig := getDefaultHookConfig()
-				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
-				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
-			}
-
-			// For all other tests, just grab the devices and check the results
-			getDevices()
-
-			require.Equal(t, tc.expectedDevices, devices)
-		})
-	}
-}
-
-func TestGetDevicesFromEnvvar(t *testing.T) {
-	all := "all"
-	empty := ""
-	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
-	gpuID := "GPU-12345"
-	anotherGPUID := "GPU-67890"
-
-	var tests = []struct {
-		description     string
-		envSwarmGPU     *string
-		env             map[string]string
-		legacyImage     bool
-		expectedDevices *string
-	}{
-		{
-			description: "empty env returns nil for non-legacy image",
-		},
-		{
-			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: "",
-			},
-		},
-		{
-			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: "void",
-			},
-		},
-		{
-			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: "none",
-			},
-			expectedDevices: &empty,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: gpuID,
-			},
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: gpuID,
-			},
-			legacyImage:     true,
-			expectedDevices: &gpuID,
-		},
-		{
-			description:     "empty env returns all for legacy image",
-			legacyImage:     true,
-			expectedDevices: &all,
-		},
-		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
-		// not enabled
-		{
-			description: "missing NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envDockerResourceGPUs: anotherGPUID,
-			},
-		},
-		{
-			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   "",
-				envDockerResourceGPUs: anotherGPUID,
-			},
-		},
-		{
-			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   "void",
-				envDockerResourceGPUs: anotherGPUID,
-			},
-		},
-		{
-			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   "none",
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &empty,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   gpuID,
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   gpuID,
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			legacyImage:     true,
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "empty env returns all for legacy image",
-			env: map[string]string{
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			legacyImage:     true,
-			expectedDevices: &all,
-		},
-		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
-		// enabled
-		{
-			description: "empty env returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-		},
-		{
-			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: "",
-			},
-		},
-		{
-			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: "void",
-			},
-		},
-		{
-			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: "none",
-			},
-			expectedDevices: &empty,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: gpuID,
-			},
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: gpuID,
-			},
-			legacyImage:     true,
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS is selected if present",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &anotherGPUID,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envNVVisibleDevices:   gpuID,
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &anotherGPUID,
-		},
-	}
-
-	for i, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			envSwarmGPU = tc.envSwarmGPU
-			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
-			if tc.expectedDevices == nil {
-				require.Nil(t, devices, "%d: %v", i, tc)
-				return
-			}
-
-			require.NotNil(t, devices, "%d: %v", i, tc)
-			require.Equal(t, *tc.expectedDevices, *devices, "%d: %v", i, tc)
-		})
-	}
-}
-
-func TestGetDriverCapabilities(t *testing.T) {
-
-	supportedCapabilities := "compute,utility,display,video"
-
-	testCases := []struct {
-		description           string
-		env                   map[string]string
-		legacyImage           bool
-		supportedCapabilities string
-		expectedPanic         bool
-		expectedCapabilities  string
-	}{
-		{
-			description: "Env is set for legacy image",
-			env: map[string]string{
-				envNVDriverCapabilities: "display,video",
-			},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  "display,video",
-		},
-		{
-			description: "Env is all for legacy image",
-			env: map[string]string{
-				envNVDriverCapabilities: "all",
-			},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  supportedCapabilities,
-		},
-		{
-			description: "Env is empty for legacy image",
-			env: map[string]string{
-				envNVDriverCapabilities: "",
-			},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
-		},
-		{
-			description:           "Env unset for legacy image is 'all'",
-			env:                   map[string]string{},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  supportedCapabilities,
-		},
-		{
-			description: "Env is set for modern image",
-			env: map[string]string{
-				envNVDriverCapabilities: "display,video",
-			},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  "display,video",
-		},
-		{
-			description:           "Env unset for modern image is default",
-			env:                   map[string]string{},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
-		},
-		{
-			description: "Env is all for modern image",
-			env: map[string]string{
-				envNVDriverCapabilities: "all",
-			},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  supportedCapabilities,
-		},
-		{
-			description: "Env is empty for modern image",
-			env: map[string]string{
-				envNVDriverCapabilities: "",
-			},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
-		},
-		{
-			description: "Invalid capabilities panic",
-			env: map[string]string{
-				envNVDriverCapabilities: "compute,utility",
-			},
-			supportedCapabilities: "not-compute,not-utility",
-			expectedPanic:         true,
-		},
-		{
-			description:           "Default is restricted for modern image",
-			legacyImage:           false,
-			supportedCapabilities: "compute",
-			expectedCapabilities:  "compute",
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			var capabilites DriverCapabilities
-
-			getDriverCapabilities := func() {
-				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
-				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
-			}
-
-			if tc.expectedPanic {
-				require.Panics(t, getDriverCapabilities)
-				return
-			}
-
-			getDriverCapabilities()
-			require.EqualValues(t, tc.expectedCapabilities, capabilites)
-		})
-	}
-}

cmd/nvidia-container-toolkit/hook_config.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"log"
-	"os"
-	"path"
-	"reflect"
-
-	"github.com/BurntSushi/toml"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-)
-
-const (
-	configPath = "/etc/nvidia-container-runtime/config.toml"
-	driverPath = "/run/nvidia/driver"
-)
-
-var defaultPaths = [...]string{
-	path.Join(driverPath, configPath),
-	configPath,
-}
-
-// CLIConfig : options for nvidia-container-cli.
-type CLIConfig struct {
-	Root        *string  `toml:"root"`
-	Path        *string  `toml:"path"`
-	Environment []string `toml:"environment"`
-	Debug       *string  `toml:"debug"`
-	Ldcache     *string  `toml:"ldcache"`
-	LoadKmods   bool     `toml:"load-kmods"`
-	NoPivot     bool     `toml:"no-pivot"`
-	NoCgroups   bool     `toml:"no-cgroups"`
-	User        *string  `toml:"user"`
-	Ldconfig    *string  `toml:"ldconfig"`
-}
-
-// HookConfig : options for the nvidia-container-toolkit.
-type HookConfig struct {
-	DisableRequire                 bool               `toml:"disable-require"`
-	SwarmResource                  *string            `toml:"swarm-resource"`
-	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
-	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
-
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-func getDefaultHookConfig() HookConfig {
-	return HookConfig{
-		DisableRequire:                 false,
-		SwarmResource:                  nil,
-		AcceptEnvvarUnprivileged:       true,
-		AcceptDeviceListAsVolumeMounts: false,
-		SupportedDriverCapabilities:    allDriverCapabilities,
-		NvidiaContainerCLI: CLIConfig{
-			Root:        nil,
-			Path:        nil,
-			Environment: []string{},
-			Debug:       nil,
-			Ldcache:     nil,
-			LoadKmods:   true,
-			NoPivot:     false,
-			NoCgroups:   false,
-			User:        nil,
-			Ldconfig:    nil,
-		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
-	}
-}
-
-func getHookConfig() (config HookConfig) {
-	var err error
-
-	if len(*configflag) > 0 {
-		config = getDefaultHookConfig()
-		_, err = toml.DecodeFile(*configflag, &config)
-		if err != nil {
-			log.Panicln("couldn't open configuration file:", err)
-		}
-	} else {
-		for _, p := range defaultPaths {
-			config = getDefaultHookConfig()
-			_, err = toml.DecodeFile(p, &config)
-			if err == nil {
-				break
-			} else if !os.IsNotExist(err) {
-				log.Panicln("couldn't open default configuration file:", err)
-			}
-		}
-	}
-
-	if config.SupportedDriverCapabilities == all {
-		config.SupportedDriverCapabilities = allDriverCapabilities
-	}
-	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
-	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
-		configName := config.getConfigOption("SupportedDriverCapabilities")
-		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
-	}
-
-	return config
-}
-
-// getConfigOption returns the toml config option associated with the
-// specified struct field.
-func (c HookConfig) getConfigOption(fieldName string) string {
-	t := reflect.TypeOf(c)
-	f, ok := t.FieldByName(fieldName)
-	if !ok {
-		return fieldName
-	}
-	v, ok := f.Tag.Lookup("toml")
-	if !ok {
-		return fieldName
-	}
-	return v
-}

cmd/nvidia-ctk-installer/container/README.md

@@ -15,28 +15,23 @@ docker setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`.
 
 Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
 
-**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
 
 | Flags                                                       | Installed Runtimes              | Default Runtime       |
 |-------------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| **NONE SPECIFIED**                                          | `nvidia`                        | `nvidia`              |
-| `--runtime-name nvidia`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--runtime-name nvidia`                                     | `nvidia`                        | `nvidia`              |
-| `--runtime-name NAME`                                       | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--runtime-name NAME`                                       | `NAME`                          | `NAME`                |
-| `--runtime-name nvidia-experimental`                        | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default`                                          | `nvidia`                        | `nvidia`              |
-| `--set-as-default`                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name nvidia`                    | `nvidia`                        | `nvidia`              |
-| `--set-as-default --runtime-name nvidia`                    | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name NAME`                      | `NAME`                          | `NAME`                |
-| `--set-as-default --runtime-name NAME`                      | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default=false`                                    | `nvidia`                        | **NOT SET**           |
-| `--set-as-default --runtime-name nvidia-experimental`       | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default=false --runtime-name NAME`                | `NAME`                          | **NOT SET**           |
-| `--set-as-default=false`                                    | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia`              | `nvidia`                        | **NOT SET**           |
-| `--set-as-default=false --runtime-name NAME`                | `NAME`, `nvidia-experimental`   | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia`              | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET**           |
 
 These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
 
@@ -48,7 +43,7 @@ containerd setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`.
 
 Adding the `--set-as-default` flag as follows:
 ```bash
@@ -59,19 +54,15 @@ containerd setup \
 ```
 will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
 
-**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
 
 | Flags                                                  | Installed Runtime Classes       | Default Runtime Class |
 |--------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                     | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| **NONE SPECIFIED**                                     | `nvidia`                        | **NOT SET**           |
-| `--runtime-class NAME`                                 | `NAME`, `nvidia-experimental`   | **NOT SET**           |
+| `--runtime-class NAME`                                 | `NAME`                          | **NOT SET**           |
-| `--runtime-class nvidia`                               | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--runtime-class nvidia`                               | `nvidia`                        | **NOT SET**           |
-| `--runtime-class nvidia-experimental`                  | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default`                                     | `nvidia`                        | `nvidia`              |
-| `--set-as-default`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-class NAME`                | `NAME`                          | `NAME`                |
-| `--set-as-default --runtime-class NAME`                | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default --runtime-class nvidia`              | `nvidia`                        | `nvidia`              |
-| `--set-as-default --runtime-class nvidia`              | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
 
 These combinations also hold for the environment variables that map to the command line flags.

cmd/nvidia-ctk-installer/container/container.go

@@ -0,0 +1,177 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package container
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+)
+
+const (
+	restartModeNone    = "none"
+	restartModeSignal  = "signal"
+	restartModeSystemd = "systemd"
+)
+
+// Options defines the shared options for the CLIs to configure containers runtimes.
+type Options struct {
+	Config string
+	Socket string
+	// EnabledCDI indicates whether CDI should be enabled.
+	EnableCDI     bool
+	RuntimeName   string
+	RuntimeDir    string
+	SetAsDefault  bool
+	RestartMode   string
+	HostRootMount string
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context, o *Options) error {
+	if o.RuntimeDir != "" {
+		logrus.Debug("Runtime directory already set; ignoring arguments")
+		return nil
+	}
+
+	args := c.Args()
+
+	logrus.Infof("Parsing arguments: %v", args.Slice())
+	if c.NArg() != 1 {
+		return fmt.Errorf("incorrect number of arguments")
+	}
+
+	o.RuntimeDir = args.Get(0)
+
+	logrus.Infof("Successfully parsed arguments")
+
+	return nil
+}
+
+// Configure applies the options to the specified config
+func (o Options) Configure(cfg engine.Interface) error {
+	err := o.UpdateConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+	return o.flush(cfg)
+}
+
+// Unconfigure removes the options from the specified config
+func (o Options) Unconfigure(cfg engine.Interface) error {
+	err := o.RevertConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+	return o.flush(cfg)
+}
+
+// flush flushes the specified config to disk
+func (o Options) flush(cfg engine.Interface) error {
+	logrus.Infof("Flushing config to %v", o.Config)
+	n, err := cfg.Save(o.Config)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+	if n == 0 {
+		logrus.Infof("Config file is empty, removed")
+	}
+	return nil
+}
+
+// UpdateConfig updates the specified config to include the nvidia runtimes
+func (o Options) UpdateConfig(cfg engine.Interface) error {
+	runtimes := operator.GetRuntimes(
+		operator.WithNvidiaRuntimeName(o.RuntimeName),
+		operator.WithSetAsDefault(o.SetAsDefault),
+		operator.WithRoot(o.RuntimeDir),
+	)
+	for name, runtime := range runtimes {
+		err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault)
+		if err != nil {
+			return fmt.Errorf("failed to update runtime %q: %v", name, err)
+		}
+	}
+
+	if o.EnableCDI {
+		cfg.EnableCDI()
+	}
+
+	return nil
+}
+
+// RevertConfig reverts the specified config to remove the nvidia runtimes
+func (o Options) RevertConfig(cfg engine.Interface) error {
+	runtimes := operator.GetRuntimes(
+		operator.WithNvidiaRuntimeName(o.RuntimeName),
+		operator.WithSetAsDefault(o.SetAsDefault),
+		operator.WithRoot(o.RuntimeDir),
+	)
+	for name := range runtimes {
+		err := cfg.RemoveRuntime(name)
+		if err != nil {
+			return fmt.Errorf("failed to remove runtime %q: %v", name, err)
+		}
+	}
+
+	return nil
+}
+
+// Restart restarts the specified service
+func (o Options) Restart(service string, withSignal func(string) error) error {
+	switch o.RestartMode {
+	case restartModeNone:
+		logrus.Warningf("Skipping restart of %v due to --restart-mode=%v", service, o.RestartMode)
+		return nil
+	case restartModeSignal:
+		return withSignal(o.Socket)
+	case restartModeSystemd:
+		return o.SystemdRestart(service)
+	}
+
+	return fmt.Errorf("invalid restart mode specified: %v", o.RestartMode)
+}
+
+// SystemdRestart restarts the specified service using systemd
+func (o Options) SystemdRestart(service string) error {
+	var args []string
+	var msg string
+	if o.HostRootMount != "" {
+		msg = " on host"
+		args = append(args, "chroot", o.HostRootMount)
+	}
+	args = append(args, "systemctl", "restart", service)
+
+	logrus.Infof("Restarting %v%v using systemd: %v", service, msg, args)
+
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error restarting %v using systemd: %v", service, err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/operator/operator.go

@@ -0,0 +1,133 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package operator
+
+import "path/filepath"
+
+const (
+	defaultRuntimeName = "nvidia"
+
+	defaultRoot = "/usr/bin"
+)
+
+// Runtime defines a runtime to be configured.
+// The path and whether the runtime is the default runtime can be specified
+type Runtime struct {
+	name         string
+	Path         string
+	SetAsDefault bool
+}
+
+// Runtimes defines a set of runtimes to be configured for use in the GPU Operator
+type Runtimes map[string]Runtime
+
+type config struct {
+	root              string
+	nvidiaRuntimeName string
+	setAsDefault      bool
+}
+
+// GetRuntimes returns the set of runtimes to be configured for use with the GPU Operator.
+func GetRuntimes(opts ...Option) Runtimes {
+	c := &config{}
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	if c.root == "" {
+		c.root = defaultRoot
+	}
+	if c.nvidiaRuntimeName == "" {
+		c.nvidiaRuntimeName = defaultRuntimeName
+	}
+
+	runtimes := make(Runtimes)
+	runtimes.add(c.nvidiaRuntime())
+
+	modes := []string{"cdi", "legacy"}
+	for _, mode := range modes {
+		runtimes.add(c.modeRuntime(mode))
+	}
+	return runtimes
+}
+
+// DefaultRuntimeName returns the name of the default runtime.
+func (r Runtimes) DefaultRuntimeName() string {
+	for _, runtime := range r {
+		if runtime.SetAsDefault {
+			return runtime.name
+		}
+	}
+	return ""
+}
+
+// Add a runtime to the set of runtimes.
+func (r Runtimes) add(runtime Runtime) {
+	r[runtime.name] = runtime
+}
+
+// nvidiaRuntime creates a runtime that corresponds to the nvidia runtime.
+// If name is equal to one of the predefined runtimes, `nvidia` is used as the runtime name instead.
+func (c config) nvidiaRuntime() Runtime {
+	predefinedRuntimes := map[string]struct{}{
+		"nvidia-cdi":    {},
+		"nvidia-legacy": {},
+	}
+	name := c.nvidiaRuntimeName
+	if _, isPredefinedRuntime := predefinedRuntimes[name]; isPredefinedRuntime {
+		name = defaultRuntimeName
+	}
+	return c.newRuntime(name, "nvidia-container-runtime")
+}
+
+// modeRuntime creates a runtime for the specified mode.
+func (c config) modeRuntime(mode string) Runtime {
+	return c.newRuntime("nvidia-"+mode, "nvidia-container-runtime."+mode)
+}
+
+// newRuntime creates a runtime based on the configuration
+func (c config) newRuntime(name string, binary string) Runtime {
+	return Runtime{
+		name:         name,
+		Path:         filepath.Join(c.root, binary),
+		SetAsDefault: c.setAsDefault && name == c.nvidiaRuntimeName,
+	}
+}
+
+// Option is a functional option for configuring set of runtimes.
+type Option func(*config)
+
+// WithRoot sets the root directory for the runtime binaries.
+func WithRoot(root string) Option {
+	return func(c *config) {
+		c.root = root
+	}
+}
+
+// WithNvidiaRuntimeName sets the name of the nvidia runtime.
+func WithNvidiaRuntimeName(name string) Option {
+	return func(c *config) {
+		c.nvidiaRuntimeName = name
+	}
+}
+
+// WithSetAsDefault sets the default runtime to the nvidia runtime.
+func WithSetAsDefault(set bool) Option {
+	return func(c *config) {
+		c.setAsDefault = set
+	}
+}

cmd/nvidia-ctk-installer/container/operator/operator_test.go

@@ -0,0 +1,141 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package operator
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptions(t *testing.T) {
+	testCases := []struct {
+		setAsDefault           bool
+		nvidiaRuntimeName      string
+		root                   string
+		expectedDefaultRuntime string
+		expectedRuntimes       Runtimes
+	}{
+		{
+			expectedRuntimes: Runtimes{
+				"nvidia": Runtime{
+					name: "nvidia",
+					Path: "/usr/bin/nvidia-container-runtime",
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:           true,
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimes: Runtimes{
+				"nvidia": Runtime{
+					name:         "nvidia",
+					Path:         "/usr/bin/nvidia-container-runtime",
+					SetAsDefault: true,
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:           true,
+			nvidiaRuntimeName:      "nvidia",
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimes: Runtimes{
+				"nvidia": Runtime{
+					name:         "nvidia",
+					Path:         "/usr/bin/nvidia-container-runtime",
+					SetAsDefault: true,
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:           true,
+			nvidiaRuntimeName:      "NAME",
+			expectedDefaultRuntime: "NAME",
+			expectedRuntimes: Runtimes{
+				"NAME": Runtime{
+					name:         "NAME",
+					Path:         "/usr/bin/nvidia-container-runtime",
+					SetAsDefault: true,
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:      false,
+			nvidiaRuntimeName: "NAME",
+			expectedRuntimes: Runtimes{
+				"NAME": Runtime{
+					name: "NAME",
+					Path: "/usr/bin/nvidia-container-runtime",
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			runtimes := GetRuntimes(
+				WithNvidiaRuntimeName(tc.nvidiaRuntimeName),
+				WithSetAsDefault(tc.setAsDefault),
+				WithRoot(tc.root),
+			)
+
+			require.EqualValues(t, tc.expectedRuntimes, runtimes)
+			require.Equal(t, tc.expectedDefaultRuntime, runtimes.DefaultRuntimeName())
+		})
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/config_v1_test.go

@@ -0,0 +1,584 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package containerd
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		legacyConfig                 bool
+		setAsDefault                 bool
+		runtimeName                  string
+		expectedDefaultRuntimeName   interface{}
+		expectedDefaultRuntimeBinary interface{}
+	}{
+		{},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			runtimeName:                  "NAME",
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   "nvidia",
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			runtimeName:                  "NAME",
+			expectedDefaultRuntimeName:   "NAME",
+			expectedDefaultRuntimeBinary: nil,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName:  tc.runtimeName,
+				RuntimeDir:   runtimeDir,
+				SetAsDefault: tc.setAsDefault,
+			}
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithUseLegacyConfig(tc.legacyConfig),
+				containerd.WithConfigVersion(1),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			cfg := v1.(*containerd.ConfigV1)
+			defaultRuntimeName := cfg.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+
+			defaultRuntime := cfg.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"})
+			if tc.expectedDefaultRuntimeBinary == nil {
+				require.Nil(t, defaultRuntime)
+			} else {
+				require.NotNil(t, defaultRuntime)
+
+				expected, err := defaultRuntimeTomlConfigV1(tc.expectedDefaultRuntimeBinary.(string))
+				require.NoError(t, err)
+
+				configContents, _ := toml.Marshal(defaultRuntime.(*toml.Tree))
+				expectedContents, _ := toml.Marshal(expected)
+
+				require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, tc)
+			}
+		})
+	}
+}
+
+func TestUpdateV1Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithConfigVersion(1),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v1.String())
+		})
+	}
+}
+
+func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(runcConfigMapV1("/runc-binary"))),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithConfigVersion(1),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v1.String())
+		})
+	}
+}
+
+func TestUpdateV1EnableCDI(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		enableCDI              bool
+		expectedEnableCDIValue interface{}
+	}{
+		{},
+		{
+			enableCDI:              false,
+			expectedEnableCDIValue: nil,
+		},
+		{
+			enableCDI:              true,
+			expectedEnableCDIValue: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v", tc.enableCDI), func(t *testing.T) {
+			o := &container.Options{
+				EnableCDI:   tc.enableCDI,
+				RuntimeName: "nvidia",
+				RuntimeDir:  runtimeDir,
+			}
+
+			cfg, err := toml.Empty.Load()
+			require.NoError(t, err)
+
+			v1 := &containerd.ConfigV1{
+				Logger:      logger,
+				Tree:        cfg,
+				RuntimeType: runtimeType,
+			}
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			enableCDIValue := v1.GetPath([]string{"plugins", "cri", "containerd", "enable_cdi"})
+			require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue)
+		})
+	}
+}
+
+func TestRevertV1Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
+							},
+							"default_runtime":      defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: "nvidia",
+			}
+
+			expected, err := toml.TreeFromMap(tc.expected)
+			require.NoError(t, err)
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(tc.config)),
+				containerd.WithRuntimeType(runtimeType),
+
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.RevertConfig(v1)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v1.String())
+		})
+	}
+}
+
+func defaultRuntimeTomlConfigV1(binary string) (*toml.Tree, error) {
+	return toml.TreeFromMap(defaultRuntimeV1(binary))
+}
+
+func defaultRuntimeV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+			"Runtime":    binary,
+		},
+	}
+}
+
+func runtimeMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+			"Runtime":    binary,
+		},
+	}
+}
+
+func runcConfigMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"plugins": map[string]interface{}{
+			"cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": map[string]interface{}{
+							"runtime_type":                    "runc_runtime_type",
+							"runtime_root":                    "runc_runtime_root",
+							"runtime_engine":                  "runc_runtime_engine",
+							"privileged_without_host_devices": true,
+							"options": map[string]interface{}{
+								"runc-option": "value",
+								"BinaryName":  binary,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go

@@ -0,0 +1,520 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package containerd
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	runtimeType = "runtime_type"
+)
+
+func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		setAsDefault               bool
+		runtimeName                string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{},
+		{
+			setAsDefault:               false,
+			runtimeName:                "nvidia",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               false,
+			runtimeName:                "NAME",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "nvidia",
+			expectedDefaultRuntimeName: "nvidia",
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName:  tc.runtimeName,
+				RuntimeDir:   runtimeDir,
+				SetAsDefault: tc.setAsDefault,
+			}
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			cfg := v2.(*containerd.Config)
+
+			defaultRuntimeName := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+		})
+	}
+}
+
+func TestUpdateV2Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v2.String())
+		})
+	}
+
+}
+
+func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(runcConfigMapV2("/runc-binary"))),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v2.String())
+		})
+	}
+}
+
+func TestUpdateV2ConfigEnableCDI(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		enableCDI              bool
+		expectedEnableCDIValue interface{}
+	}{
+		{},
+		{
+			enableCDI:              false,
+			expectedEnableCDIValue: nil,
+		},
+		{
+			enableCDI:              true,
+			expectedEnableCDIValue: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v", tc.enableCDI), func(t *testing.T) {
+			o := &container.Options{
+				EnableCDI:    tc.enableCDI,
+				RuntimeName:  "nvidia",
+				RuntimeDir:   runtimeDir,
+				SetAsDefault: false,
+			}
+
+			cfg, err := toml.LoadMap(map[string]interface{}{})
+			require.NoError(t, err)
+
+			v2 := &containerd.Config{
+				Logger:               logger,
+				Tree:                 cfg,
+				RuntimeType:          runtimeType,
+				CRIRuntimePluginName: "io.containerd.grpc.v1.cri",
+			}
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			enableCDIValue := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_cdi"})
+			require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue)
+		})
+	}
+}
+
+func TestRevertV2Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+							},
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: "nvidia",
+			}
+
+			expected, err := toml.TreeFromMap(tc.expected)
+			require.NoError(t, err)
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(tc.config)),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.RevertConfig(v2)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v2.String())
+		})
+	}
+}
+
+func runtimeMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+		},
+	}
+}
+
+func runcConfigMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"version": 2,
+		"plugins": map[string]interface{}{
+			"io.containerd.grpc.v1.cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": map[string]interface{}{
+							"runtime_type":                    "runc_runtime_type",
+							"runtime_root":                    "runc_runtime_root",
+							"runtime_engine":                  "runc_runtime_engine",
+							"privileged_without_host_devices": true,
+							"options": map[string]interface{}{
+								"runc-option": "value",
+								"BinaryName":  binary,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go

@@ -0,0 +1,184 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package containerd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	Name = "containerd"
+
+	DefaultConfig      = "/etc/containerd/config.toml"
+	DefaultSocket      = "/run/containerd/containerd.sock"
+	DefaultRestartMode = "signal"
+
+	defaultRuntmeType = "io.containerd.runc.v2"
+)
+
+// Options stores the containerd-specific options
+type Options struct {
+	useLegacyConfig bool
+	runtimeType     string
+
+	ContainerRuntimeModesCDIAnnotationPrefixes cli.StringSlice
+
+	runtimeConfigOverrideJSON string
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.BoolFlag{
+			Name: "use-legacy-config",
+			Usage: "Specify whether a legacy (pre v1.3) config should be used. " +
+				"This ensures that a version 1 container config is created by default and that the " +
+				"containerd.runtimes.default_runtime config section is used to define the default " +
+				"runtime instead of container.default_runtime_name.",
+			Destination: &opts.useLegacyConfig,
+			EnvVars:     []string{"CONTAINERD_USE_LEGACY_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-type",
+			Usage:       "The runtime_type to use for the configured runtime classes",
+			Value:       defaultRuntmeType,
+			Destination: &opts.runtimeType,
+			EnvVars:     []string{"CONTAINERD_RUNTIME_TYPE"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "nvidia-container-runtime-modes.cdi.annotation-prefixes",
+			Destination: &opts.ContainerRuntimeModesCDIAnnotationPrefixes,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-config-override",
+			Destination: &opts.runtimeConfigOverrideJSON,
+			Usage:       "specify additional runtime options as a JSON string. The paths are relative to the runtime config.",
+			Value:       "{}",
+			EnvVars:     []string{"RUNTIME_CONFIG_OVERRIDE", "CONTAINERD_RUNTIME_CONFIG_OVERRIDE"},
+		},
+	}
+
+	return flags
+}
+
+// Setup updates a containerd configuration to include the nvidia-containerd-runtime and reloads it
+func Setup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o, co)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Configure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to configure containerd: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts a containerd configuration to remove the nvidia-containerd-runtime and reloads it
+func Cleanup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o, co)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Unconfigure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to unconfigure containerd: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// RestartContainerd restarts containerd depending on the value of restartModeFlag
+func RestartContainerd(o *container.Options) error {
+	return o.Restart("containerd", SignalContainerd)
+}
+
+// containerAnnotationsFromCDIPrefixes returns the container annotations to set for the given CDI prefixes.
+func (o *Options) containerAnnotationsFromCDIPrefixes() []string {
+	var annotations []string
+	for _, prefix := range o.ContainerRuntimeModesCDIAnnotationPrefixes.Value() {
+		annotations = append(annotations, prefix+"*")
+	}
+
+	return annotations
+}
+
+func (o *Options) runtimeConfigOverride() (map[string]interface{}, error) {
+	if o.runtimeConfigOverrideJSON == "" {
+		return nil, nil
+	}
+
+	runtimeOptions := make(map[string]interface{})
+	if err := json.Unmarshal([]byte(o.runtimeConfigOverrideJSON), &runtimeOptions); err != nil {
+		return nil, fmt.Errorf("failed to read %v as JSON: %w", o.runtimeConfigOverrideJSON, err)
+	}
+
+	return runtimeOptions, nil
+}
+
+func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error) {
+	cfg, err := getRuntimeConfig(o, co)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load containerd config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}
+
+func getRuntimeConfig(o *container.Options, co *Options) (engine.Interface, error) {
+	return containerd.New(
+		containerd.WithPath(o.Config),
+		containerd.WithConfigSource(
+			toml.LoadFirst(
+				containerd.CommandLineSource(o.HostRootMount),
+				toml.FromFile(o.Config),
+			),
+		),
+		containerd.WithRuntimeType(co.runtimeType),
+		containerd.WithUseLegacyConfig(co.useLegacyConfig),
+		containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
+	)
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd_linux.go

@@ -0,0 +1,113 @@
+/**
+# Copyright 2020-2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	socketMessageToGetPID = ""
+)
+
+// SignalContainerd sends a SIGHUP signal to the containerd daemon
+func SignalContainerd(socket string) error {
+	log.Infof("Sending SIGHUP signal to containerd")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'containerd' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warningf("Error signaling containerd, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warningf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled containerd")
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd_other.go

@@ -0,0 +1,29 @@
+//go:build !linux
+// +build !linux
+
+/**
+# Copyright 2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"errors"
+)
+
+// SignalContainerd is unsupported on non-linux platforms.
+func SignalContainerd(socket string) error {
+	return errors.New("SignalContainerd is unsupported on non-linux platforms")
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd_test.go

@@ -0,0 +1,72 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRuntimeOptions(t *testing.T) {
+	testCases := []struct {
+		description   string
+		options       Options
+		expected      map[string]interface{}
+		expectedError error
+	}{
+		{
+			description: "empty is nil",
+		},
+		{
+			description: "empty json",
+			options: Options{
+				runtimeConfigOverrideJSON: "{}",
+			},
+			expected:      map[string]interface{}{},
+			expectedError: nil,
+		},
+		{
+			description: "SystemdCgroup is true",
+			options: Options{
+				runtimeConfigOverrideJSON: "{\"SystemdCgroup\": true}",
+			},
+			expected: map[string]interface{}{
+				"SystemdCgroup": true,
+			},
+			expectedError: nil,
+		},
+		{
+			description: "SystemdCgroup is false",
+			options: Options{
+				runtimeConfigOverrideJSON: "{\"SystemdCgroup\": false}",
+			},
+			expected: map[string]interface{}{
+				"SystemdCgroup": false,
+			},
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			runtimeOptions, err := tc.options.runtimeConfigOverride()
+			require.ErrorIs(t, tc.expectedError, err)
+			require.EqualValues(t, tc.expected, runtimeOptions)
+		})
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/crio/crio.go

@@ -0,0 +1,210 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	Name = "crio"
+
+	defaultConfigMode = "hook"
+
+	// Hook-based settings
+	defaultHooksDir     = "/usr/share/containers/oci/hooks.d"
+	defaultHookFilename = "oci-nvidia-hook.json"
+
+	// Config-based settings
+	DefaultConfig      = "/etc/crio/crio.conf"
+	DefaultSocket      = "/var/run/crio/crio.sock"
+	DefaultRestartMode = "systemd"
+)
+
+// Options defines the cri-o specific options.
+type Options struct {
+	configMode string
+
+	// hook-specific options
+	hooksDir     string
+	hookFilename string
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "hooks-dir",
+			Usage:       "path to the cri-o hooks directory",
+			Value:       defaultHooksDir,
+			Destination: &opts.hooksDir,
+			EnvVars:     []string{"CRIO_HOOKS_DIR"},
+			DefaultText: defaultHooksDir,
+		},
+		&cli.StringFlag{
+			Name:        "hook-filename",
+			Usage:       "filename of the cri-o hook that will be created / removed in the hooks directory",
+			Value:       defaultHookFilename,
+			Destination: &opts.hookFilename,
+			EnvVars:     []string{"CRIO_HOOK_FILENAME"},
+			DefaultText: defaultHookFilename,
+		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the configuration mode to use. One of [hook | config]",
+			Value:       defaultConfigMode,
+			Destination: &opts.configMode,
+			EnvVars:     []string{"CRIO_CONFIG_MODE"},
+		},
+	}
+
+	return flags
+}
+
+// Setup installs the prestart hook required to launch GPU-enabled containers
+func Setup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	switch co.configMode {
+	case "hook":
+		return setupHook(o, co)
+	case "config":
+		return setupConfig(o)
+	default:
+		return fmt.Errorf("invalid config-mode '%v'", co.configMode)
+	}
+}
+
+// setupHook installs the prestart hook required to launch GPU-enabled containers
+func setupHook(o *container.Options, co *Options) error {
+	log.Infof("Installing prestart hook")
+
+	hookPath := filepath.Join(co.hooksDir, co.hookFilename)
+	err := ocihook.CreateHook(hookPath, filepath.Join(o.RuntimeDir, config.NVIDIAContainerRuntimeHookExecutable))
+	if err != nil {
+		return fmt.Errorf("error creating hook: %v", err)
+	}
+
+	return nil
+}
+
+// setupConfig updates the cri-o config for the NVIDIA container runtime
+func setupConfig(o *container.Options) error {
+	log.Infof("Updating config file")
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Configure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to configure cri-o: %v", err)
+	}
+
+	err = RestartCrio(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart crio: %v", err)
+	}
+
+	return nil
+}
+
+// Cleanup removes the specified prestart hook
+func Cleanup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	switch co.configMode {
+	case "hook":
+		return cleanupHook(co)
+	case "config":
+		return cleanupConfig(o)
+	default:
+		return fmt.Errorf("invalid config-mode '%v'", co.configMode)
+	}
+}
+
+// cleanupHook removes the prestart hook
+func cleanupHook(co *Options) error {
+	log.Infof("Removing prestart hook")
+
+	hookPath := filepath.Join(co.hooksDir, co.hookFilename)
+	err := os.Remove(hookPath)
+	if err != nil {
+		return fmt.Errorf("error removing hook '%v': %v", hookPath, err)
+	}
+
+	return nil
+}
+
+// cleanupConfig removes the NVIDIA container runtime from the cri-o config
+func cleanupConfig(o *container.Options) error {
+	log.Infof("Reverting config file modifications")
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Unconfigure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to unconfigure cri-o: %v", err)
+	}
+
+	err = RestartCrio(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart crio: %v", err)
+	}
+
+	return nil
+}
+
+// RestartCrio restarts crio depending on the value of restartModeFlag
+func RestartCrio(o *container.Options) error {
+	return o.Restart("crio", func(string) error { return fmt.Errorf("supporting crio via signal is unsupported") })
+}
+
+func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load crio config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}
+
+func getRuntimeConfig(o *container.Options) (engine.Interface, error) {
+	return crio.New(
+		crio.WithPath(o.Config),
+		crio.WithConfigSource(
+			toml.LoadFirst(
+				crio.CommandLineSource(o.HostRootMount),
+				toml.FromFile(o.Config),
+			),
+		),
+	)
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker.go

@@ -0,0 +1,111 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
+)
+
+const (
+	Name = "docker"
+
+	DefaultConfig      = "/etc/docker/daemon.json"
+	DefaultSocket      = "/var/run/docker.sock"
+	DefaultRestartMode = "signal"
+)
+
+type Options struct{}
+
+func Flags(opts *Options) []cli.Flag {
+	return nil
+}
+
+// Setup updates docker configuration to include the nvidia runtime and reloads it
+func Setup(c *cli.Context, o *container.Options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Configure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to configure docker: %v", err)
+	}
+
+	err = RestartDocker(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart docker: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts docker configuration to remove the nvidia runtime and reloads it
+func Cleanup(c *cli.Context, o *container.Options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Unconfigure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to unconfigure docker: %v", err)
+	}
+
+	err = RestartDocker(o)
+	if err != nil {
+		return fmt.Errorf("unable to signal docker: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// RestartDocker restarts docker depending on the value of restartModeFlag
+func RestartDocker(o *container.Options) error {
+	return o.Restart("docker", SignalDocker)
+}
+
+func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
+	cfg, err := docker.New(
+		docker.WithPath(o.Config),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load docker config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}
+
+func getRuntimeConfig(o *container.Options) (engine.Interface, error) {
+	return docker.New(
+		docker.WithPath(o.Config),
+	)
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker_linux.go

@@ -0,0 +1,113 @@
+/**
+# Copyright 2021-2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	socketMessageToGetPID = "GET /info HTTP/1.0\r\n\r\n"
+)
+
+// SignalDocker sends a SIGHUP signal to docker daemon
+func SignalDocker(socket string) error {
+	log.Infof("Sending SIGHUP signal to docker")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'docker' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warningf("Error signaling docker, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warningf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled docker")
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker_other.go

@@ -0,0 +1,29 @@
+//go:build !linux
+// +build !linux
+
+/**
+# Copyright 2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"errors"
+)
+
+// SignalDocker is unsupported on non-linux platforms.
+func SignalDocker(socket string) error {
+	return errors.New("SignalDocker is unsupported on non-linux platforms")
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker_test.go

@@ -14,13 +14,16 @@
 # limitations under the License.
 */
 
-package main
+package docker
 
 import (
 	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
 )
 
 func TestUpdateConfigDefaultRuntime(t *testing.T) {
@@ -41,11 +44,6 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: "NAME",
 		},
-		{
-			setAsDefault:               true,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: "nvidia-experimental",
-		},
 		{
 			setAsDefault:               true,
 			runtimeName:                "nvidia",
@@ -54,15 +52,15 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		o := &options{
+		o := &container.Options{
-			setAsDefault: tc.setAsDefault,
+			RuntimeName:  tc.runtimeName,
-			runtimeName:  tc.runtimeName,
+			RuntimeDir:   runtimeDir,
-			runtimeDir:   runtimeDir,
+			SetAsDefault: tc.setAsDefault,
 		}
 
-		config := map[string]interface{}{}
+		config := docker.Config(map[string]interface{}{})
 
-		err := UpdateConfig(config, o)
+		err := o.UpdateConfig(&config)
 		require.NoError(t, err, "%d: %v", i, tc)
 
 		defaultRuntimeName := config["default-runtime"]
@@ -74,7 +72,7 @@ func TestUpdateConfig(t *testing.T) {
 	const runtimeDir = "/test/runtime/dir"
 
 	testCases := []struct {
-		config         map[string]interface{}
+		config         docker.Config
 		setAsDefault   bool
 		runtimeName    string
 		expectedConfig map[string]interface{}
@@ -88,8 +86,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -105,25 +107,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
 					},
-				},
+					"nvidia-legacy": map[string]interface{}{
-			},
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
-		},
-		{
-			config:       map[string]interface{}{},
-			setAsDefault: false,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
 						"args": []string{},
 					},
 				},
@@ -145,8 +134,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -171,8 +164,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -191,28 +188,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
 					},
-				},
+					"nvidia-legacy": map[string]interface{}{
-			},
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
-		},
-		{
-			config: map[string]interface{}{
-				"default-runtime": "runc",
-			},
-			setAsDefault: true,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"default-runtime": "nvidia-experimental",
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
 						"args": []string{},
 					},
 				},
@@ -239,8 +220,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -249,12 +234,15 @@ func TestUpdateConfig(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		options := &options{
+		tc := tc
-			setAsDefault: tc.setAsDefault,
+
-			runtimeName:  tc.runtimeName,
+		o := &container.Options{
-			runtimeDir:   runtimeDir,
+			RuntimeName:  tc.runtimeName,
+			RuntimeDir:   runtimeDir,
+			SetAsDefault: tc.setAsDefault,
 		}
-		err := UpdateConfig(tc.config, options)
+
+		err := o.UpdateConfig(&tc.config)
 		require.NoError(t, err, "%d: %v", i, tc)
 
 		configContent, err := json.MarshalIndent(tc.config, "", "    ")
@@ -269,7 +257,7 @@ func TestUpdateConfig(t *testing.T) {
 
 func TestRevertConfig(t *testing.T) {
 	testCases := []struct {
-		config         map[string]interface{}
+		config         docker.Config
 		expectedConfig map[string]interface{}
 	}{
 		{
@@ -290,7 +278,7 @@ func TestRevertConfig(t *testing.T) {
 		{
 			config: map[string]interface{}{
 				"runtimes": map[string]interface{}{
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
@@ -305,8 +293,12 @@ func TestRevertConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia-cdi": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -368,7 +360,9 @@ func TestRevertConfig(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		err := RevertConfig(tc.config)
+		tc := tc
+		o := &container.Options{}
+		err := o.RevertConfig(&tc.config)
 
 		require.NoError(t, err, "%d: %v", i, tc)
 
@@ -381,43 +375,3 @@ func TestRevertConfig(t *testing.T) {
 		require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
 	}
 }
-
-func TestFlagsDefaultRuntime(t *testing.T) {
-	testCases := []struct {
-		setAsDefault bool
-		runtimeName  string
-		expected     string
-	}{
-		{
-			expected: "",
-		},
-		{
-			runtimeName: "not-bool",
-			expected:    "",
-		},
-		{
-			setAsDefault: false,
-			runtimeName:  "nvidia",
-			expected:     "",
-		},
-		{
-			setAsDefault: true,
-			runtimeName:  "nvidia",
-			expected:     "nvidia",
-		},
-		{
-			setAsDefault: true,
-			runtimeName:  "nvidia-experimental",
-			expected:     "nvidia-experimental",
-		},
-	}
-
-	for i, tc := range testCases {
-		f := options{
-			setAsDefault: tc.setAsDefault,
-			runtimeName:  tc.runtimeName,
-		}
-
-		require.Equal(t, tc.expected, f.getDefaultRuntime(), "%d: %v", i, tc)
-	}
-}

cmd/nvidia-ctk-installer/container/runtime/runtime.go

@@ -0,0 +1,192 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/toolkit"
+)
+
+const (
+	defaultSetAsDefault = true
+	// defaultRuntimeName specifies the NVIDIA runtime to be use as the default runtime if setting the default runtime is enabled
+	defaultRuntimeName   = "nvidia"
+	defaultHostRootMount = "/host"
+
+	runtimeSpecificDefault = "RUNTIME_SPECIFIC_DEFAULT"
+)
+
+type Options struct {
+	container.Options
+
+	containerdOptions containerd.Options
+	crioOptions       crio.Options
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "Path to the runtime config file",
+			Value:       runtimeSpecificDefault,
+			Destination: &opts.Config,
+			EnvVars:     []string{"RUNTIME_CONFIG", "CONTAINERD_CONFIG", "DOCKER_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "socket",
+			Usage:       "Path to the runtime socket file",
+			Value:       runtimeSpecificDefault,
+			Destination: &opts.Socket,
+			EnvVars:     []string{"RUNTIME_SOCKET", "CONTAINERD_SOCKET", "DOCKER_SOCKET"},
+		},
+		&cli.StringFlag{
+			Name:        "restart-mode",
+			Usage:       "Specify how the runtime should be restarted; If 'none' is selected it will not be restarted [signal | systemd | none ]",
+			Value:       runtimeSpecificDefault,
+			Destination: &opts.RestartMode,
+			EnvVars:     []string{"RUNTIME_RESTART_MODE"},
+		},
+		&cli.BoolFlag{
+			Name:        "enable-cdi-in-runtime",
+			Usage:       "Enable CDI in the configured runt	ime",
+			Destination: &opts.EnableCDI,
+			EnvVars:     []string{"RUNTIME_ENABLE_CDI"},
+		},
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "Specify the path to the host root to be used when restarting the runtime using systemd",
+			Value:       defaultHostRootMount,
+			Destination: &opts.HostRootMount,
+			EnvVars:     []string{"HOST_ROOT_MOUNT"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-name",
+			Aliases:     []string{"nvidia-runtime-name", "runtime-class"},
+			Usage:       "Specify the name of the `nvidia` runtime. If set-as-default is selected, the runtime is used as the default runtime.",
+			Value:       defaultRuntimeName,
+			Destination: &opts.RuntimeName,
+			EnvVars:     []string{"NVIDIA_RUNTIME_NAME", "CONTAINERD_RUNTIME_CLASS", "DOCKER_RUNTIME_NAME"},
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Usage:       "Set the `nvidia` runtime as the default runtime.",
+			Value:       defaultSetAsDefault,
+			Destination: &opts.SetAsDefault,
+			EnvVars:     []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "CONTAINERD_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+	}
+
+	flags = append(flags, containerd.Flags(&opts.containerdOptions)...)
+	flags = append(flags, crio.Flags(&opts.crioOptions)...)
+
+	return flags
+}
+
+// ValidateOptions checks whether the specified options are valid
+func ValidateOptions(c *cli.Context, opts *Options, runtime string, toolkitRoot string, to *toolkit.Options) error {
+	// We set this option here to ensure that it is available in future calls.
+	opts.RuntimeDir = toolkitRoot
+
+	if !c.IsSet("enable-cdi-in-runtime") {
+		opts.EnableCDI = to.CDI.Enabled
+	}
+
+	// Apply the runtime-specific config changes.
+	switch runtime {
+	case containerd.Name:
+		if opts.Config == runtimeSpecificDefault {
+			opts.Config = containerd.DefaultConfig
+		}
+		if opts.Socket == runtimeSpecificDefault {
+			opts.Socket = containerd.DefaultSocket
+		}
+		if opts.RestartMode == runtimeSpecificDefault {
+			opts.RestartMode = containerd.DefaultRestartMode
+		}
+	case crio.Name:
+		if opts.Config == runtimeSpecificDefault {
+			opts.Config = crio.DefaultConfig
+		}
+		if opts.Socket == runtimeSpecificDefault {
+			opts.Socket = crio.DefaultSocket
+		}
+		if opts.RestartMode == runtimeSpecificDefault {
+			opts.RestartMode = crio.DefaultRestartMode
+		}
+	case docker.Name:
+		if opts.Config == runtimeSpecificDefault {
+			opts.Config = docker.DefaultConfig
+		}
+		if opts.Socket == runtimeSpecificDefault {
+			opts.Socket = docker.DefaultSocket
+		}
+		if opts.RestartMode == runtimeSpecificDefault {
+			opts.RestartMode = docker.DefaultRestartMode
+		}
+	default:
+		return fmt.Errorf("undefined runtime %v", runtime)
+	}
+
+	return nil
+}
+
+func Setup(c *cli.Context, opts *Options, runtime string) error {
+	switch runtime {
+	case containerd.Name:
+		return containerd.Setup(c, &opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.Setup(c, &opts.Options, &opts.crioOptions)
+	case docker.Name:
+		return docker.Setup(c, &opts.Options)
+	default:
+		return fmt.Errorf("undefined runtime %v", runtime)
+	}
+}
+
+func Cleanup(c *cli.Context, opts *Options, runtime string) error {
+	switch runtime {
+	case containerd.Name:
+		return containerd.Cleanup(c, &opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.Cleanup(c, &opts.Options, &opts.crioOptions)
+	case docker.Name:
+		return docker.Cleanup(c, &opts.Options)
+	default:
+		return fmt.Errorf("undefined runtime %v", runtime)
+	}
+}
+
+func GetLowlevelRuntimePaths(opts *Options, runtime string) ([]string, error) {
+	switch runtime {
+	case containerd.Name:
+		return containerd.GetLowlevelRuntimePaths(&opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.GetLowlevelRuntimePaths(&opts.Options)
+	case docker.Name:
+		return docker.GetLowlevelRuntimePaths(&opts.Options)
+	default:
+		return nil, fmt.Errorf("undefined runtime %v", runtime)
+	}
+}

cmd/nvidia-ctk-installer/container/toolkit/executable.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 */
 
-package main
+package toolkit
 
 import (
 	"fmt"
@@ -23,8 +23,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
-
-	log "github.com/sirupsen/logrus"
 )
 
 type executableTarget struct {
@@ -33,6 +31,7 @@ type executableTarget struct {
 }
 
 type executable struct {
+	fileInstaller
 	source   string
 	target   executableTarget
 	env      map[string]string
@@ -43,21 +42,21 @@ type executable struct {
 // install installs an executable component of the NVIDIA container toolkit. The source executable
 // is copied to a `.real` file and a wapper is created to set up the environment as required.
 func (e executable) install(destFolder string) (string, error) {
-	log.Infof("Installing executable '%v' to %v", e.source, destFolder)
+	e.logger.Infof("Installing executable '%v' to %v", e.source, destFolder)
 
 	dotfileName := e.dotfileName()
 
-	installedDotfileName, err := installFileToFolderWithName(destFolder, dotfileName, e.source)
+	installedDotfileName, err := e.installFileToFolderWithName(destFolder, dotfileName, e.source)
 	if err != nil {
 		return "", fmt.Errorf("error installing file '%v' as '%v': %v", e.source, dotfileName, err)
 	}
-	log.Infof("Installed '%v'", installedDotfileName)
+	e.logger.Infof("Installed '%v'", installedDotfileName)
 
 	wrapperFilename, err := e.installWrapper(destFolder, installedDotfileName)
 	if err != nil {
 		return "", fmt.Errorf("error wrapping '%v': %v", installedDotfileName, err)
 	}
-	log.Infof("Installed wrapper '%v'", wrapperFilename)
+	e.logger.Infof("Installed wrapper '%v'", wrapperFilename)
 
 	return wrapperFilename, nil
 }

cmd/nvidia-ctk-installer/container/toolkit/executable_test.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 */
 
-package main
+package toolkit
 
 import (
 	"bytes"
@@ -23,10 +23,13 @@ import (
 	"strings"
 	"testing"
 
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
 func TestWrapper(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
 	const shebang = "#! /bin/sh"
 	const destFolder = "/dest/folder"
 	const dotfileName = "source.real"
@@ -98,6 +101,8 @@ func TestWrapper(t *testing.T) {
 	for i, tc := range testCases {
 		buf := &bytes.Buffer{}
 
+		tc.e.logger = logger
+
 		err := tc.e.writeWrapperTo(buf, destFolder, dotfileName)
 		require.NoError(t, err)
 
@@ -107,6 +112,8 @@ func TestWrapper(t *testing.T) {
 }
 
 func TestInstallExecutable(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
 	inputFolder, err := os.MkdirTemp("", "")
 	require.NoError(t, err)
 	defer os.RemoveAll(inputFolder)
@@ -121,6 +128,9 @@ func TestInstallExecutable(t *testing.T) {
 	require.NoError(t, sourceFile.Close())
 
 	e := executable{
+		fileInstaller: fileInstaller{
+			logger: logger,
+		},
 		source: source,
 		target: executableTarget{
 			dotfileName: "input.real",

cmd/nvidia-ctk-installer/container/toolkit/file-installer.go

@@ -0,0 +1,95 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package toolkit
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type fileInstaller struct {
+	logger logger.Interface
+	// sourceRoot specifies the root that is searched for the components to install.
+	sourceRoot string
+}
+
+// installFileToFolder copies a source file to a destination folder.
+// The path of the input file is ignored.
+// e.g. installFileToFolder("/some/path/file.txt", "/output/path")
+// will result in a file "/output/path/file.txt" being generated
+func (t *fileInstaller) installFileToFolder(destFolder string, src string) (string, error) {
+	name := filepath.Base(src)
+	return t.installFileToFolderWithName(destFolder, name, src)
+}
+
+// cp src destFolder/name
+func (t *fileInstaller) installFileToFolderWithName(destFolder string, name, src string) (string, error) {
+	dest := filepath.Join(destFolder, name)
+	err := t.installFile(dest, src)
+	if err != nil {
+		return "", fmt.Errorf("error copying '%v' to '%v': %v", src, dest, err)
+	}
+	return dest, nil
+}
+
+// installFile copies a file from src to dest and maintains
+// file modes
+func (t *fileInstaller) installFile(dest string, src string) error {
+	src = filepath.Join(t.sourceRoot, src)
+	t.logger.Infof("Installing '%v' to '%v'", src, dest)
+
+	source, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("error opening source: %v", err)
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dest)
+	if err != nil {
+		return fmt.Errorf("error creating destination: %v", err)
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return fmt.Errorf("error copying file: %v", err)
+	}
+
+	err = applyModeFromSource(dest, src)
+	if err != nil {
+		return fmt.Errorf("error setting destination file mode: %v", err)
+	}
+	return nil
+}
+
+// applyModeFromSource sets the file mode for a destination file
+// to match that of a specified source file
+func applyModeFromSource(dest string, src string) error {
+	sourceInfo, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("error getting file info for '%v': %v", src, err)
+	}
+	err = os.Chmod(dest, sourceInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("error setting mode for '%v': %v", dest, err)
+	}
+	return nil
+}

cmd/nvidia-ctk-installer/container/toolkit/options.go

@@ -0,0 +1,40 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package toolkit
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// An Option provides a mechanism to configure an Installer.
+type Option func(*Installer)
+
+func WithLogger(logger logger.Interface) Option {
+	return func(i *Installer) {
+		i.logger = logger
+	}
+}
+
+func WithToolkitRoot(toolkitRoot string) Option {
+	return func(i *Installer) {
+		i.toolkitRoot = toolkitRoot
+	}
+}
+
+func WithSourceRoot(sourceRoot string) Option {
+	return func(i *Installer) {
+		i.sourceRoot = sourceRoot
+	}
+}

cmd/nvidia-ctk-installer/container/toolkit/replacements.go

@@ -14,7 +14,7 @@
 # limitations under the License.
 */
 
-package main
+package toolkit
 
 import "strings"
 

cmd/nvidia-ctk-installer/container/toolkit/runtime.go

@@ -0,0 +1,85 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package toolkit
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
+)
+
+const (
+	nvidiaContainerRuntimeSource = "/usr/bin/nvidia-container-runtime"
+)
+
+// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
+// and implementing the required wrapper
+func (t *Installer) installContainerRuntimes(toolkitDir string) error {
+	runtimes := operator.GetRuntimes()
+	for _, runtime := range runtimes {
+		r := t.newNvidiaContainerRuntimeInstaller(runtime.Path)
+
+		_, err := r.install(toolkitDir)
+		if err != nil {
+			return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
+		}
+	}
+	return nil
+}
+
+// newNVidiaContainerRuntimeInstaller returns a new executable installer for the NVIDIA container runtime.
+// This installer will copy the specified source executable to the toolkit directory.
+// The executable is copied to a file with the same name as the source, but with a ".real" suffix and a wrapper is
+// created to allow for the configuration of the runtime environment.
+func (t *Installer) newNvidiaContainerRuntimeInstaller(source string) *executable {
+	wrapperName := filepath.Base(source)
+	dotfileName := wrapperName + ".real"
+	target := executableTarget{
+		dotfileName: dotfileName,
+		wrapperName: wrapperName,
+	}
+	return t.newRuntimeInstaller(source, target, nil)
+}
+
+func (t *Installer) newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {
+	preLines := []string{
+		"",
+		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
+		"if [ \"${?}\" != \"0\" ]; then",
+		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
+		"	exec runc \"$@\"",
+		"fi",
+		"",
+	}
+
+	runtimeEnv := make(map[string]string)
+	runtimeEnv["XDG_CONFIG_HOME"] = filepath.Join(destDirPattern, ".config")
+	for k, v := range env {
+		runtimeEnv[k] = v
+	}
+
+	r := executable{
+		fileInstaller: t.fileInstaller,
+		source:        source,
+		target:        target,
+		env:           runtimeEnv,
+		preLines:      preLines,
+	}
+
+	return &r
+}

cmd/nvidia-ctk-installer/container/toolkit/runtime_test.go

@@ -14,18 +14,25 @@
 # limitations under the License.
 */
 
-package main
+package toolkit
 
 import (
 	"bytes"
 	"strings"
 	"testing"
 
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
 func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
-	r := newNvidiaContainerRuntimeInstaller()
+	logger, _ := testlog.NewNullLogger()
+	i := Installer{
+		fileInstaller: fileInstaller{
+			logger: logger,
+		},
+	}
+	r := i.newNvidiaContainerRuntimeInstaller(nvidiaContainerRuntimeSource)
 
 	const shebang = "#! /bin/sh"
 	const destFolder = "/dest/folder"
@@ -55,36 +62,3 @@ func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
 	exepectedContents := strings.Join(expectedLines, "\n")
 	require.Equal(t, exepectedContents, buf.String())
 }
-
-func TestExperimentalContainerRuntimeInstallerWrapper(t *testing.T) {
-	r := newNvidiaContainerRuntimeExperimentalInstaller("/some/root/usr/lib64")
-
-	const shebang = "#! /bin/sh"
-	const destFolder = "/dest/folder"
-	const dotfileName = "source.real"
-
-	buf := &bytes.Buffer{}
-
-	err := r.writeWrapperTo(buf, destFolder, dotfileName)
-	require.NoError(t, err)
-
-	expectedLines := []string{
-		shebang,
-		"",
-		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
-		"if [ \"${?}\" != \"0\" ]; then",
-		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
-		"	exec runc \"$@\"",
-		"fi",
-		"",
-		"LD_LIBRARY_PATH=/some/root/usr/lib64:$LD_LIBRARY_PATH \\",
-		"PATH=/dest/folder:$PATH \\",
-		"XDG_CONFIG_HOME=/dest/folder/.config \\",
-		"source.real \\",
-		"\t\"$@\"",
-		"",
-	}
-
-	exepectedContents := strings.Join(expectedLines, "\n")
-	require.Equal(t, exepectedContents, buf.String())
-}

cmd/nvidia-ctk-installer/container/toolkit/toolkit.go

@@ -0,0 +1,748 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package toolkit
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+	"tags.cncf.io/container-device-interface/pkg/parser"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	transformroot "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform/root"
+)
+
+const (
+	// DefaultNvidiaDriverRoot specifies the default NVIDIA driver run directory
+	DefaultNvidiaDriverRoot = "/run/nvidia/driver"
+
+	nvidiaContainerCliSource         = "/usr/bin/nvidia-container-cli"
+	nvidiaContainerRuntimeHookSource = "/usr/bin/nvidia-container-runtime-hook"
+
+	nvidiaContainerToolkitConfigSource = "/etc/nvidia-container-runtime/config.toml"
+	configFilename                     = "config.toml"
+)
+
+type cdiOptions struct {
+	Enabled   bool
+	outputDir string
+	kind      string
+	vendor    string
+	class     string
+}
+
+type Options struct {
+	DriverRoot        string
+	DevRoot           string
+	DriverRootCtrPath string
+	DevRootCtrPath    string
+
+	ContainerRuntimeMode     string
+	ContainerRuntimeDebug    string
+	ContainerRuntimeLogLevel string
+
+	ContainerRuntimeModesCdiDefaultKind        string
+	ContainerRuntimeModesCDIAnnotationPrefixes cli.StringSlice
+
+	ContainerRuntimeRuntimes cli.StringSlice
+
+	ContainerRuntimeHookSkipModeDetection bool
+
+	ContainerCLIDebug string
+
+	// CDI stores the CDI options for the toolkit.
+	CDI cdiOptions
+
+	createDeviceNodes cli.StringSlice
+
+	acceptNVIDIAVisibleDevicesWhenUnprivileged bool
+	acceptNVIDIAVisibleDevicesAsVolumeMounts   bool
+
+	ignoreErrors bool
+
+	optInFeatures cli.StringSlice
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Aliases:     []string{"nvidia-driver-root"},
+			Value:       DefaultNvidiaDriverRoot,
+			Destination: &opts.DriverRoot,
+			EnvVars:     []string{"NVIDIA_DRIVER_ROOT", "DRIVER_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root-ctr-path",
+			Value:       DefaultNvidiaDriverRoot,
+			Destination: &opts.DriverRootCtrPath,
+			EnvVars:     []string{"DRIVER_ROOT_CTR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "dev-root",
+			Usage:       "Specify the root where `/dev` is located. If this is not specified, the driver-root is assumed.",
+			Destination: &opts.DevRoot,
+			EnvVars:     []string{"NVIDIA_DEV_ROOT", "DEV_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "dev-root-ctr-path",
+			Usage:       "Specify the root where `/dev` is located in the container. If this is not specified, the driver-root-ctr-path is assumed.",
+			Destination: &opts.DevRootCtrPath,
+			EnvVars:     []string{"DEV_ROOT_CTR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.debug",
+			Aliases:     []string{"nvidia-container-runtime-debug"},
+			Usage:       "Specify the location of the debug log file for the NVIDIA Container Runtime",
+			Destination: &opts.ContainerRuntimeDebug,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_DEBUG"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.log-level",
+			Aliases:     []string{"nvidia-container-runtime-debug-log-level"},
+			Destination: &opts.ContainerRuntimeLogLevel,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_LOG_LEVEL"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.mode",
+			Aliases:     []string{"nvidia-container-runtime-mode"},
+			Destination: &opts.ContainerRuntimeMode,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.modes.cdi.default-kind",
+			Destination: &opts.ContainerRuntimeModesCdiDefaultKind,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_DEFAULT_KIND"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "nvidia-container-runtime.modes.cdi.annotation-prefixes",
+			Destination: &opts.ContainerRuntimeModesCDIAnnotationPrefixes,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "nvidia-container-runtime.runtimes",
+			Destination: &opts.ContainerRuntimeRuntimes,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_RUNTIMES"},
+		},
+		&cli.BoolFlag{
+			Name:        "nvidia-container-runtime-hook.skip-mode-detection",
+			Value:       true,
+			Destination: &opts.ContainerRuntimeHookSkipModeDetection,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_HOOK_SKIP_MODE_DETECTION"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-cli.debug",
+			Aliases:     []string{"nvidia-container-cli-debug"},
+			Usage:       "Specify the location of the debug log file for the NVIDIA Container CLI",
+			Destination: &opts.ContainerCLIDebug,
+			EnvVars:     []string{"NVIDIA_CONTAINER_CLI_DEBUG"},
+		},
+		&cli.BoolFlag{
+			Name:        "accept-nvidia-visible-devices-envvar-when-unprivileged",
+			Usage:       "Set the accept-nvidia-visible-devices-envvar-when-unprivileged config option",
+			Value:       true,
+			Destination: &opts.acceptNVIDIAVisibleDevicesWhenUnprivileged,
+			EnvVars:     []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_ENVVAR_WHEN_UNPRIVILEGED"},
+		},
+		&cli.BoolFlag{
+			Name:        "accept-nvidia-visible-devices-as-volume-mounts",
+			Usage:       "Set the accept-nvidia-visible-devices-as-volume-mounts config option",
+			Destination: &opts.acceptNVIDIAVisibleDevicesAsVolumeMounts,
+			EnvVars:     []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_AS_VOLUME_MOUNTS"},
+		},
+		&cli.BoolFlag{
+			Name:        "cdi-enabled",
+			Aliases:     []string{"enable-cdi"},
+			Usage:       "enable the generation of a CDI specification",
+			Destination: &opts.CDI.Enabled,
+			EnvVars:     []string{"CDI_ENABLED", "ENABLE_CDI"},
+		},
+		&cli.StringFlag{
+			Name:        "cdi-output-dir",
+			Usage:       "the directory where the CDI output files are to be written. If this is set to '', no CDI specification is generated.",
+			Value:       "/var/run/cdi",
+			Destination: &opts.CDI.outputDir,
+			EnvVars:     []string{"CDI_OUTPUT_DIR"},
+		},
+		&cli.StringFlag{
+			Name:        "cdi-kind",
+			Usage:       "the vendor string to use for the generated CDI specification",
+			Value:       "management.nvidia.com/gpu",
+			Destination: &opts.CDI.kind,
+			EnvVars:     []string{"CDI_KIND"},
+		},
+		&cli.BoolFlag{
+			Name:        "ignore-errors",
+			Usage:       "ignore errors when installing the NVIDIA Container toolkit. This is used for testing purposes only.",
+			Hidden:      true,
+			Destination: &opts.ignoreErrors,
+		},
+		&cli.StringSliceFlag{
+			Name:        "create-device-nodes",
+			Usage:       "(Only applicable with --cdi-enabled) specifies which device nodes should be created. If any one of the options is set to '' or 'none', no device nodes will be created.",
+			Value:       cli.NewStringSlice("control"),
+			Destination: &opts.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "opt-in-features",
+			Hidden:      true,
+			Destination: &opts.optInFeatures,
+			EnvVars:     []string{"NVIDIA_CONTAINER_TOOLKIT_OPT_IN_FEATURES"},
+		},
+	}
+
+	return flags
+}
+
+// An Installer is used to install the NVIDIA Container Toolkit from the toolkit container.
+type Installer struct {
+	fileInstaller
+	// toolkitRoot specifies the destination path at which the toolkit is installed.
+	toolkitRoot string
+}
+
+// NewInstaller creates an installer for the NVIDIA Container Toolkit.
+func NewInstaller(opts ...Option) *Installer {
+	i := &Installer{}
+	for _, opt := range opts {
+		opt(i)
+	}
+
+	if i.logger == nil {
+		i.logger = logger.New()
+	}
+	return i
+}
+
+// ValidateOptions checks whether the specified options are valid
+func (t *Installer) ValidateOptions(opts *Options) error {
+	if t == nil {
+		return fmt.Errorf("toolkit installer is not initilized")
+	}
+	if t.toolkitRoot == "" {
+		return fmt.Errorf("invalid --toolkit-root option: %v", t.toolkitRoot)
+	}
+
+	vendor, class := parser.ParseQualifier(opts.CDI.kind)
+	if err := parser.ValidateVendorName(vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := parser.ValidateClassName(class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	opts.CDI.vendor = vendor
+	opts.CDI.class = class
+
+	if opts.CDI.Enabled && opts.CDI.outputDir == "" {
+		t.logger.Warning("Skipping CDI spec generation (no output directory specified)")
+		opts.CDI.Enabled = false
+	}
+
+	isDisabled := false
+	for _, mode := range opts.createDeviceNodes.Value() {
+		if mode != "" && mode != "none" && mode != "control" {
+			return fmt.Errorf("invalid --create-device-nodes value: %v", mode)
+		}
+		if mode == "" || mode == "none" {
+			isDisabled = true
+			break
+		}
+	}
+	if !opts.CDI.Enabled && !isDisabled {
+		t.logger.Info("disabling device node creation since --cdi-enabled=false")
+		isDisabled = true
+	}
+	if isDisabled {
+		opts.createDeviceNodes = *cli.NewStringSlice()
+	}
+
+	return nil
+}
+
+// Install installs the components of the NVIDIA container toolkit.
+// Any existing installation is removed.
+func (t *Installer) Install(cli *cli.Context, opts *Options) error {
+	if t == nil {
+		return fmt.Errorf("toolkit installer is not initilized")
+	}
+	t.logger.Infof("Installing NVIDIA container toolkit to '%v'", t.toolkitRoot)
+
+	t.logger.Infof("Removing existing NVIDIA container toolkit installation")
+	err := os.RemoveAll(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error removing toolkit directory: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error removing toolkit directory: %v", err))
+	}
+
+	toolkitConfigDir := filepath.Join(t.toolkitRoot, ".config", "nvidia-container-runtime")
+	toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename)
+
+	err = t.createDirectories(t.toolkitRoot, toolkitConfigDir)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("could not create required directories: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not create required directories: %v", err))
+	}
+
+	err = t.installContainerLibraries(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA container library: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container library: %v", err))
+	}
+
+	err = t.installContainerRuntimes(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container runtime: %v", err))
+	}
+
+	nvidiaContainerCliExecutable, err := t.installContainerCLI(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA container CLI: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container CLI: %v", err))
+	}
+
+	nvidiaContainerRuntimeHookPath, err := t.installRuntimeHook(t.toolkitRoot, toolkitConfigPath)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err))
+	}
+
+	nvidiaCTKPath, err := t.installContainerToolkitCLI(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA Container Toolkit CLI: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA Container Toolkit CLI: %v", err))
+	}
+
+	nvidiaCDIHookPath, err := t.installContainerCDIHookCLI(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA Container CDI Hook CLI: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA Container CDI Hook CLI: %v", err))
+	}
+
+	err = t.installToolkitConfig(cli, toolkitConfigPath, nvidiaContainerCliExecutable, nvidiaCTKPath, nvidiaContainerRuntimeHookPath, opts)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container toolkit config: %v", err))
+	}
+
+	err = t.createDeviceNodes(opts)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error creating device nodes: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error creating device nodes: %v", err))
+	}
+
+	err = t.generateCDISpec(opts, nvidiaCDIHookPath)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error generating CDI specification: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error generating CDI specification: %v", err))
+	}
+
+	return nil
+}
+
+// installContainerLibraries locates and installs the libraries that are part of
+// the nvidia-container-toolkit.
+// A predefined set of library candidates are considered, with the first one
+// resulting in success being installed to the toolkit folder. The install process
+// resolves the symlink for the library and copies the versioned library itself.
+func (t *Installer) installContainerLibraries(toolkitRoot string) error {
+	t.logger.Infof("Installing NVIDIA container library to '%v'", toolkitRoot)
+
+	libs := []string{
+		"libnvidia-container.so.1",
+		"libnvidia-container-go.so.1",
+	}
+
+	for _, l := range libs {
+		err := t.installLibrary(l, toolkitRoot)
+		if err != nil {
+			return fmt.Errorf("failed to install %s: %v", l, err)
+		}
+	}
+
+	return nil
+}
+
+// installLibrary installs the specified library to the toolkit directory.
+func (t *Installer) installLibrary(libName string, toolkitRoot string) error {
+	libraryPath, err := t.findLibrary(libName)
+	if err != nil {
+		return fmt.Errorf("error locating NVIDIA container library: %v", err)
+	}
+
+	installedLibPath, err := t.installFileToFolder(toolkitRoot, libraryPath)
+	if err != nil {
+		return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitRoot, err)
+	}
+	t.logger.Infof("Installed '%v' to '%v'", libraryPath, installedLibPath)
+
+	if filepath.Base(installedLibPath) == libName {
+		return nil
+	}
+
+	err = t.installSymlink(toolkitRoot, libName, installedLibPath)
+	if err != nil {
+		return fmt.Errorf("error installing symlink for NVIDIA container library: %v", err)
+	}
+
+	return nil
+}
+
+// installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring
+// that the settings are updated to match the desired install and nvidia driver directories.
+func (t *Installer) installToolkitConfig(c *cli.Context, toolkitConfigPath string, nvidiaContainerCliExecutablePath string, nvidiaCTKPath string, nvidaContainerRuntimeHookPath string, opts *Options) error {
+	t.logger.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
+
+	cfg, err := config.New(
+		config.WithConfigFile(nvidiaContainerToolkitConfigSource),
+	)
+	if err != nil {
+		return fmt.Errorf("could not open source config file: %v", err)
+	}
+
+	targetConfig, err := os.Create(toolkitConfigPath)
+	if err != nil {
+		return fmt.Errorf("could not create target config file: %v", err)
+	}
+	defer targetConfig.Close()
+
+	// Read the ldconfig path from the config as this may differ per platform
+	// On ubuntu-based systems this ends in `.real`
+	ldconfigPath := fmt.Sprintf("%s", cfg.GetDefault("nvidia-container-cli.ldconfig", "/sbin/ldconfig"))
+	// Use the driver run root as the root:
+	driverLdconfigPath := config.NormalizeLDConfigPath("@" + filepath.Join(opts.DriverRoot, strings.TrimPrefix(ldconfigPath, "@/")))
+
+	configValues := map[string]interface{}{
+		// Set the options in the root toml table
+		"accept-nvidia-visible-devices-envvar-when-unprivileged": opts.acceptNVIDIAVisibleDevicesWhenUnprivileged,
+		"accept-nvidia-visible-devices-as-volume-mounts":         opts.acceptNVIDIAVisibleDevicesAsVolumeMounts,
+		// Set the nvidia-container-cli options
+		"nvidia-container-cli.root":     opts.DriverRoot,
+		"nvidia-container-cli.path":     nvidiaContainerCliExecutablePath,
+		"nvidia-container-cli.ldconfig": driverLdconfigPath,
+		// Set nvidia-ctk options
+		"nvidia-ctk.path": nvidiaCTKPath,
+		// Set the nvidia-container-runtime-hook options
+		"nvidia-container-runtime-hook.path":                nvidaContainerRuntimeHookPath,
+		"nvidia-container-runtime-hook.skip-mode-detection": opts.ContainerRuntimeHookSkipModeDetection,
+	}
+
+	toolkitRuntimeList := opts.ContainerRuntimeRuntimes.Value()
+	if len(toolkitRuntimeList) > 0 {
+		configValues["nvidia-container-runtime.runtimes"] = toolkitRuntimeList
+	}
+
+	for _, optInFeature := range opts.optInFeatures.Value() {
+		configValues["features."+optInFeature] = true
+	}
+
+	for key, value := range configValues {
+		cfg.Set(key, value)
+	}
+
+	// Set the optional config options
+	optionalConfigValues := map[string]interface{}{
+		"nvidia-container-runtime.debug":                         opts.ContainerRuntimeDebug,
+		"nvidia-container-runtime.log-level":                     opts.ContainerRuntimeLogLevel,
+		"nvidia-container-runtime.mode":                          opts.ContainerRuntimeMode,
+		"nvidia-container-runtime.modes.cdi.annotation-prefixes": opts.ContainerRuntimeModesCDIAnnotationPrefixes,
+		"nvidia-container-runtime.modes.cdi.default-kind":        opts.ContainerRuntimeModesCdiDefaultKind,
+		"nvidia-container-runtime.runtimes":                      opts.ContainerRuntimeRuntimes,
+		"nvidia-container-cli.debug":                             opts.ContainerCLIDebug,
+	}
+
+	for key, value := range optionalConfigValues {
+		if !c.IsSet(key) {
+			t.logger.Infof("Skipping unset option: %v", key)
+			continue
+		}
+		if value == nil {
+			t.logger.Infof("Skipping option with nil value: %v", key)
+			continue
+		}
+
+		switch v := value.(type) {
+		case string:
+			if v == "" {
+				continue
+			}
+		case cli.StringSlice:
+			if len(v.Value()) == 0 {
+				continue
+			}
+			value = v.Value()
+		default:
+			t.logger.Warningf("Unexpected type for option %v=%v: %T", key, value, v)
+		}
+
+		cfg.Set(key, value)
+	}
+
+	if _, err := cfg.WriteTo(targetConfig); err != nil {
+		return fmt.Errorf("error writing config: %v", err)
+	}
+
+	os.Stdout.WriteString("Using config:\n")
+	if _, err = cfg.WriteTo(os.Stdout); err != nil {
+		t.logger.Warningf("Failed to output config to STDOUT: %v", err)
+	}
+
+	return nil
+}
+
+// installContainerToolkitCLI installs the nvidia-ctk CLI executable and wrapper.
+func (t *Installer) installContainerToolkitCLI(toolkitDir string) (string, error) {
+	e := executable{
+		fileInstaller: t.fileInstaller,
+		source:        "/usr/bin/nvidia-ctk",
+		target: executableTarget{
+			dotfileName: "nvidia-ctk.real",
+			wrapperName: "nvidia-ctk",
+		},
+	}
+
+	return e.install(toolkitDir)
+}
+
+// installContainerCDIHookCLI installs the nvidia-cdi-hook CLI executable and wrapper.
+func (t *Installer) installContainerCDIHookCLI(toolkitDir string) (string, error) {
+	e := executable{
+		fileInstaller: t.fileInstaller,
+		source:        "/usr/bin/nvidia-cdi-hook",
+		target: executableTarget{
+			dotfileName: "nvidia-cdi-hook.real",
+			wrapperName: "nvidia-cdi-hook",
+		},
+	}
+
+	return e.install(toolkitDir)
+}
+
+// installContainerCLI sets up the NVIDIA container CLI executable, copying the executable
+// and implementing the required wrapper
+func (t *Installer) installContainerCLI(toolkitRoot string) (string, error) {
+	t.logger.Infof("Installing NVIDIA container CLI from '%v'", nvidiaContainerCliSource)
+
+	env := map[string]string{
+		"LD_LIBRARY_PATH": toolkitRoot,
+	}
+
+	e := executable{
+		fileInstaller: t.fileInstaller,
+		source:        nvidiaContainerCliSource,
+		target: executableTarget{
+			dotfileName: "nvidia-container-cli.real",
+			wrapperName: "nvidia-container-cli",
+		},
+		env: env,
+	}
+
+	installedPath, err := e.install(toolkitRoot)
+	if err != nil {
+		return "", fmt.Errorf("error installing NVIDIA container CLI: %v", err)
+	}
+	return installedPath, nil
+}
+
+// installRuntimeHook sets up the NVIDIA runtime hook, copying the executable
+// and implementing the required wrapper
+func (t *Installer) installRuntimeHook(toolkitRoot string, configFilePath string) (string, error) {
+	t.logger.Infof("Installing NVIDIA container runtime hook from '%v'", nvidiaContainerRuntimeHookSource)
+
+	argLines := []string{
+		fmt.Sprintf("-config \"%s\"", configFilePath),
+	}
+
+	e := executable{
+		fileInstaller: t.fileInstaller,
+		source:        nvidiaContainerRuntimeHookSource,
+		target: executableTarget{
+			dotfileName: "nvidia-container-runtime-hook.real",
+			wrapperName: "nvidia-container-runtime-hook",
+		},
+		argLines: argLines,
+	}
+
+	installedPath, err := e.install(toolkitRoot)
+	if err != nil {
+		return "", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
+	}
+
+	err = t.installSymlink(toolkitRoot, "nvidia-container-toolkit", installedPath)
+	if err != nil {
+		return "", fmt.Errorf("error installing symlink to NVIDIA container runtime hook: %v", err)
+	}
+
+	return installedPath, nil
+}
+
+// installSymlink creates a symlink in the toolkitDirectory that points to the specified target.
+// Note: The target is assumed to be local to the toolkit directory
+func (t *Installer) installSymlink(toolkitRoot string, link string, target string) error {
+	symlinkPath := filepath.Join(toolkitRoot, link)
+	targetPath := filepath.Base(target)
+	t.logger.Infof("Creating symlink '%v' -> '%v'", symlinkPath, targetPath)
+
+	err := os.Symlink(targetPath, symlinkPath)
+	if err != nil {
+		return fmt.Errorf("error creating symlink '%v' => '%v': %v", symlinkPath, targetPath, err)
+	}
+	return nil
+}
+
+// findLibrary searches a set of candidate libraries in the specified root for
+// a given library name
+func (t *Installer) findLibrary(libName string) (string, error) {
+	t.logger.Infof("Finding library %v (root=%v)", libName)
+
+	candidateDirs := []string{
+		"/usr/lib64",
+		"/usr/lib/x86_64-linux-gnu",
+		"/usr/lib/aarch64-linux-gnu",
+	}
+
+	for _, d := range candidateDirs {
+		l := filepath.Join(t.sourceRoot, d, libName)
+		t.logger.Infof("Checking library candidate '%v'", l)
+
+		libraryCandidate, err := t.resolveLink(l)
+		if err != nil {
+			t.logger.Infof("Skipping library candidate '%v': %v", l, err)
+			continue
+		}
+
+		return strings.TrimPrefix(libraryCandidate, t.sourceRoot), nil
+	}
+
+	return "", fmt.Errorf("error locating library '%v'", libName)
+}
+
+// resolveLink finds the target of a symlink or the file itself in the
+// case of a regular file.
+// This is equivalent to running `readlink -f ${l}`
+func (t *Installer) resolveLink(l string) (string, error) {
+	resolved, err := filepath.EvalSymlinks(l)
+	if err != nil {
+		return "", fmt.Errorf("error resolving link '%v': %v", l, err)
+	}
+	if l != resolved {
+		t.logger.Infof("Resolved link: '%v' => '%v'", l, resolved)
+	}
+	return resolved, nil
+}
+
+func (t *Installer) createDirectories(dir ...string) error {
+	for _, d := range dir {
+		t.logger.Infof("Creating directory '%v'", d)
+		err := os.MkdirAll(d, 0755)
+		if err != nil {
+			return fmt.Errorf("error creating directory: %v", err)
+		}
+	}
+	return nil
+}
+
+func (t *Installer) createDeviceNodes(opts *Options) error {
+	modes := opts.createDeviceNodes.Value()
+	if len(modes) == 0 {
+		return nil
+	}
+
+	devices, err := nvdevices.New(
+		nvdevices.WithDevRoot(opts.DevRootCtrPath),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create library: %v", err)
+	}
+
+	for _, mode := range modes {
+		t.logger.Infof("Creating %v device nodes at %v", mode, opts.DevRootCtrPath)
+		if mode != "control" {
+			t.logger.Warningf("Unrecognised device mode: %v", mode)
+			continue
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create control device nodes: %v", err)
+		}
+	}
+	return nil
+}
+
+// generateCDISpec generates a CDI spec for use in management containers
+func (t *Installer) generateCDISpec(opts *Options, nvidiaCDIHookPath string) error {
+	if !opts.CDI.Enabled {
+		return nil
+	}
+	t.logger.Info("Generating CDI spec for management containers")
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(t.logger),
+		nvcdi.WithMode(nvcdi.ModeManagement),
+		nvcdi.WithDriverRoot(opts.DriverRootCtrPath),
+		nvcdi.WithDevRoot(opts.DevRootCtrPath),
+		nvcdi.WithNVIDIACDIHookPath(nvidiaCDIHookPath),
+		nvcdi.WithVendor(opts.CDI.vendor),
+		nvcdi.WithClass(opts.CDI.class),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI library for management containers: %v", err)
+	}
+
+	spec, err := cdilib.GetSpec()
+	if err != nil {
+		return fmt.Errorf("failed to genereate CDI spec for management containers: %v", err)
+	}
+
+	transformer := transformroot.NewDriverTransformer(
+		transformroot.WithDriverRoot(opts.DriverRootCtrPath),
+		transformroot.WithTargetDriverRoot(opts.DriverRoot),
+		transformroot.WithDevRoot(opts.DevRootCtrPath),
+		transformroot.WithTargetDevRoot(opts.DevRoot),
+	)
+	if err := transformer.Transform(spec.Raw()); err != nil {
+		return fmt.Errorf("failed to transform driver root in CDI spec: %v", err)
+	}
+
+	name, err := cdi.GenerateNameForSpec(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI name for management containers: %v", err)
+	}
+	err = spec.Save(filepath.Join(opts.CDI.outputDir, name))
+	if err != nil {
+		return fmt.Errorf("failed to save CDI spec for management containers: %v", err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/toolkit/toolkit_test.go

@@ -0,0 +1,217 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package toolkit
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestInstall(t *testing.T) {
+	t.Setenv("__NVCT_TESTING_DEVICES_ARE_FILES", "true")
+	logger, _ := testlog.NewNullLogger()
+
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	artifactRoot := filepath.Join(moduleRoot, "testdata", "installer", "artifacts")
+
+	testCases := []struct {
+		description     string
+		hostRoot        string
+		packageType     string
+		cdiEnabled      bool
+		expectedError   error
+		expectedCdiSpec string
+	}{
+		{
+			hostRoot:    "rootfs-empty",
+			packageType: "deb",
+		},
+		{
+			hostRoot:    "rootfs-empty",
+			packageType: "rpm",
+		},
+		{
+			hostRoot:      "rootfs-empty",
+			packageType:   "deb",
+			cdiEnabled:    true,
+			expectedError: fmt.Errorf("no NVIDIA device nodes found"),
+		},
+		{
+			hostRoot:    "rootfs-1",
+			packageType: "deb",
+			cdiEnabled:  true,
+			expectedCdiSpec: `---
+cdiVersion: 0.5.0
+containerEdits:
+  env:
+  - NVIDIA_VISIBLE_DEVICES=void
+  hooks:
+  - args:
+    - nvidia-cdi-hook
+    - create-symlinks
+    - --link
+    - libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so
+    hookName: createContainer
+    path: {{ .toolkitRoot }}/nvidia-cdi-hook
+  - args:
+    - nvidia-cdi-hook
+    - update-ldcache
+    - --folder
+    - /lib/x86_64-linux-gnu
+    hookName: createContainer
+    path: {{ .toolkitRoot }}/nvidia-cdi-hook
+  mounts:
+  - containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77
+    hostPath: /host/driver/root/lib/x86_64-linux-gnu/libcuda.so.999.88.77
+    options:
+    - ro
+    - nosuid
+    - nodev
+    - bind
+devices:
+- containerEdits:
+    deviceNodes:
+    - hostPath: /host/driver/root/dev/nvidia0
+      path: /dev/nvidia0
+    - hostPath: /host/driver/root/dev/nvidiactl
+      path: /dev/nvidiactl
+    - hostPath: /host/driver/root/dev/nvidia-caps-imex-channels/channel0
+      path: /dev/nvidia-caps-imex-channels/channel0
+    - hostPath: /host/driver/root/dev/nvidia-caps-imex-channels/channel1
+      path: /dev/nvidia-caps-imex-channels/channel1
+    - hostPath: /host/driver/root/dev/nvidia-caps-imex-channels/channel2047
+      path: /dev/nvidia-caps-imex-channels/channel2047
+  name: all
+kind: example.com/class
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		// hostRoot := filepath.Join(moduleRoot, "testdata", "lookup", tc.hostRoot)
+		t.Run(tc.description, func(t *testing.T) {
+			testRoot := t.TempDir()
+			toolkitRoot := filepath.Join(testRoot, "toolkit-test")
+			cdiOutputDir := filepath.Join(moduleRoot, "toolkit-test", "/var/cdi")
+			sourceRoot := filepath.Join(artifactRoot, tc.packageType)
+			options := Options{
+				DriverRoot:        "/host/driver/root",
+				DriverRootCtrPath: filepath.Join(moduleRoot, "testdata", "lookup", tc.hostRoot),
+				CDI: cdiOptions{
+					Enabled:   tc.cdiEnabled,
+					outputDir: cdiOutputDir,
+					kind:      "example.com/class",
+				},
+			}
+
+			ti := NewInstaller(
+				WithLogger(logger),
+				WithToolkitRoot(toolkitRoot),
+				WithSourceRoot(sourceRoot),
+			)
+			require.NoError(t, ti.ValidateOptions(&options))
+
+			err := ti.Install(&cli.Context{}, &options)
+			if tc.expectedError == nil {
+				require.NoError(t, err)
+			} else {
+				require.Contains(t, err.Error(), tc.expectedError.Error())
+			}
+
+			require.DirExists(t, toolkitRoot)
+			requireSymlink(t, toolkitRoot, "libnvidia-container.so.1", "libnvidia-container.so.99.88.77")
+			requireSymlink(t, toolkitRoot, "libnvidia-container-go.so.1", "libnvidia-container-go.so.99.88.77")
+
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-cdi-hook")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-cli")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime-hook")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime.cdi")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime.legacy")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-ctk")
+
+			requireSymlink(t, toolkitRoot, "nvidia-container-toolkit", "nvidia-container-runtime-hook")
+
+			// TODO: Add checks for wrapper contents
+			// grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+			// grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+
+			require.DirExists(t, filepath.Join(toolkitRoot, ".config"))
+			require.DirExists(t, filepath.Join(toolkitRoot, ".config", "nvidia-container-runtime"))
+			require.FileExists(t, filepath.Join(toolkitRoot, ".config", "nvidia-container-runtime", "config.toml"))
+
+			cfgToml, err := config.New(config.WithConfigFile(filepath.Join(toolkitRoot, ".config", "nvidia-container-runtime", "config.toml")))
+			require.NoError(t, err)
+
+			cfg, err := cfgToml.Config()
+			require.NoError(t, err)
+
+			// Ensure that the config file has the required contents.
+			// TODO: Add checks for additional config options.
+			require.Equal(t, "/host/driver/root", cfg.NVIDIAContainerCLIConfig.Root)
+			require.Equal(t, "@/host/driver/root/sbin/ldconfig", string(cfg.NVIDIAContainerCLIConfig.Ldconfig))
+			require.EqualValues(t, filepath.Join(toolkitRoot, "nvidia-container-cli"), cfg.NVIDIAContainerCLIConfig.Path)
+			require.EqualValues(t, filepath.Join(toolkitRoot, "nvidia-ctk"), cfg.NVIDIACTKConfig.Path)
+
+			if len(tc.expectedCdiSpec) > 0 {
+				cdiSpecFile := filepath.Join(cdiOutputDir, "example.com-class.yaml")
+				require.FileExists(t, cdiSpecFile)
+				info, err := os.Stat(cdiSpecFile)
+				require.NoError(t, err)
+				require.NotZero(t, info.Mode()&0004)
+				contents, err := os.ReadFile(cdiSpecFile)
+				require.NoError(t, err)
+				require.Equal(t, strings.ReplaceAll(tc.expectedCdiSpec, "{{ .toolkitRoot }}", toolkitRoot), string(contents))
+			}
+		})
+	}
+}
+
+func requireWrappedExecutable(t *testing.T, toolkitRoot string, expectedExecutable string) {
+	requireExecutable(t, toolkitRoot, expectedExecutable)
+	requireExecutable(t, toolkitRoot, expectedExecutable+".real")
+}
+
+func requireExecutable(t *testing.T, toolkitRoot string, expectedExecutable string) {
+	executable := filepath.Join(toolkitRoot, expectedExecutable)
+	require.FileExists(t, executable)
+	info, err := os.Lstat(executable)
+	require.NoError(t, err)
+	require.Zero(t, info.Mode()&os.ModeSymlink)
+	require.NotZero(t, info.Mode()&0111)
+}
+
+func requireSymlink(t *testing.T, toolkitRoot string, expectedLink string, expectedTarget string) {
+	link := filepath.Join(toolkitRoot, expectedLink)
+	require.FileExists(t, link)
+	target, err := symlinks.Resolve(link)
+	require.NoError(t, err)
+	require.Equal(t, expectedTarget, target)
+}

cmd/nvidia-ctk-installer/main.go

@@ -0,0 +1,327 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/sys/unix"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/toolkit"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+const (
+	toolkitPidFilename = "toolkit.pid"
+	defaultPidFile     = "/run/nvidia/toolkit/" + toolkitPidFilename
+	toolkitSubDir      = "toolkit"
+
+	defaultRuntime     = "docker"
+	defaultRuntimeArgs = ""
+)
+
+var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}}
+var defaultLowLevelRuntimes = []string{"docker-runc", "runc", "crun"}
+
+var waitingForSignal = make(chan bool, 1)
+var signalReceived = make(chan bool, 1)
+
+// options stores the command line arguments
+type options struct {
+	noDaemon    bool
+	runtime     string
+	runtimeArgs string
+	root        string
+	pidFile     string
+	sourceRoot  string
+
+	toolkitOptions toolkit.Options
+	runtimeOptions runtime.Options
+}
+
+func (o options) toolkitRoot() string {
+	return filepath.Join(o.root, toolkitSubDir)
+}
+
+// Version defines the CLI version. This is set at build time using LD FLAGS
+var Version = "development"
+
+func main() {
+	logger := logger.New()
+
+	remainingArgs, root, err := ParseArgs(logger, os.Args)
+	if err != nil {
+		logger.Errorf("Error: unable to parse arguments: %v", err)
+		os.Exit(1)
+	}
+
+	c := NewApp(logger, root)
+
+	// Run the CLI
+	logger.Infof("Starting %v", c.Name)
+	if err := c.Run(remainingArgs); err != nil {
+		logger.Errorf("error running nvidia-toolkit: %v", err)
+		os.Exit(1)
+	}
+
+	logger.Infof("Completed %v", c.Name)
+}
+
+// An app represents the nvidia-ctk-installer.
+type app struct {
+	logger logger.Interface
+	// defaultRoot stores the root to use if the --root flag is not specified.
+	defaultRoot string
+
+	toolkit *toolkit.Installer
+}
+
+// NewApp creates the CLI app fro the specified options.
+// defaultRoot is used as the root if not specified via the --root flag.
+func NewApp(logger logger.Interface, defaultRoot string) *cli.App {
+	a := app{
+		logger:      logger,
+		defaultRoot: defaultRoot,
+	}
+	return a.build()
+}
+
+func (a app) build() *cli.App {
+	options := options{
+		toolkitOptions: toolkit.Options{},
+	}
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "nvidia-toolkit"
+	c.Usage = "Install the nvidia-container-toolkit for use by a given runtime"
+	c.UsageText = "[DESTINATION] [-n | --no-daemon] [-r | --runtime] [-u | --runtime-args]"
+	c.Description = "DESTINATION points to the host path underneath which the nvidia-container-toolkit should be installed.\nIt will be installed at ${DESTINATION}/toolkit"
+	c.Version = Version
+	c.Before = func(ctx *cli.Context) error {
+		return a.Before(ctx, &options)
+	}
+	c.Action = func(ctx *cli.Context) error {
+		return a.Run(ctx, &options)
+	}
+
+	// Setup flags for the CLI
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "no-daemon",
+			Aliases:     []string{"n"},
+			Usage:       "terminate immediately after setting up the runtime. Note that no cleanup will be performed",
+			Destination: &options.noDaemon,
+			EnvVars:     []string{"NO_DAEMON"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Aliases:     []string{"r"},
+			Usage:       "the runtime to setup on this node. One of {'docker', 'crio', 'containerd'}",
+			Value:       defaultRuntime,
+			Destination: &options.runtime,
+			EnvVars:     []string{"RUNTIME"},
+		},
+		// TODO: Remove runtime-args
+		&cli.StringFlag{
+			Name:        "runtime-args",
+			Aliases:     []string{"u"},
+			Usage:       "arguments to pass to 'docker', 'crio', or 'containerd' setup command",
+			Value:       defaultRuntimeArgs,
+			Destination: &options.runtimeArgs,
+			EnvVars:     []string{"RUNTIME_ARGS"},
+		},
+		&cli.StringFlag{
+			Name:        "root",
+			Value:       a.defaultRoot,
+			Usage:       "the folder where the NVIDIA Container Toolkit is to be installed. It will be installed to `ROOT`/toolkit",
+			Destination: &options.root,
+			EnvVars:     []string{"ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "source-root",
+			Value:       "/",
+			Usage:       "The folder where the required toolkit artifacts can be found",
+			Destination: &options.sourceRoot,
+			EnvVars:     []string{"SOURCE_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "pid-file",
+			Value:       defaultPidFile,
+			Usage:       "the path to a toolkit.pid file to ensure that only a single configuration instance is running",
+			Destination: &options.pidFile,
+			EnvVars:     []string{"TOOLKIT_PID_FILE", "PID_FILE"},
+		},
+	}
+
+	c.Flags = append(c.Flags, toolkit.Flags(&options.toolkitOptions)...)
+	c.Flags = append(c.Flags, runtime.Flags(&options.runtimeOptions)...)
+
+	return c
+}
+
+func (a *app) Before(c *cli.Context, o *options) error {
+	a.toolkit = toolkit.NewInstaller(
+		toolkit.WithLogger(a.logger),
+		toolkit.WithSourceRoot(o.sourceRoot),
+		toolkit.WithToolkitRoot(o.toolkitRoot()),
+	)
+	return a.validateFlags(c, o)
+}
+
+func (a *app) validateFlags(c *cli.Context, o *options) error {
+	if o.root == "" {
+		return fmt.Errorf("the install root must be specified")
+	}
+	if _, exists := availableRuntimes[o.runtime]; !exists {
+		return fmt.Errorf("unknown runtime: %v", o.runtime)
+	}
+	if filepath.Base(o.pidFile) != toolkitPidFilename {
+		return fmt.Errorf("invalid toolkit.pid path %v", o.pidFile)
+	}
+
+	if err := a.toolkit.ValidateOptions(&o.toolkitOptions); err != nil {
+		return err
+	}
+	if err := runtime.ValidateOptions(c, &o.runtimeOptions, o.runtime, o.toolkitRoot(), &o.toolkitOptions); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Run installs the NVIDIA Container Toolkit and updates the requested runtime.
+// If the application is run as a daemon, the application waits and unconfigures
+// the runtime on termination.
+func (a *app) Run(c *cli.Context, o *options) error {
+	err := a.initialize(o.pidFile)
+	if err != nil {
+		return fmt.Errorf("unable to initialize: %v", err)
+	}
+	defer a.shutdown(o.pidFile)
+
+	if len(o.toolkitOptions.ContainerRuntimeRuntimes.Value()) == 0 {
+		lowlevelRuntimePaths, err := runtime.GetLowlevelRuntimePaths(&o.runtimeOptions, o.runtime)
+		if err != nil {
+			return fmt.Errorf("unable to determine runtime options: %w", err)
+		}
+		lowlevelRuntimePaths = append(lowlevelRuntimePaths, defaultLowLevelRuntimes...)
+
+		o.toolkitOptions.ContainerRuntimeRuntimes = *cli.NewStringSlice(lowlevelRuntimePaths...)
+	}
+
+	err = a.toolkit.Install(c, &o.toolkitOptions)
+	if err != nil {
+		return fmt.Errorf("unable to install toolkit: %v", err)
+	}
+
+	err = runtime.Setup(c, &o.runtimeOptions, o.runtime)
+	if err != nil {
+		return fmt.Errorf("unable to setup runtime: %v", err)
+	}
+
+	if !o.noDaemon {
+		err = a.waitForSignal()
+		if err != nil {
+			return fmt.Errorf("unable to wait for signal: %v", err)
+		}
+
+		err = runtime.Cleanup(c, &o.runtimeOptions, o.runtime)
+		if err != nil {
+			return fmt.Errorf("unable to cleanup runtime: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// ParseArgs checks if a single positional argument was defined and extracts this the root.
+// If no positional arguments are defined, it is assumed that the root is specified as a flag.
+func ParseArgs(logger logger.Interface, args []string) ([]string, string, error) {
+	logger.Infof("Parsing arguments")
+
+	if len(args) < 2 {
+		return args, "", nil
+	}
+
+	var lastPositionalArg int
+	for i, arg := range args {
+		if strings.HasPrefix(arg, "-") {
+			break
+		}
+		lastPositionalArg = i
+	}
+
+	if lastPositionalArg == 0 {
+		return args, "", nil
+	}
+
+	if lastPositionalArg == 1 {
+		return append([]string{args[0]}, args[2:]...), args[1], nil
+	}
+
+	return nil, "", fmt.Errorf("unexpected positional argument(s) %v", args[2:lastPositionalArg+1])
+}
+
+func (a *app) initialize(pidFile string) error {
+	a.logger.Infof("Initializing")
+
+	if dir := filepath.Dir(pidFile); dir != "" {
+		err := os.MkdirAll(dir, 0755)
+		if err != nil {
+			return fmt.Errorf("unable to create folder for pidfile: %w", err)
+		}
+	}
+
+	f, err := os.Create(pidFile)
+	if err != nil {
+		return fmt.Errorf("unable to create pidfile: %v", err)
+	}
+
+	err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	if err != nil {
+		a.logger.Warningf("Unable to get exclusive lock on '%v'", pidFile)
+		a.logger.Warningf("This normally means an instance of the NVIDIA toolkit Container is already running, aborting")
+		return fmt.Errorf("unable to get flock on pidfile: %v", err)
+	}
+
+	_, err = f.WriteString(fmt.Sprintf("%v\n", os.Getpid()))
+	if err != nil {
+		return fmt.Errorf("unable to write PID to pidfile: %v", err)
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGPIPE, syscall.SIGTERM)
+	go func() {
+		<-sigs
+		select {
+		case <-waitingForSignal:
+			signalReceived <- true
+		default:
+			a.logger.Infof("Signal received, exiting early")
+			a.shutdown(pidFile)
+			os.Exit(0)
+		}
+	}()
+
+	return nil
+}
+
+func (a *app) waitForSignal() error {
+	a.logger.Infof("Waiting for signal")
+	waitingForSignal <- true
+	<-signalReceived
+	return nil
+}
+
+func (a *app) shutdown(pidFile string) {
+	a.logger.Infof("Shutting Down")
+
+	err := os.Remove(pidFile)
+	if err != nil {
+		a.logger.Warningf("Unable to remove pidfile: %v", err)
+	}
+}

cmd/nvidia-ctk-installer/main_test.go

@@ -0,0 +1,501 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestParseArgs(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	testCases := []struct {
+		args              []string
+		expectedRemaining []string
+		expectedRoot      string
+		expectedError     error
+	}{
+		{
+			args:              []string{},
+			expectedRemaining: []string{},
+			expectedRoot:      "",
+			expectedError:     nil,
+		},
+		{
+			args:              []string{"app"},
+			expectedRemaining: []string{"app"},
+		},
+		{
+			args:              []string{"app", "root"},
+			expectedRemaining: []string{"app"},
+			expectedRoot:      "root",
+		},
+		{
+			args:              []string{"app", "--flag"},
+			expectedRemaining: []string{"app", "--flag"},
+		},
+		{
+			args:              []string{"app", "root", "--flag"},
+			expectedRemaining: []string{"app", "--flag"},
+			expectedRoot:      "root",
+		},
+		{
+			args:          []string{"app", "root", "not-root", "--flag"},
+			expectedError: fmt.Errorf("unexpected positional argument(s) [not-root]"),
+		},
+		{
+			args:          []string{"app", "root", "not-root"},
+			expectedError: fmt.Errorf("unexpected positional argument(s) [not-root]"),
+		},
+		{
+			args:          []string{"app", "root", "not-root", "also"},
+			expectedError: fmt.Errorf("unexpected positional argument(s) [not-root also]"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			remaining, root, err := ParseArgs(logger, tc.args)
+			if tc.expectedError != nil {
+				require.EqualError(t, err, tc.expectedError.Error())
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.ElementsMatch(t, tc.expectedRemaining, remaining)
+			require.Equal(t, tc.expectedRoot, root)
+		})
+	}
+}
+
+func TestApp(t *testing.T) {
+	t.Setenv("__NVCT_TESTING_DEVICES_ARE_FILES", "true")
+	logger, _ := testlog.NewNullLogger()
+
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	artifactRoot := filepath.Join(moduleRoot, "testdata", "installer", "artifacts")
+	hostRoot := filepath.Join(moduleRoot, "testdata", "lookup", "rootfs-1")
+
+	testCases := []struct {
+		description           string
+		args                  []string
+		expectedToolkitConfig string
+		expectedRuntimeConfig string
+	}{
+		{
+			description: "no args",
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["docker-runc", "runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+        },
+        "nvidia-cdi": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+        },
+        "nvidia-legacy": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+        }
+    }
+}`,
+		},
+		{
+			description: "CDI enabled enables CDI in docker",
+			args:        []string{"--cdi-enabled", "--create-device-nodes=none"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["docker-runc", "runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `{
+    "default-runtime": "nvidia",
+    "features": {
+        "cdi": true
+    },
+    "runtimes": {
+        "nvidia": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+        },
+        "nvidia-cdi": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+        },
+        "nvidia-legacy": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+        }
+    }
+}`,
+		},
+		{
+			description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in Docker",
+			args:        []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["docker-runc", "runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+        },
+        "nvidia-cdi": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+        },
+        "nvidia-legacy": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+        }
+    }
+}`,
+		},
+		{
+			description: "CDI enabled enables CDI in containerd",
+			args:        []string{"--cdi-enabled", "--runtime=containerd"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["docker-runc", "runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `version = 2
+
+[plugins]
+
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_cdi = true
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+`,
+		},
+		{
+			description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in containerd",
+			args:        []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false", "--runtime=containerd"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["docker-runc", "runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `version = 2
+
+[plugins]
+
+  [plugins."io.containerd.grpc.v1.cri"]
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			testRoot := t.TempDir()
+
+			cdiOutputDir := filepath.Join(testRoot, "/var/run/cdi")
+			runtimeConfigFile := filepath.Join(testRoot, "config.file")
+
+			toolkitRoot := filepath.Join(testRoot, "toolkit-test")
+			toolkitConfigFile := filepath.Join(toolkitRoot, "toolkit/.config/nvidia-container-runtime/config.toml")
+
+			app := NewApp(logger, toolkitRoot)
+
+			testArgs := []string{
+				"nvidia-ctk-installer",
+				"--no-daemon",
+				"--cdi-output-dir=" + cdiOutputDir,
+				"--config=" + runtimeConfigFile,
+				"--create-device-nodes=none",
+				"--driver-root-ctr-path=" + hostRoot,
+				"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
+				"--restart-mode=none",
+				"--source-root=" + filepath.Join(artifactRoot, "deb"),
+			}
+
+			err := app.Run(append(testArgs, tc.args...))
+
+			require.NoError(t, err)
+
+			require.FileExists(t, toolkitConfigFile)
+			toolkitConfigFileContents, err := os.ReadFile(toolkitConfigFile)
+			require.NoError(t, err)
+			require.EqualValues(t, strings.ReplaceAll(tc.expectedToolkitConfig, "{{ .toolkitRoot }}", toolkitRoot), string(toolkitConfigFileContents))
+
+			require.FileExists(t, runtimeConfigFile)
+			runtimeConfigFileContents, err := os.ReadFile(runtimeConfigFile)
+			require.NoError(t, err)
+			require.EqualValues(t, strings.ReplaceAll(tc.expectedRuntimeConfig, "{{ .toolkitRoot }}", toolkitRoot), string(runtimeConfigFileContents))
+		})
+	}
+
+}

cmd/nvidia-ctk/README.md

@@ -1,3 +1,73 @@
 # NVIDIA Container Toolkit CLI
 
 The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.
+
+## Configure the NVIDIA Container Toolkit
+
+The `config` command of the `nvidia-ctk` CLI allows a user to display and manipulate the NVIDIA Container Toolkit
+configuration.
+
+For example, running the following command:
+```bash
+nvidia-ctk config default
+```
+will display the default config for the detected platform.
+
+Whereas
+```bash
+nvidia-ctk config
+```
+will display the effective NVIDIA Container Toolkit config using the configured config file, and running:
+
+Individual config options can be set by specifying these are key-value pairs to the `--set` argument:
+
+```bash
+nvidia-ctk config --set nvidia-container-cli.no-cgroups=true
+```
+
+By default, all commands output to `STDOUT`, but specifying the `--output` flag writes the config to the specified file.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://tags.cncf.io/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,55 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/list"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
+		list.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,299 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	cdi "tags.cncf.io/container-device-interface/pkg/parser"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+)
+
+const (
+	allDeviceName = "all"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	output               string
+	format               string
+	deviceNameStrategies cli.StringSlice
+	driverRoot           string
+	devRoot              string
+	nvidiaCDIHookPath    string
+	ldconfigPath         string
+	mode                 string
+	vendor               string
+	class                string
+
+	configSearchPaths  cli.StringSlice
+	librarySearchPaths cli.StringSlice
+
+	csv struct {
+		files          cli.StringSlice
+		ignorePatterns cli.StringSlice
+	}
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "config-search-path",
+			Usage:       "Specify the path to search for config files when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.configSearchPaths,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       spec.FormatYAML,
+			Destination: &opts.format,
+		},
+		&cli.StringFlag{
+			Name:    "mode",
+			Aliases: []string{"discovery-mode"},
+			Usage: "The mode to use when discovering the available entities. " +
+				"One of [" + strings.Join(nvcdi.AllModes[string](), " | ") + "]. " +
+				"If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       string(nvcdi.ModeAuto),
+			Destination: &opts.mode,
+		},
+		&cli.StringFlag{
+			Name:        "dev-root",
+			Usage:       "Specify the root where `/dev` is located. If this is not specified, the driver-root is assumed.",
+			Destination: &opts.devRoot,
+		},
+		&cli.StringSliceFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. If this is specified multiple times, the devices will be duplicated for each strategy. One of [index | uuid | type-index]",
+			Value:       cli.NewStringSlice(nvcdi.DeviceNameStrategyIndex, nvcdi.DeviceNameStrategyUUID),
+			Destination: &opts.deviceNameStrategies,
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.driverRoot,
+		},
+		&cli.StringSliceFlag{
+			Name:        "library-search-path",
+			Usage:       "Specify the path to search for libraries when discovering the entities that should be included in the CDI specification.\n\tNote: This option only applies to CSV mode.",
+			Destination: &opts.librarySearchPaths,
+		},
+		&cli.StringFlag{
+			Name:    "nvidia-cdi-hook-path",
+			Aliases: []string{"nvidia-ctk-path"},
+			Usage: "Specify the path to use for the nvidia-cdi-hook in the generated CDI specification. " +
+				"If not specified, the PATH will be searched for `nvidia-cdi-hook`. " +
+				"NOTE: That if this is specified as `nvidia-ctk`, the PATH will be searched for `nvidia-ctk` instead.",
+			Destination: &opts.nvidiaCDIHookPath,
+		},
+		&cli.StringFlag{
+			Name:        "ldconfig-path",
+			Usage:       "Specify the path to use for ldconfig in the generated CDI specification",
+			Destination: &opts.ldconfigPath,
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &opts.vendor,
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &opts.class,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.file",
+			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CSV mode.",
+			Value:       cli.NewStringSlice(csv.DefaultFileList()...),
+			Destination: &opts.csv.files,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.ignore-pattern",
+			Usage:       "Specify a pattern the CSV mount specifications.",
+			Destination: &opts.csv.ignorePatterns,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	opts.format = strings.ToLower(opts.format)
+	switch opts.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", opts.format)
+	}
+
+	opts.mode = strings.ToLower(opts.mode)
+	if !nvcdi.IsValidMode(opts.mode) {
+		return fmt.Errorf("invalid discovery mode: %v", opts.mode)
+	}
+
+	for _, strategy := range opts.deviceNameStrategies.Value() {
+		_, err := nvcdi.NewDeviceNamer(strategy)
+		if err != nil {
+			return err
+		}
+	}
+
+	opts.nvidiaCDIHookPath = config.ResolveNVIDIACDIHookPath(m.logger, opts.nvidiaCDIHookPath)
+
+	if outputFileFormat := formatFromFilename(opts.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			opts.format = outputFileFormat
+		} else if outputFileFormat != opts.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", opts.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(opts.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(opts.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := m.generateSpec(opts)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
+
+	if opts.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return spec.Save(opts.output)
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
+	}
+
+	return ""
+}
+
+func (m command) generateSpec(opts *options) (spec.Interface, error) {
+	var deviceNamers []nvcdi.DeviceNamer
+	for _, strategy := range opts.deviceNameStrategies.Value() {
+		deviceNamer, err := nvcdi.NewDeviceNamer(strategy)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create device namer: %v", err)
+		}
+		deviceNamers = append(deviceNamers, deviceNamer)
+	}
+
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(opts.driverRoot),
+		nvcdi.WithDevRoot(opts.devRoot),
+		nvcdi.WithNVIDIACDIHookPath(opts.nvidiaCDIHookPath),
+		nvcdi.WithLdconfigPath(opts.ldconfigPath),
+		nvcdi.WithDeviceNamers(deviceNamers...),
+		nvcdi.WithMode(opts.mode),
+		nvcdi.WithConfigSearchPaths(opts.configSearchPaths.Value()),
+		nvcdi.WithLibrarySearchPaths(opts.librarySearchPaths.Value()),
+		nvcdi.WithCSVFiles(opts.csv.files.Value()),
+		nvcdi.WithCSVIgnorePatterns(opts.csv.ignorePatterns.Value()),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
+	}
+
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+
+	commonEdits, err := cdilib.GetCommonEdits()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
+	}
+
+	return spec.New(
+		spec.WithVendor(opts.vendor),
+		spec.WithClass(opts.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(opts.format),
+		spec.WithMergedDeviceOptions(
+			transform.WithName(allDeviceName),
+			transform.WithSkipIfExists(true),
+		),
+		spec.WithPermissions(0644),
+	)
+}

cmd/nvidia-ctk/cdi/list/list.go

@@ -0,0 +1,104 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package list
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	cdiSpecDirs cli.StringSlice
+}
+
+// NewCommand constructs a cdi list command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the command
+	c := cli.Command{
+		Name:  "list",
+		Usage: "List the available CDI devices",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "spec-dir",
+			Usage:       "specify the directories to scan for CDI specifications",
+			Value:       cli.NewStringSlice(cdi.DefaultSpecDirs...),
+			Destination: &cfg.cdiSpecDirs,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+	if len(cfg.cdiSpecDirs.Value()) == 0 {
+		return errors.New("at least one CDI specification directory must be specified")
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	registry, err := cdi.NewCache(
+		cdi.WithAutoRefresh(false),
+		cdi.WithSpecDirs(cfg.cdiSpecDirs.Value()...),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI cache: %v", err)
+	}
+
+	_ = registry.Refresh()
+	if errors := registry.GetErrors(); len(errors) > 0 {
+		m.logger.Warningf("The following registry errors were reported:")
+		for k, err := range errors {
+			m.logger.Warningf("%v: %v", k, err)
+		}
+	}
+
+	devices := registry.ListDevices()
+	m.logger.Infof("Found %d CDI devices", len(devices))
+	for _, device := range devices {
+		fmt.Printf("%s\n", device)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,169 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	transformroot "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform/root"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from       string
+	to         string
+	relativeTo string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "relative-to",
+			Usage:       "specify whether the transform is relative to the host or to the container. One of [ host | container ]",
+			Value:       "host",
+			Destination: &opts.relativeTo,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	switch opts.relativeTo {
+	case "host":
+	case "container":
+	default:
+		return fmt.Errorf("invalid --relative-to value: %v", opts.relativeTo)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transformroot.New(
+		transformroot.WithRoot(opts.from),
+		transformroot.WithTargetRoot(opts.to),
+		transformroot.WithRelativeTo(opts.relativeTo),
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,52 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/config/config.go

@@ -0,0 +1,244 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	createdefault "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/create-default"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// options stores the subcommand options
+type options struct {
+	flags.Options
+	setListSeparator string
+	sets             cli.StringSlice
+}
+
+// NewCommand constructs an config command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'config' command
+	c := cli.Command{
+		Name:  "config",
+		Usage: "Interact with the NVIDIA Container Toolkit configuration",
+		Before: func(ctx *cli.Context) error {
+			return validateFlags(ctx, &opts)
+		},
+		Action: func(ctx *cli.Context) error {
+			return run(ctx, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config-file",
+			Aliases:     []string{"config", "c"},
+			Usage:       "Specify the config file to modify.",
+			Value:       config.GetConfigFilePath(),
+			Destination: &opts.Config,
+		},
+		&cli.StringSliceFlag{
+			Name: "set",
+			Usage: "Set a config value using the pattern 'key[=value]'. " +
+				"Specifying only 'key' is equivalent to 'key=true' for boolean settings. " +
+				"This flag can be specified multiple times, but only the last value for a specific " +
+				"config option is applied. " +
+				"If the setting represents a list, the elements are colon-separated.",
+			Destination: &opts.sets,
+		},
+		&cli.StringFlag{
+			Name:        "set-list-separator",
+			Usage:       "Specify a separator for lists applied using the set command.",
+			Hidden:      true,
+			Value:       ":",
+			Destination: &opts.setListSeparator,
+		},
+		&cli.BoolFlag{
+			Name:        "in-place",
+			Aliases:     []string{"i"},
+			Usage:       "Modify the config file in-place",
+			Destination: &opts.InPlace,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	c.Subcommands = []*cli.Command{
+		createdefault.NewCommand(m.logger),
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, opts *options) error {
+	if opts.setListSeparator == "" {
+		return fmt.Errorf("set-list-separator must be set")
+	}
+	return nil
+}
+
+func run(c *cli.Context, opts *options) error {
+	cfgToml, err := config.New(
+		config.WithConfigFile(opts.Config),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create config: %v", err)
+	}
+
+	for _, set := range opts.sets.Value() {
+		key, value, err := setFlagToKeyValue(set, opts.setListSeparator)
+		if err != nil {
+			return fmt.Errorf("invalid --set option %v: %w", set, err)
+		}
+		if value == nil {
+			_ = cfgToml.Delete(key)
+		} else {
+			cfgToml.Set(key, value)
+		}
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	if _, err := cfgToml.Save(output); err != nil {
+		return fmt.Errorf("failed to save config: %v", err)
+	}
+
+	return nil
+}
+
+var errInvalidConfigOption = errors.New("invalid config option")
+var errUndefinedField = errors.New("undefined field")
+var errInvalidFormat = errors.New("invalid format")
+
+// setFlagToKeyValue converts a --set flag to a key-value pair.
+// The set flag is of the form key[=value], with the value being optional if key refers to a
+// boolean config option.
+func setFlagToKeyValue(setFlag string, setListSeparator string) (string, interface{}, error) {
+	setParts := strings.SplitN(setFlag, "=", 2)
+	key := setParts[0]
+
+	field, err := getField(key)
+	if err != nil {
+		return key, nil, fmt.Errorf("%w: %w", errInvalidConfigOption, err)
+	}
+
+	kind := field.Kind()
+	if len(setParts) != 2 {
+		if kind == reflect.Bool || (kind == reflect.Pointer && field.Elem().Kind() == reflect.Bool) {
+			return key, true, nil
+		}
+		return key, nil, fmt.Errorf("%w: expected key=value; got %v", errInvalidFormat, setFlag)
+	}
+
+	value := setParts[1]
+	if kind == reflect.Pointer && value != "nil" {
+		kind = field.Elem().Kind()
+	}
+	switch kind {
+	case reflect.Pointer:
+		return key, nil, nil
+	case reflect.Bool:
+		b, err := strconv.ParseBool(value)
+		if err != nil {
+			return key, value, fmt.Errorf("%w: %w", errInvalidFormat, err)
+		}
+		return key, b, nil
+	case reflect.String:
+		return key, value, nil
+	case reflect.Slice:
+		valueParts := strings.Split(value, setListSeparator)
+		switch field.Elem().Kind() {
+		case reflect.String:
+			return key, valueParts, nil
+		case reflect.Int:
+			var output []int64
+			for _, v := range valueParts {
+				vi, err := strconv.ParseInt(v, 10, 0)
+				if err != nil {
+					return key, nil, fmt.Errorf("%w: %w", errInvalidFormat, err)
+				}
+				output = append(output, vi)
+			}
+			return key, output, nil
+		}
+	}
+	return key, nil, fmt.Errorf("unsupported type for %v (%v)", setParts, kind)
+}
+
+func getField(key string) (reflect.Type, error) {
+	s, err := getStruct(reflect.TypeOf(config.Config{}), strings.Split(key, ".")...)
+	if err != nil {
+		return nil, err
+	}
+	return s.Type, err
+}
+
+func getStruct(current reflect.Type, paths ...string) (reflect.StructField, error) {
+	if len(paths) < 1 {
+		return reflect.StructField{}, fmt.Errorf("%w: no fields selected", errUndefinedField)
+	}
+	tomlField := paths[0]
+	for i := 0; i < current.NumField(); i++ {
+		f := current.Field(i)
+		v, ok := f.Tag.Lookup("toml")
+		if !ok {
+			continue
+		}
+		if strings.SplitN(v, ",", 2)[0] != tomlField {
+			continue
+		}
+		if len(paths) == 1 {
+			return f, nil
+		}
+		return getStruct(f.Type, paths[1:]...)
+	}
+	return reflect.StructField{}, fmt.Errorf("%w: %q", errUndefinedField, tomlField)
+}

cmd/nvidia-ctk/config/config_test.go

@@ -0,0 +1,143 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetFlagToKeyValue(t *testing.T) {
+	// TODO: We need to enable this test again since switching to reflect.
+	testCases := []struct {
+		description      string
+		setFlag          string
+		setListSeparator string
+		expectedKey      string
+		expectedValue    interface{}
+		expectedError    error
+	}{
+		{
+			description:   "option not present returns an error",
+			setFlag:       "undefined=new-value",
+			expectedKey:   "undefined",
+			expectedError: errInvalidConfigOption,
+		},
+		{
+			description:   "undefined nexted option returns error",
+			setFlag:       "nvidia-container-cli.undefined",
+			expectedKey:   "nvidia-container-cli.undefined",
+			expectedError: errInvalidConfigOption,
+		},
+		{
+			description:   "boolean option assumes true",
+			setFlag:       "disable-require",
+			expectedKey:   "disable-require",
+			expectedValue: true,
+		},
+		{
+			description:   "boolean option returns true",
+			setFlag:       "disable-require=true",
+			expectedKey:   "disable-require",
+			expectedValue: true,
+		},
+		{
+			description:   "boolean option returns false",
+			setFlag:       "disable-require=false",
+			expectedKey:   "disable-require",
+			expectedValue: false,
+		},
+		{
+			description:   "invalid boolean option returns error",
+			setFlag:       "disable-require=something",
+			expectedKey:   "disable-require",
+			expectedValue: "something",
+			expectedError: errInvalidFormat,
+		},
+		{
+			description:   "string option requires value",
+			setFlag:       "swarm-resource",
+			expectedKey:   "swarm-resource",
+			expectedValue: nil,
+			expectedError: errInvalidFormat,
+		},
+		{
+			description:   "string option returns value",
+			setFlag:       "swarm-resource=string-value",
+			expectedKey:   "swarm-resource",
+			expectedValue: "string-value",
+		},
+		{
+			description:   "string option returns value with equals",
+			setFlag:       "swarm-resource=string-value=more",
+			expectedKey:   "swarm-resource",
+			expectedValue: "string-value=more",
+		},
+		{
+			description:   "string option treats bool value as string",
+			setFlag:       "swarm-resource=true",
+			expectedKey:   "swarm-resource",
+			expectedValue: "true",
+		},
+		{
+			description:   "string option treats int value as string",
+			setFlag:       "swarm-resource=5",
+			expectedKey:   "swarm-resource",
+			expectedValue: "5",
+		},
+		{
+			description:   "[]string option returns single value",
+			setFlag:       "nvidia-container-cli.environment=string-value",
+			expectedKey:   "nvidia-container-cli.environment",
+			expectedValue: []string{"string-value"},
+		},
+		{
+			description:      "[]string option returns multiple values",
+			setFlag:          "nvidia-container-cli.environment=first,second",
+			setListSeparator: ",",
+			expectedKey:      "nvidia-container-cli.environment",
+			expectedValue:    []string{"first", "second"},
+		},
+		{
+			description:      "[]string option returns values with equals",
+			setFlag:          "nvidia-container-cli.environment=first=1,second=2",
+			setListSeparator: ",",
+			expectedKey:      "nvidia-container-cli.environment",
+			expectedValue:    []string{"first=1", "second=2"},
+		},
+		{
+			description:      "[]string option returns multiple values semi-colon",
+			setFlag:          "nvidia-container-cli.environment=first;second",
+			setListSeparator: ";",
+			expectedKey:      "nvidia-container-cli.environment",
+			expectedValue:    []string{"first", "second"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			if tc.setListSeparator == "" {
+				tc.setListSeparator = ","
+			}
+			k, v, err := setFlagToKeyValue(tc.setFlag, tc.setListSeparator)
+			require.ErrorIs(t, err, tc.expectedError)
+			require.EqualValues(t, tc.expectedKey, k)
+			require.EqualValues(t, tc.expectedValue, v)
+		})
+	}
+}

cmd/nvidia-ctk/config/create-default/create-default.go

@@ -0,0 +1,94 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package defaultsubcommand
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a default command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := flags.Options{}
+
+	// Create the 'default' command
+	c := cli.Command{
+		Name:    "default",
+		Aliases: []string{"create-default", "generate-default"},
+		Usage:   "Generate the default NVIDIA Container Toolkit configuration file",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *flags.Options) error {
+	return opts.Validate()
+}
+
+func (m command) run(c *cli.Context, opts *flags.Options) error {
+	cfgToml, err := config.New()
+	if err != nil {
+		return fmt.Errorf("unable to load or create config: %v", err)
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	if _, err = cfgToml.Save(output); err != nil {
+		return fmt.Errorf("failed to write output: %v", err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/config/flags/options.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package flags
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Options stores options for the config commands
+type Options struct {
+	Config  string
+	Output  string
+	InPlace bool
+}
+
+// Validate checks whether the options are valid.
+func (o Options) Validate() error {
+	if o.InPlace && o.Output != "" {
+		return fmt.Errorf("cannot specify both --in-place and --output")
+	}
+
+	return nil
+}
+
+// GetOutput returns the effective output
+func (o Options) GetOutput() string {
+	if o.InPlace {
+		return o.Config
+	}
+
+	return o.Output
+}
+
+// EnsureOutputFolder creates the output folder if it does not exist.
+// If the output folder is not specified (i.e. output to STDOUT), it is ignored.
+func (o Options) EnsureOutputFolder() error {
+	output := o.GetOutput()
+	if output == "" {
+		return nil
+	}
+	if dir := filepath.Dir(output); dir != "" {
+		return os.MkdirAll(dir, 0755)
+	}
+	return nil
+}
+
+// CreateOutput creates the writer for the output.
+func (o Options) CreateOutput() (io.WriteCloser, error) {
+	output := o.GetOutput()
+	if output == "" {
+		return nullCloser{os.Stdout}, nil
+	}
+
+	return os.Create(output)
+}
+
+// nullCloser is a writer that does nothing on Close.
+type nullCloser struct {
+	io.Writer
+}
+
+// Close is a no-op for a nullCloser.
+func (d nullCloser) Close() error {
+	return nil
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -1,229 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package symlinks
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	hostRoot      string
-	filenames     cli.StringSlice
-	links         cli.StringSlice
-	containerSpec string
-}
-
-// NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the '' command
-	c := cli.Command{
-		Name:  "create-symlinks",
-		Usage: "A hook to create symlinks in the container. This can be used to proces CSV mount specs",
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "host-root",
-			Usage:       "The root on the host filesystem to use to resolve symlinks",
-			Destination: &cfg.hostRoot,
-		},
-		&cli.StringSliceFlag{
-			Name:        "csv-filename",
-			Usage:       "Specify a (CSV) filename to process",
-			Destination: &cfg.filenames,
-		},
-		&cli.StringSliceFlag{
-			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as source:target",
-			Destination: &cfg.links,
-		},
-		&cli.StringFlag{
-			Name:        "container-spec",
-			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
-			Destination: &cfg.containerSpec,
-		},
-	}
-
-	return &c
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	s, err := oci.LoadContainerState(cfg.containerSpec)
-	if err != nil {
-		return fmt.Errorf("failed to load container state: %v", err)
-	}
-
-	containerRoot, err := s.GetContainerRoot()
-	if err != nil {
-		return fmt.Errorf("failed to determined container root: %v", err)
-	}
-
-	csvFiles := cfg.filenames.Value()
-
-	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
-
-	var candidates []string
-	for _, file := range csvFiles {
-		mountSpecs, err := csv.NewCSVFileParser(m.logger, file).Parse()
-		if err != nil {
-			m.logger.Debugf("Skipping CSV file %v: %v", file, err)
-			continue
-		}
-
-		for _, ms := range mountSpecs {
-			if ms.Type != csv.MountSpecSym {
-				continue
-			}
-			targets, err := chainLocator.Locate(ms.Path)
-			if err != nil {
-				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
-			}
-			candidates = append(candidates, targets...)
-		}
-	}
-
-	created := make(map[string]bool)
-	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
-	for _, candidate := range candidates {
-		targets, err := m.Locate(candidate)
-		if err != nil {
-			m.logger.Debugf("Skipping invalid link: %v", err)
-			continue
-		} else if len(targets) != 1 {
-			m.logger.Debugf("Unexepected number of targets: %v", targets)
-			continue
-		} else if targets[0] == candidate {
-			m.logger.Debugf("%v is not a symlink", candidate)
-			continue
-		}
-
-		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
-		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
-		}
-	}
-
-	links := cfg.links.Value()
-	for _, l := range links {
-		parts := strings.Split(l, ":")
-		if len(parts) != 2 {
-			m.logger.Warnf("Invalid link specification %v", l)
-			continue
-		}
-
-		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
-		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", parts, err)
-		}
-	}
-
-	return nil
-
-}
-
-func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
-	linkPath, err := changeRoot(hostRoot, containerRoot, link)
-	if err != nil {
-		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
-	}
-	if created[linkPath] {
-		m.logger.Debugf("Link %v already created", linkPath)
-		return nil
-	}
-
-	targetPath, err := changeRoot(hostRoot, "/", target)
-	if err != nil {
-		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
-	}
-
-	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)
-	err = os.MkdirAll(filepath.Dir(linkPath), 0755)
-	if err != nil {
-		return fmt.Errorf("failed to create directory: %v", err)
-	}
-	err = os.Symlink(target, linkPath)
-	if err != nil {
-		return fmt.Errorf("failed to create symlink: %v", err)
-	}
-
-	return nil
-}
-
-func changeRoot(current string, new string, path string) (string, error) {
-	if !filepath.IsAbs(path) {
-		return path, nil
-	}
-
-	relative := path
-	if current != "" {
-		r, err := filepath.Rel(current, path)
-		if err != nil {
-			return "", err
-		}
-		relative = r
-	}
-
-	return filepath.Join(new, relative), nil
-}
-
-// Locate returns the link target of the specified filename or an empty slice if the
-// specified filename is not a symlink.
-func (m command) Locate(filename string) ([]string, error) {
-	info, err := os.Lstat(filename)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get file info: %v", info)
-	}
-	if info.Mode()&os.ModeSymlink == 0 {
-		m.logger.Debugf("%v is not a symlink", filename)
-		return nil, nil
-	}
-
-	target, err := os.Readlink(filename)
-	if err != nil {
-		return nil, fmt.Errorf("error checking symlink: %v", err)
-	}
-
-	m.logger.Debugf("Resolved link: '%v' => '%v'", filename, target)
-
-	return []string{target}, nil
-}

cmd/nvidia-ctk/hook/hook.go

@@ -17,18 +17,18 @@
 package hook
 
 import (
-	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/commands"
-	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/sirupsen/logrus"
+
 	"github.com/urfave/cli/v2"
 )
 
 type hookCommand struct {
-	logger *logrus.Logger
+	logger logger.Interface
 }
 
 // NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
+func NewCommand(logger logger.Interface) *cli.Command {
 	c := hookCommand{
 		logger: logger,
 	}
@@ -43,10 +43,7 @@ func (m hookCommand) build() *cli.Command {
 		Usage: "A collection of hooks that may be injected into an OCI spec",
 	}
 
-	hook.Subcommands = []*cli.Command{
+	hook.Subcommands = commands.New(m.logger)
-		ldcache.NewCommand(m.logger),
-		symlinks.NewCommand(m.logger),
-	}
 
 	return &hook
 }

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -1,129 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package ldcache
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	folders       cli.StringSlice
-	containerSpec string
-}
-
-// NewCommand constructs an update-ldcache command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build the update-ldcache command
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the 'update-ldcache' command
-	c := cli.Command{
-		Name:  "update-ldcache",
-		Usage: "Update ldcache in a container by running ldconfig",
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:        "folder",
-			Usage:       "Specifiy a folder to add to /etc/ld.so.conf before updating the ld cache",
-			Destination: &cfg.folders,
-		},
-		&cli.StringFlag{
-			Name:        "container-spec",
-			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
-			Destination: &cfg.containerSpec,
-		},
-	}
-
-	return &c
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	s, err := oci.LoadContainerState(cfg.containerSpec)
-	if err != nil {
-		return fmt.Errorf("failed to load container state: %v", err)
-	}
-
-	containerRoot, err := s.GetContainerRoot()
-	if err != nil {
-		return fmt.Errorf("failed to determined container root: %v", err)
-	}
-
-	err = m.createConfig(containerRoot, cfg.folders.Value())
-	if err != nil {
-		return fmt.Errorf("failed to update ld.so.conf: %v", err)
-	}
-
-	args := []string{"/sbin/ldconfig"}
-	if containerRoot != "" {
-		args = append(args, "-r", containerRoot)
-	}
-
-	return syscall.Exec(args[0], args, nil)
-}
-
-// createConfig creates (or updates) /etc/ld.so.conf.d/nvcr-<RANDOM_STRING>.conf in the container
-// to include the required paths.
-func (m command) createConfig(root string, folders []string) error {
-	if len(folders) == 0 {
-		m.logger.Debugf("No folders to add to /etc/ld.so.conf")
-		return nil
-	}
-
-	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
-	if err != nil {
-		return fmt.Errorf("failed to create config file: %v", err)
-	}
-	defer configFile.Close()
-
-	m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name())
-
-	configured := make(map[string]bool)
-	for _, folder := range folders {
-		if configured[folder] {
-			continue
-		}
-		_, err = configFile.WriteString(fmt.Sprintf("%s\n", folder))
-		if err != nil {
-			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
-		}
-		configured[folder] = true
-	}
-
-	return nil
-}

cmd/nvidia-ctk/info/info.go

@@ -0,0 +1,48 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'info' command
+	info := cli.Command{
+		Name:  "info",
+		Usage: "Provide information about the system",
+	}
+
+	info.Subcommands = []*cli.Command{}
+
+	return &info
+}

cmd/nvidia-ctk/main.go

@@ -19,24 +19,33 @@ package main
 import (
 	"os"
 
+	"github.com/sirupsen/logrus"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	log "github.com/sirupsen/logrus"
+
 	cli "github.com/urfave/cli/v2"
 )
 
-var logger = log.New()
+// options defines the options that can be set for the CLI through config files,
-
-// config defines the options that can be set for the CLI through config files,
 // environment variables, or command line flags
-type config struct {
+type options struct {
 	// Debug indicates whether the CLI is started in "debug" mode
 	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
 }
 
 func main() {
-	// Create a config struct to hold the parsed environment variables or command line flags
+	logger := logrus.New()
-	config := config{}
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
 
 	// Create the top-level CLI
 	c := cli.NewApp()
@@ -52,16 +61,25 @@ func main() {
 			Name:        "debug",
 			Aliases:     []string{"d"},
 			Usage:       "Enable debug-level logging",
-			Destination: &config.Debug,
+			Destination: &opts.Debug,
 			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CTK_QUIET"},
+		},
 	}
 
 	// Set log-level for all subcommands
 	c.Before = func(c *cli.Context) error {
-		logLevel := log.InfoLevel
+		logLevel := logrus.InfoLevel
-		if config.Debug {
+		if opts.Debug {
-			logLevel = log.DebugLevel
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
 		}
 		logger.SetLevel(logLevel)
 		return nil
@@ -70,12 +88,17 @@ func main() {
 	// Define the subcommands
 	c.Commands = []*cli.Command{
 		hook.NewCommand(logger),
+		runtime.NewCommand(logger),
+		infoCLI.NewCommand(logger),
+		cdi.NewCommand(logger),
+		system.NewCommand(logger),
+		config.NewCommand(logger),
 	}
 
 	// Run the CLI
 	err := c.Run(os.Args)
 	if err != nil {
-		log.Errorf("%v", err)
+		logger.Errorf("%v", err)
-		log.Exit(1)
+		os.Exit(1)
 	}
 }

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -0,0 +1,355 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package configure
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	defaultRuntime = "docker"
+
+	// defaultNVIDIARuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	defaultNVIDIARuntimeName = "nvidia"
+	// defaultNVIDIARuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	defaultNVIDIARuntimeExecutable          = "nvidia-container-runtime"
+	defaultNVIDIARuntimeExpecutablePath     = "/usr/bin/nvidia-container-runtime"
+	defaultNVIDIARuntimeHookExpecutablePath = "/usr/bin/nvidia-container-runtime-hook"
+
+	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
+	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
+	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
+
+	defaultConfigSource = configSourceFile
+	configSourceCommand = "command"
+	configSourceFile    = "file"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a configure command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line config
+type config struct {
+	dryRun         bool
+	runtime        string
+	configFilePath string
+	configSource   string
+	mode           string
+	hookFilePath   string
+
+	runtimeConfigOverrideJSON string
+
+	nvidiaRuntime struct {
+		name         string
+		path         string
+		hookPath     string
+		setAsDefault bool
+	}
+
+	// cdi-specific options
+	cdi struct {
+		enabled bool
+	}
+}
+
+func (m command) build() *cli.Command {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the 'configure' command
+	configure := cli.Command{
+		Name:  "configure",
+		Usage: "Add a runtime to the specified container engine",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &config)
+		},
+		Action: func(c *cli.Context) error {
+			return m.configureWrapper(c, &config)
+		},
+	}
+
+	configure.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "update the runtime configuration as required but don't write changes to disk",
+			Destination: &config.dryRun,
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Usage:       "the target runtime engine; one of [containerd, crio, docker]",
+			Value:       defaultRuntime,
+			Destination: &config.runtime,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to the config file for the target runtime",
+			Destination: &config.configFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the config mode for runtimes that support multiple configuration mechanisms",
+			Destination: &config.mode,
+		},
+		&cli.StringFlag{
+			Name:        "config-source",
+			Usage:       "the source to retrieve the container runtime configuration; one of [command, file]\"",
+			Destination: &config.configSource,
+			Value:       defaultConfigSource,
+		},
+		&cli.StringFlag{
+			Name:        "oci-hook-path",
+			Usage:       "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
+			Destination: &config.hookFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-name",
+			Usage:       "specify the name of the NVIDIA runtime that will be added",
+			Value:       defaultNVIDIARuntimeName,
+			Destination: &config.nvidiaRuntime.name,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-path",
+			Aliases:     []string{"runtime-path"},
+			Usage:       "specify the path to the NVIDIA runtime executable",
+			Value:       defaultNVIDIARuntimeExecutable,
+			Destination: &config.nvidiaRuntime.path,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-hook-path",
+			Usage:       "specify the path to the NVIDIA Container Runtime hook executable",
+			Value:       defaultNVIDIARuntimeHookExpecutablePath,
+			Destination: &config.nvidiaRuntime.hookPath,
+		},
+		&cli.BoolFlag{
+			Name:        "nvidia-set-as-default",
+			Aliases:     []string{"set-as-default"},
+			Usage:       "set the NVIDIA runtime as the default runtime",
+			Destination: &config.nvidiaRuntime.setAsDefault,
+		},
+		&cli.BoolFlag{
+			Name:        "cdi.enabled",
+			Aliases:     []string{"cdi.enable", "enable-cdi"},
+			Usage:       "Enable CDI in the configured runtime",
+			Destination: &config.cdi.enabled,
+		},
+	}
+
+	return &configure
+}
+
+func (m command) validateFlags(c *cli.Context, config *config) error {
+	if config.mode == "oci-hook" {
+		if !filepath.IsAbs(config.nvidiaRuntime.hookPath) {
+			return fmt.Errorf("the NVIDIA runtime hook path %q is not an absolute path", config.nvidiaRuntime.hookPath)
+		}
+		return nil
+	}
+	if config.mode != "" && config.mode != "config-file" {
+		m.logger.Warningf("Ignoring unsupported config mode for %v: %q", config.runtime, config.mode)
+	}
+	config.mode = "config-file"
+
+	switch config.runtime {
+	case "containerd", "crio", "docker":
+		break
+	default:
+		return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+
+	switch config.runtime {
+	case "containerd", "crio":
+		if config.nvidiaRuntime.path == defaultNVIDIARuntimeExecutable {
+			config.nvidiaRuntime.path = defaultNVIDIARuntimeExpecutablePath
+		}
+		if !filepath.IsAbs(config.nvidiaRuntime.path) {
+			return fmt.Errorf("the NVIDIA runtime path %q is not an absolute path", config.nvidiaRuntime.path)
+		}
+	}
+
+	if config.runtime != "containerd" && config.runtime != "docker" {
+		if config.cdi.enabled {
+			m.logger.Warningf("Ignoring cdi.enabled flag for %v", config.runtime)
+		}
+		config.cdi.enabled = false
+	}
+
+	if config.runtimeConfigOverrideJSON != "" && config.runtime != "containerd" {
+		m.logger.Warningf("Ignoring runtime-config-override flag for %v", config.runtime)
+		config.runtimeConfigOverrideJSON = ""
+	}
+
+	switch config.configSource {
+	case configSourceCommand:
+		if config.runtime == "docker" {
+			m.logger.Warningf("A %v Config Source is not supported for %v; using %v", config.configSource, config.runtime, configSourceFile)
+			config.configSource = configSourceFile
+		}
+	case configSourceFile:
+		break
+	default:
+		return fmt.Errorf("unrecognized Config Source: %v", config.configSource)
+	}
+
+	if config.configFilePath == "" {
+		switch config.runtime {
+		case "containerd":
+			config.configFilePath = defaultContainerdConfigFilePath
+		case "crio":
+			config.configFilePath = defaultCrioConfigFilePath
+		case "docker":
+			config.configFilePath = defaultDockerConfigFilePath
+		}
+	}
+
+	return nil
+}
+
+// configureWrapper updates the specified container engine config to enable the NVIDIA runtime
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.mode {
+	case "oci-hook":
+		return m.configureOCIHook(c, config)
+	case "config-file":
+		return m.configureConfigFile(c, config)
+	}
+	return fmt.Errorf("unsupported config-mode: %v", config.mode)
+}
+
+// configureConfigFile updates the specified container engine config file to enable the NVIDIA runtime.
+func (m command) configureConfigFile(c *cli.Context, config *config) error {
+	configSource, err := config.resolveConfigSource()
+	if err != nil {
+		return err
+	}
+
+	var cfg engine.Interface
+	switch config.runtime {
+	case "containerd":
+		cfg, err = containerd.New(
+			containerd.WithLogger(m.logger),
+			containerd.WithPath(config.configFilePath),
+			containerd.WithConfigSource(configSource),
+		)
+	case "crio":
+		cfg, err = crio.New(
+			crio.WithLogger(m.logger),
+			crio.WithPath(config.configFilePath),
+			crio.WithConfigSource(configSource),
+		)
+	case "docker":
+		cfg, err = docker.New(
+			docker.WithLogger(m.logger),
+			docker.WithPath(config.configFilePath),
+		)
+	default:
+		err = fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+	}
+	if err != nil || cfg == nil {
+		return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
+	}
+
+	err = cfg.AddRuntime(
+		config.nvidiaRuntime.name,
+		config.nvidiaRuntime.path,
+		config.nvidiaRuntime.setAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.cdi.enabled {
+		cfg.EnableCDI()
+	}
+
+	outputPath := config.getOutputConfigPath()
+	n, err := cfg.Save(outputPath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	if outputPath != "" {
+		if n == 0 {
+			m.logger.Infof("Removed empty config from %v", outputPath)
+		} else {
+			m.logger.Infof("Wrote updated config to %v", outputPath)
+		}
+		m.logger.Infof("It is recommended that %v daemon be restarted.", config.runtime)
+	}
+
+	return nil
+}
+
+// resolveConfigSource returns the default config source or the user provided config source
+func (c *config) resolveConfigSource() (toml.Loader, error) {
+	switch c.configSource {
+	case configSourceCommand:
+		return c.getCommandConfigSource(), nil
+	case configSourceFile:
+		return toml.FromFile(c.configFilePath), nil
+	default:
+		return nil, fmt.Errorf("unrecognized config source: %s", c.configSource)
+	}
+}
+
+// getConfigSourceCommand returns the default cli command to fetch the current runtime config
+func (c *config) getCommandConfigSource() toml.Loader {
+	switch c.runtime {
+	case "containerd":
+		return containerd.CommandLineSource("")
+	case "crio":
+		return crio.CommandLineSource("")
+	}
+	return toml.Empty
+}
+
+// getOutputConfigPath returns the configured config path or "" if dry-run is enabled
+func (c *config) getOutputConfigPath() string {
+	if c.dryRun {
+		return ""
+	}
+	return c.configFilePath
+}
+
+// configureOCIHook creates and configures the OCI hook for the NVIDIA runtime
+func (m *command) configureOCIHook(c *cli.Context, config *config) error {
+	err := ocihook.CreateHook(config.hookFilePath, config.nvidiaRuntime.hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating OCI hook: %v", err)
+	}
+	return nil
+}

cmd/nvidia-ctk/runtime/runtime.go

@@ -0,0 +1,50 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type runtimeCommand struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := runtimeCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m runtimeCommand) build() *cli.Command {
+	// Create the 'runtime' command
+	runtime := cli.Command{
+		Name:  "runtime",
+		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	runtime.Subcommands = []*cli.Command{
+		configure.NewCommand(m.logger),
+	}
+
+	return &runtime
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -0,0 +1,188 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/go-nvlib/pkg/nvpci"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+)
+
+type allPossible struct {
+	logger       logger.Interface
+	devRoot      string
+	deviceMajors devices.Devices
+	migCaps      nvcaps.MigCaps
+}
+
+// newAllPossible returns a new allPossible device node lister.
+// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func newAllPossible(logger logger.Interface, devRoot string) (nodeLister, error) {
+	deviceMajors, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed reading device majors: %v", err)
+	}
+
+	var requiredMajors []devices.Name
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
+	}
+	if migCaps == nil {
+		migCaps = make(nvcaps.MigCaps)
+	} else {
+		requiredMajors = append(requiredMajors, devices.NVIDIACaps)
+	}
+
+	requiredMajors = append(requiredMajors, devices.NVIDIAGPU, devices.NVIDIAUVM)
+	for _, name := range requiredMajors {
+		if !deviceMajors.Exists(name) {
+			return nil, fmt.Errorf("missing required device major %s", name)
+		}
+	}
+
+	l := allPossible{
+		logger:       logger,
+		devRoot:      devRoot,
+		deviceMajors: deviceMajors,
+		migCaps:      migCaps,
+	}
+
+	return l, nil
+}
+
+// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func (m allPossible) DeviceNodes() ([]deviceNode, error) {
+	gpus, err := nvpci.New(
+		nvpci.WithPCIDevicesRoot(filepath.Join(m.devRoot, nvpci.PCIDevicesRoot)),
+		nvpci.WithLogger(m.logger),
+	).GetGPUs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GPU information: %v", err)
+	}
+
+	count := len(gpus)
+	if count == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
+		return nil, nil
+	}
+
+	deviceNodes, err := m.getControlDeviceNodes()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
+	}
+
+	for gpu := 0; gpu < count; gpu++ {
+		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
+		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
+	}
+
+	return deviceNodes, nil
+}
+
+// getControlDeviceNodes generates a list of control devices
+func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
+	var deviceNodes []deviceNode
+
+	// Define the control devices for standard GPUs.
+	controlDevices := []deviceNode{
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
+	}
+
+	deviceNodes = append(deviceNodes, controlDevices...)
+
+	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
+		migControlMinor, exist := m.migCaps[migControlDevice]
+		if !exist {
+			continue
+		}
+
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			migControlMinor.DevicePath(),
+			int(migControlMinor),
+		)
+
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes, nil
+}
+
+// getGPUDeviceNodes generates a list of device nodes for a given GPU.
+func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
+	d := m.newDeviceNode(
+		devices.NVIDIAGPU,
+		fmt.Sprintf("/dev/nvidia%d", gpu),
+		gpu,
+	)
+
+	return []deviceNode{d}
+}
+
+// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
+func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
+	var selectedCapMinors []nvcaps.MigMinor
+	for gi := 0; ; gi++ {
+		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+		giMinor, exist := m.migCaps[giCap]
+		if !exist {
+			break
+		}
+		selectedCapMinors = append(selectedCapMinors, giMinor)
+		for ci := 0; ; ci++ {
+			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+			ciMinor, exist := m.migCaps[ciCap]
+			if !exist {
+				break
+			}
+			selectedCapMinors = append(selectedCapMinors, ciMinor)
+		}
+	}
+
+	var deviceNodes []deviceNode
+	for _, capMinor := range selectedCapMinors {
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			capMinor.DevicePath(),
+			int(capMinor),
+		)
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes
+}
+
+// newDeviceNode creates a new device node with the specified path and major/minor numbers.
+// The path is adjusted for the specified driver root.
+func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
+	major, _ := m.deviceMajors.Get(deviceName)
+
+	return deviceNode{
+		path:  filepath.Join(m.devRoot, path),
+		major: uint32(major),
+		minor: uint32(minor),
+	}
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -0,0 +1,340 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+)
+
+const (
+	defaultDevCharPath = "/dev/char"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	devCharPath       string
+	driverRoot        string
+	dryRun            bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'create-dev-char-symlinks' command
+	c := cli.Command{
+		Name:  "create-dev-char-symlinks",
+		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "dev-char-path",
+			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
+			Value:       defaultDevCharPath,
+			Destination: &cfg.devCharPath,
+			EnvVars:     []string{"DEV_CHAR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
+			Value:       "/",
+			Destination: &cfg.driverRoot,
+			EnvVars:     []string{"NVIDIA_DRIVER_ROOT", "DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-all",
+			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
+			Destination: &cfg.createAll,
+			EnvVars:     []string{"CREATE_ALL"},
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "Load the NVIDIA kernel modules before creating symlinks. This is only applicable when --create-all is set.",
+			Destination: &cfg.loadKernelModules,
+			EnvVars:     []string{"LOAD_KERNEL_MODULES"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-device-nodes",
+			Usage:       "Create the NVIDIA control device nodes in the driver root if they do not exist. This is only applicable when --create-all is set",
+			Destination: &cfg.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "If set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &cfg.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	if cfg.createAll {
+		return fmt.Errorf("create-all and watch are mutually exclusive")
+	}
+
+	if cfg.loadKernelModules && !cfg.createAll {
+		m.logger.Warning("load-kernel-modules is only applicable when create-all is set; ignoring")
+		cfg.loadKernelModules = false
+	}
+
+	if cfg.createDeviceNodes && !cfg.createAll {
+		m.logger.Warning("create-device-nodes is only applicable when create-all is set; ignoring")
+		cfg.createDeviceNodes = false
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	l, err := NewSymlinkCreator(
+		WithLogger(m.logger),
+		WithDevCharPath(cfg.devCharPath),
+		WithDriverRoot(cfg.driverRoot),
+		WithDryRun(cfg.dryRun),
+		WithCreateAll(cfg.createAll),
+		WithLoadKernelModules(cfg.loadKernelModules),
+		WithCreateDeviceNodes(cfg.createDeviceNodes),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink creator: %v", err)
+	}
+
+	err = l.CreateLinks()
+	if err != nil {
+		return fmt.Errorf("failed to create links: %v", err)
+	}
+	return nil
+}
+
+type linkCreator struct {
+	logger            logger.Interface
+	lister            nodeLister
+	driverRoot        string
+	devRoot           string
+	devCharPath       string
+	dryRun            bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
+}
+
+// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
+type Creator interface {
+	CreateLinks() error
+}
+
+// Option is a functional option for configuring the linkCreator.
+type Option func(*linkCreator)
+
+// NewSymlinkCreator creates a new linkCreator.
+func NewSymlinkCreator(opts ...Option) (Creator, error) {
+	c := linkCreator{}
+	for _, opt := range opts {
+		opt(&c)
+	}
+	if c.logger == nil {
+		c.logger = logger.New()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+	if c.devRoot == "" {
+		c.devRoot = "/"
+	}
+	if c.devCharPath == "" {
+		c.devCharPath = defaultDevCharPath
+	}
+
+	if err := c.setup(); err != nil {
+		return nil, err
+	}
+
+	if c.createAll {
+		lister, err := newAllPossible(c.logger, c.devRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
+		}
+		c.lister = lister
+	} else {
+		c.lister = existing{c.logger, c.devRoot}
+	}
+	return c, nil
+}
+
+func (m linkCreator) setup() error {
+	if !m.loadKernelModules && !m.createDeviceNodes {
+		return nil
+	}
+
+	if m.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(m.dryRun),
+			nvmodules.WithRoot(m.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if m.createDeviceNodes {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(m.dryRun),
+			nvdevices.WithDevRoot(m.devRoot),
+		)
+		if err != nil {
+			return err
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA device nodes: %v", err)
+		}
+	}
+	return nil
+}
+
+// WithDriverRoot sets the driver root path.
+// This is the path in which kernel modules must be loaded.
+func WithDriverRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.driverRoot = root
+	}
+}
+
+// WithDevRoot sets the root path for the /dev directory.
+func WithDevRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.devRoot = root
+	}
+}
+
+// WithDevCharPath sets the path at which the symlinks will be created.
+func WithDevCharPath(path string) Option {
+	return func(c *linkCreator) {
+		c.devCharPath = path
+	}
+}
+
+// WithDryRun sets the dry run flag.
+func WithDryRun(dryRun bool) Option {
+	return func(c *linkCreator) {
+		c.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger.
+func WithLogger(logger logger.Interface) Option {
+	return func(c *linkCreator) {
+		c.logger = logger
+	}
+}
+
+// WithCreateAll sets the createAll flag for the linkCreator.
+func WithCreateAll(createAll bool) Option {
+	return func(lc *linkCreator) {
+		lc.createAll = createAll
+	}
+}
+
+// WithLoadKernelModules sets the loadKernelModules flag for the linkCreator.
+func WithLoadKernelModules(loadKernelModules bool) Option {
+	return func(lc *linkCreator) {
+		lc.loadKernelModules = loadKernelModules
+	}
+}
+
+// WithCreateDeviceNodes sets the createDeviceNodes flag for the linkCreator.
+func WithCreateDeviceNodes(createDeviceNodes bool) Option {
+	return func(lc *linkCreator) {
+		lc.createDeviceNodes = createDeviceNodes
+	}
+}
+
+// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
+func (m linkCreator) CreateLinks() error {
+	deviceNodes, err := m.lister.DeviceNodes()
+	if err != nil {
+		return fmt.Errorf("failed to get device nodes: %v", err)
+	}
+
+	if len(deviceNodes) != 0 && !m.dryRun {
+		err := os.MkdirAll(m.devCharPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
+		}
+	}
+
+	for _, deviceNode := range deviceNodes {
+		target := deviceNode.path
+		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
+
+		m.logger.Infof("Creating link %s => %s", linkPath, target)
+		if m.dryRun {
+			continue
+		}
+
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warningf("Could not create symlink: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type deviceNode struct {
+	path  string
+	major uint32
+	minor uint32
+}
+
+func (d deviceNode) devCharName() string {
+	return fmt.Sprintf("%d:%d", d.major, d.minor)
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -0,0 +1,89 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+type nodeLister interface {
+	DeviceNodes() ([]deviceNode, error)
+}
+
+type existing struct {
+	logger  logger.Interface
+	devRoot string
+}
+
+// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
+// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
+func (m existing) DeviceNodes() ([]deviceNode, error) {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(m.devRoot),
+		lookup.WithOptional(true),
+	)
+
+	devices, err := locator.Locate("/dev/nvidia*")
+	if err != nil {
+		m.logger.Warningf("Error while locating device: %v", err)
+	}
+
+	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
+	if err != nil {
+		m.logger.Warningf("Error while locating caps device: %v", err)
+	}
+
+	if len(devices) == 0 && len(capDevices) == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
+		return nil, nil
+	}
+
+	var deviceNodes []deviceNode
+	for _, d := range append(devices, capDevices...) {
+		if m.nodeIsBlocked(d) {
+			continue
+		}
+		var stat unix.Stat_t
+		err := unix.Stat(d, &stat)
+		if err != nil {
+			m.logger.Warningf("Could not stat device: %v", err)
+			continue
+		}
+		deviceNodes = append(deviceNodes, newDeviceNode(d, stat))
+	}
+
+	return deviceNodes, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m existing) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing_linux.go

@@ -0,0 +1,28 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import "golang.org/x/sys/unix"
+
+func newDeviceNode(d string, stat unix.Stat_t) deviceNode {
+	deviceNode := deviceNode{
+		path:  d,
+		major: unix.Major(stat.Rdev),
+		minor: unix.Minor(stat.Rdev),
+	}
+	return deviceNode
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing_other.go

@@ -0,0 +1,30 @@
+//go:build !linux
+
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import "golang.org/x/sys/unix"
+
+func newDeviceNode(d string, stat unix.Stat_t) deviceNode {
+	deviceNode := deviceNode{
+		path:  d,
+		major: unix.Major(uint64(stat.Rdev)),
+		minor: unix.Minor(uint64(stat.Rdev)),
+	}
+	return deviceNode
+}

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -0,0 +1,142 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package createdevicenodes
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	root    string
+	devRoot string
+
+	dryRun bool
+
+	control bool
+
+	loadKernelModules bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "create-device-nodes",
+		Usage: "A utility to create NVIDIA device nodes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name: "root",
+			// TODO: Remove this alias
+			Aliases: []string{"driver-root"},
+			Usage: "the path to to the root to use to load the kernel modules. This root must be a chrootable path. " +
+				"If device nodes to be created these will be created at `ROOT`/dev unless an alternative path is specified",
+			Value:       "/",
+			Destination: &opts.root,
+			// TODO: Remove the NVIDIA_DRIVER_ROOT and DRIVER_ROOT envvars.
+			EnvVars: []string{"ROOT", "NVIDIA_DRIVER_ROOT", "DRIVER_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "dev-root",
+			Usage:       "specify the root where `/dev` is located. If this is not specified, the root is assumed.",
+			Destination: &opts.devRoot,
+			EnvVars:     []string{"NVIDIA_DEV_ROOT", "DEV_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "control-devices",
+			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
+			Destination: &opts.control,
+		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "load the NVIDIA Kernel Modules before creating devices nodes",
+			Destination: &opts.loadKernelModules,
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "if set, the command will not perform any operations",
+			Value:       false,
+			Destination: &opts.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, opts *options) error {
+	if opts.devRoot == "" && opts.root != "" {
+		m.logger.Infof("Using dev-root %q", opts.root)
+		opts.devRoot = opts.root
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	if opts.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(opts.dryRun),
+			nvmodules.WithRoot(opts.root),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if opts.control {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(opts.dryRun),
+			nvdevices.WithDevRoot(opts.devRoot),
+		)
+		if err != nil {
+			return err
+		}
+		m.logger.Infof("Creating control device nodes at %s", opts.devRoot)
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA control device nodes: %v", err)
+		}
+	}
+	return nil
+}

cmd/nvidia-ctk/system/system.go

@@ -0,0 +1,52 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	"github.com/urfave/cli/v2"
+
+	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m command) build() *cli.Command {
+	// Create the 'system' command
+	system := cli.Command{
+		Name:  "system",
+		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	system.Subcommands = []*cli.Command{
+		devchar.NewCommand(m.logger),
+		devicenodes.NewCommand(m.logger),
+	}
+
+	return &system
+}

config/config.toml.amazonlinux

@@ -1,32 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-#accept-nvidia-visible-devices-envvar-when-unprivileged = true
-#accept-nvidia-visible-devices-as-volume-mounts = false
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"