Merge pull request #1159 from elezar/fix-gitlab-references

[no-relnote] Replace GitLab with GitHub URLs

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,241 +12,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker:stable
+  image: docker
   services:
-    - name: docker:stable-dind
+    - name: docker:dind
       command: ["--experimental"]
 
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
-  BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
-  - image
-  - lint
-  - go-checks
-  - go-build
-  - unit-tests
-  - package-build
-  - image-build
-  - test
+  - pull
   - scan
   - release
+  - sign
 
-# Define the distribution targets
-.dist-amazonlinux2:
-  variables:
-    DIST: amazonlinux2
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
 
-.dist-centos7:
-  variables:
-    DIST: centos7
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-centos8:
-  variables:
-    DIST: centos8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-debian10:
-  variables:
-    DIST: debian10
-
-.dist-debian9:
-  variables:
-    DIST: debian9
-
-.dist-opensuse-leap15.1:
-  variables:
-    DIST: opensuse-leap15.1
-
-.dist-ubi8:
-  variables:
-    DIST: ubi8
-    CVE_UPDATES: "cyrus-sasl-lib"
-
-.dist-ubuntu16.04:
-  variables:
-    DIST: ubuntu16.04
-
-.dist-ubuntu18.04:
-  variables:
-    DIST: ubuntu18.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
-
-.dist-ubuntu20.04:
-  variables:
-    DIST: ubuntu20.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
-
-.dist-packaging:
-  variables:
-    DIST: packaging
-
-# Define architecture targets
-.arch-aarch64:
-  variables:
-    ARCH: aarch64
-
-.arch-amd64:
-  variables:
-    ARCH: amd64
-
-.arch-arm64:
-  variables:
-    ARCH: arm64
-
-.arch-ppc64le:
-  variables:
-    ARCH: ppc64le
-
-.arch-x86_64:
-  variables:
-    ARCH: x86_64
-
-# Define the platform targets
-.platform-amd64:
-  variables:
-    PLATFORM: linux/amd64
-
-.platform-arm64:
-  variables:
-    PLATFORM: linux/arm64
-
-# Define test helpers
-.integration:
-  stage: test
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - apk add --no-cache make bash jq
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
-  script:
-    - make -f build/container/Makefile test-${DIST}
-
-# Define the test targets
-test-packaging:
-  extends:
-    - .integration
-    - .dist-packaging
-  needs:
-    - image-packaging
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
 
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.3.10
+    - export REGCTL_VERSION=v0.4.5
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
     - chmod a+x bin/regctl
     - export PATH=$(pwd)/bin:${PATH}
-
-# .release forms the base of the deployment jobs which push images to the CI registry.
-# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
-# target os.
-.release:
-  stage: release
-  variables:
-    # Define the source image for the release
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    # OUT_IMAGE_VERSION is overridden for external releases
-    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - !reference [.regctl-setup, before_script]
-
-    # We ensure that the OUT_IMAGE_VERSION is set
-    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
-
-    # In the case where we are deploying a different version to the CI_COMMIT_SHA, we
-    # need to tag the image.
-    # Note: a leading 'v' is stripped from the version if present
-    - apk add --no-cache make bash
-  script:
-    # Log in to the "output" registry, tag the image and push the image
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"'
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"'
-
-    # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
-    # Target
-    - make -f build/container/Makefile push-${DIST}
-
-# Define a staging release step that pushes an image to an internal "staging" repository
-# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
-# outside of the release process.
-.release:staging:
-  extends:
-    - .release
-  variables:
-    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    OUT_REGISTRY: "${CI_REGISTRY}"
-    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
-
-# Define an external release step that pushes an image to an external repository.
-# This includes a devlopment image off master.
-.release:external:
-  extends:
-    - .release
-  rules:
-    - if: $CI_COMMIT_TAG
-      variables:
-        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
-    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
-      variables:
-        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
-
-# Define the release jobs
-release:staging-centos7:
-  extends:
-    - .release:staging
-    - .dist-centos7
-  needs:
-    - image-centos7
-
-release:staging-centos8:
-  extends:
-    - .release:staging
-    - .dist-centos8
-  needs:
-    - image-centos8
-
-release:staging-ubi8:
-  extends:
-    - .release:staging
-    - .dist-ubi8
-  needs:
-    - image-ubi8
-
-release:staging-ubuntu18.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu18.04
-  needs:
-    - test-toolkit-ubuntu18.04
-    - test-containerd-ubuntu18.04
-    - test-crio-ubuntu18.04
-    - test-docker-ubuntu18.04
-
-release:staging-ubuntu20.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu20.04
-  needs:
-    - test-toolkit-ubuntu20.04
-    - test-containerd-ubuntu20.04
-    - test-crio-ubuntu20.04
-    - test-docker-ubuntu20.04
-
-release:staging-packaging:
-  extends:
-    - .release:staging
-    - .dist-packaging
-  needs:
-    - test-packaging

.github/copy-pr-bot.yaml

@@ -0,0 +1,3 @@
+# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/#configuration
+
+enabled: true

.github/dependabot.yml

@@ -0,0 +1,120 @@
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+# main branch
+  - package-ecosystem: "gomod"
+    target-branch: main
+    directories:
+    - "/"
+    - "deployments/devel"
+    - "tests"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+    groups:
+      k8sio:
+        patterns:
+        - k8s.io/*
+        exclude-patterns:
+        - k8s.io/klog/*
+
+  - package-ecosystem: "docker"
+    target-branch: main
+    directories:
+    # CUDA image
+    - "/deployments/container"
+    # Golang version
+    - "/deployments/devel"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+
+  - package-ecosystem: "github-actions"
+    target-branch: main
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+
+  # Allow dependabot to update the libnvidia-container submodule.
+  - package-ecosystem: "gitsubmodule"
+    target-branch: main
+    directory: "/"
+    allow:
+    - dependency-name: "third_party/libnvidia-container"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies
+    - libnvidia-container
+
+# The release branch(es):
+  - package-ecosystem: "gomod"
+    target-branch: release-1.17
+    directories:
+    - "/"
+    # We don't update development or test dependencies on release branches
+    # - "deployments/devel"
+    # - "tests"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    labels:
+    - dependencies
+    - maintenance
+    ignore:
+    # For release branches we only consider patch updates.
+    - dependency-name: "*"
+      update-types:
+      - version-update:semver-major
+      - version-update:semver-minor
+    groups:
+      k8sio:
+        patterns:
+        - k8s.io/*
+        exclude-patterns:
+        - k8s.io/klog/*
+
+  - package-ecosystem: "docker"
+    target-branch: release-1.17
+    directories:
+    # CUDA image
+    - "/deployments/container"
+    # Golang version
+    - "/deployments/devel"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    ignore:
+    # For release branches we only apply patch updates to the golang version.
+    - dependency-name: "*golang*"
+      update-types:
+      - version-update:semver-major
+      - version-update:semver-minor
+    labels:
+    - dependencies
+    - maintenance
+
+  - package-ecosystem: "github-actions"
+    target-branch: release-1.17
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    labels:
+    - dependencies
+    - maintenance
+
+  # Github actions need to be gh-pages branches.
+  - package-ecosystem: "github-actions"
+    target-branch: gh-pages
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+    - dependencies

.github/workflows/ci.yaml

@@ -0,0 +1,53 @@
+# Copyright 2025 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI Pipeline
+
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+      - main
+      - release-*
+
+jobs:
+  code-scanning:
+    uses: ./.github/workflows/code_scanning.yaml
+
+  variables:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Generate Commit Short SHA
+        id: version
+        run: echo "version=$(echo $GITHUB_SHA | cut -c1-8)" >> "$GITHUB_OUTPUT"
+
+  golang:
+    uses: ./.github/workflows/golang.yaml
+
+  image:
+    uses: ./.github/workflows/image.yaml
+    needs: [variables, golang, code-scanning]
+    secrets: inherit
+    with:
+      version: ${{ needs.variables.outputs.version }}
+      build_multi_arch_images: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release-') }}
+
+  e2e-test:
+    needs: [image, variables]
+    secrets: inherit
+    uses: ./.github/workflows/e2e.yaml
+    with:
+      version: ${{ needs.variables.outputs.version }}

.github/workflows/code_scanning.yaml

@@ -0,0 +1,49 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "CodeQL"
+
+on:
+  workflow_call: {}
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - main
+      - release-*
+
+jobs:
+  analyze:
+    name: Analyze Go code with CodeQL
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      security-events: write
+      packages: read
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: go
+        build-mode: manual
+    - shell: bash
+      run: |
+        make build
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:go"

.github/workflows/e2e.yaml

@@ -0,0 +1,103 @@
+# Copyright 2025 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: End-to-end Tests
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      AWS_SSH_KEY:
+        required: true
+      E2E_SSH_USER:
+        required: true
+      SLACK_BOT_TOKEN:
+        required: true
+      SLACK_CHANNEL_ID:
+        required: true
+
+jobs:
+  e2e-tests:
+    runs-on: linux-amd64-cpu4
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Calculate build vars
+        id: vars
+        run: |
+          echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
+          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV
+          GOLANG_VERSION=$(./hack/golang-version.sh)
+          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION := }" >> $GITHUB_ENV
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+
+      - name: Set up Holodeck
+        uses: NVIDIA/holodeck@v0.2.12
+        with:
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
+          holodeck_config: "tests/e2e/infra/aws.yaml"
+
+      - name: Get public dns name
+        id: holodeck_public_dns_name
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
+
+      - name: Run e2e tests
+        env:
+          E2E_INSTALL_CTK: "true"
+          E2E_IMAGE_NAME: ghcr.io/nvidia/container-toolkit
+          E2E_IMAGE_TAG: ${{ inputs.version }}
+          E2E_SSH_USER: ${{ secrets.E2E_SSH_USER }}
+          E2E_SSH_HOST: ${{ steps.holodeck_public_dns_name.outputs.result }}
+        run: |
+          e2e_ssh_key=$(mktemp)
+          echo "${{ secrets.AWS_SSH_KEY }}" > "$e2e_ssh_key"
+          chmod 600 "$e2e_ssh_key"
+          export E2E_SSH_KEY="$e2e_ssh_key"
+
+          make -f tests/e2e/Makefile test
+
+      - name: Archive Ginkgo logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: ginkgo-logs
+          path: ginkgo.json
+          retention-days: 15
+      - name: Send Slack alert notification
+        if: ${{ failure() }}
+        uses: slackapi/slack-github-action@v2.1.0
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          payload: |
+            channel: ${{ secrets.SLACK_CHANNEL_ID }}
+            text: |
+              :x: On repository ${{ github.repository }}, the Workflow *${{ github.workflow }}* has failed.
+
+              Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/golang.yaml

@@ -0,0 +1,102 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Golang
+
+on:
+  workflow_call: {}
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - main
+      - release-*
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout code
+
+    - name: Get Golang version
+      id: vars
+      run: |
+        GOLANG_VERSION=$(./hack/golang-version.sh)
+        echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION := }" >> $GITHUB_ENV
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GOLANG_VERSION }}
+
+    - name: Lint
+      uses: golangci/golangci-lint-action@v8
+      with:
+        version: latest
+        args: -v --timeout 5m
+        skip-cache: true
+
+    - name: Check golang modules
+      run: | 
+        make check-vendor
+        make -C deployments/devel check-modules
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Golang version
+        id: vars
+        run: |
+          GOLANG_VERSION=$(./hack/golang-version.sh)
+          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION := }" >> $GITHUB_ENV
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+      
+      - name: Run unit tests and generate coverage report
+        run: make coverage
+
+      - name: Upload to Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: coverage.out
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Golang version
+        id: vars
+        run: |
+          GOLANG_VERSION=$(./hack/golang-version.sh)
+          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> $GITHUB_ENV
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+
+      - run: make build

.github/workflows/image.yaml

@@ -0,0 +1,120 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run this workflow on pull requests
+name: image
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      build_multi_arch_images:
+        required: true
+        type: string
+
+jobs:
+  packages:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - ubuntu18.04-arm64
+          - ubuntu18.04-amd64
+          - ubuntu18.04-ppc64le
+          - centos7-aarch64
+          - centos7-x86_64
+          - centos8-ppc64le
+        ispr:
+          - ${{ github.ref_name != 'main' && !startsWith( github.ref_name, 'release-' ) }}
+        exclude:
+          - ispr: true
+            target: ubuntu18.04-arm64
+          - ispr: true
+            target: ubuntu18.04-ppc64le
+          - ispr: true
+            target: centos7-aarch64
+          - ispr: true
+            target: centos8-ppc64le
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out code
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:master
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: build ${{ matrix.target }} packages
+        run: |
+          sudo apt-get install -y coreutils build-essential sed git bash make
+          echo "Building packages"
+          ./scripts/build-packages.sh ${{ matrix.target }}
+
+      - name: 'Upload Artifacts'
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: toolkit-container-${{ matrix.target }}-${{ github.run_id }}
+          path: ${{ github.workspace }}/dist/*
+
+  image:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - application
+          - packaging
+    needs: packages
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out code
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:master
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Get built packages
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ github.workspace }}/dist/
+          pattern: toolkit-container-*-${{ github.run_id }}
+          merge-multiple: true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image
+        env:
+          IMAGE_NAME: ghcr.io/nvidia/container-toolkit
+          VERSION: ${{ inputs.version }}
+          PUSH_ON_BUILD: "true"
+          BUILD_MULTI_ARCH_IMAGES: ${{ inputs.build_multi_arch_images }}
+        run: |
+          echo "${VERSION}"
+          make -f deployments/container/Makefile build-${{ matrix.target }}

.github/workflows/release.yaml

@@ -0,0 +1,38 @@
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run this workflow on new tags
+name: Release
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out code
+
+      - name: Prepare Artifacts
+        run: |
+          ./hack/prepare-artifacts.sh ${{ github.ref_name }}
+
+      - name: Create Draft Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          ./hack/create-release.sh ${{ github.ref_name }}

.gitignore

@@ -1,9 +1,11 @@
-dist
+/dist
+/artifacts
 *.swp
 *.swo
 /coverage.out*
-/test/output/
-/nvidia-container-runtime
-/nvidia-container-toolkit
-/nvidia-ctk
+/tests/output/
+/nvidia-*
 /shared-*
+/release-*
+/bin
+/toolkit-test

.gitlab-ci.yml

@@ -1,383 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include:
-  - .common-ci.yml
-
-build-dev-image:
-  stage: image
-  script:
-    - apk --no-cache add make bash
-    - make .build-image
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - make .push-build-image
-
-.requires-build-image:
-  image: "${BUILDIMAGE}"
-
-.go-check:
-  extends:
-    - .requires-build-image
-  stage: go-checks
-
-fmt:
-  extends:
-    - .go-check
-  script:
-    - make assert-fmt
-
-vet:
-  extends:
-    - .go-check
-  script:
-    - make vet
-
-lint:
-  extends:
-    - .go-check
-  script:
-    - make lint
-  allow_failure: true
-
-ineffassign:
-  extends:
-    - .go-check
-  script:
-    - make ineffassign
-  allow_failure: true
-
-misspell:
-  extends:
-    - .go-check
-  script:
-    - make misspell
-
-go-build:
-  extends:
-    - .requires-build-image
-  stage: go-build
-  script:
-    - make build
-
-unit-tests:
-  extends:
-    - .requires-build-image
-  stage: unit-tests
-  script:
-    - make coverage
-
-# Define the package build helpers
-.multi-arch-build:
-  before_script:
-    - apk add --no-cache coreutils build-base sed git bash make
-    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
-
-.package-artifacts:
-  variables:
-    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
-    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
-    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
-
-.package-build:
-  extends:
-    - .multi-arch-build
-    - .package-artifacts
-  stage: package-build
-  timeout: 2h 30m
-  script:
-    - ./scripts/release.sh ${DIST}-${ARCH}
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_ROOT}
-
-# Define the package build targets
-package-amazonlinux2-aarch64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-aarch64
-
-package-amazonlinux2-x86_64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-package-centos7-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-ppc64le
-
-package-centos7-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-x86_64
-
-package-centos8-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-aarch64
-
-package-centos8-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-ppc64le
-
-package-centos8-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-x86_64
-
-package-debian10-amd64:
-  extends:
-    - .package-build
-    - .dist-debian10
-    - .arch-amd64
-
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-opensuse-leap15.1-x86_64:
-  extends:
-    - .package-build
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
-package-ubuntu18.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-amd64
-
-package-ubuntu18.04-arm64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-arm64
-
-package-ubuntu18.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-ppc64le
-
-.buildx-setup:
-  before_script:
-    -  export BUILDX_VERSION=v0.6.3
-    -  apk add --no-cache curl
-    -  mkdir -p ~/.docker/cli-plugins
-    -  curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
-    -  chmod a+x ~/.docker/cli-plugins/docker-buildx
-
-    -  docker buildx create --use --platform=linux/amd64,linux/arm64
-
-    -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
-
-# Define the image build targets
-.image-build:
-  stage: image-build
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    PUSH_ON_BUILD: "true"
-  before_script:
-    - !reference [.buildx-setup, before_script]
-
-    - apk add --no-cache bash make
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-  script:
-    - make -f build/container/Makefile build-${DIST}
-
-image-centos7:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos7
-  needs:
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-
-image-centos8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos8
-  needs:
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
-image-ubi8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubi8
-  needs:
-    # Note: The ubi8 image uses the centos8 packages
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
-
-image-ubuntu20.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu20.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-packaging
-  needs:
-    - package-amazonlinux2-aarch64
-    - package-amazonlinux2-x86_64
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-    - package-centos8-aarch64
-    - package-centos8-ppc64le
-    - package-centos8-x86_64
-    - package-debian10-amd64
-    - package-debian9-amd64
-    - package-opensuse-leap15.1-x86_64
-    - package-ubuntu16.04-amd64
-    - package-ubuntu16.04-ppc64le
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
-
-# Define publish test helpers
-.test:toolkit:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "toolkit"
-
-.test:docker:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "docker"
-
-.test:containerd:
-  # TODO: The containerd tests fail due to issues with SIGHUP.
-  # Until this is resolved with retry up to twice and allow failure here.
-  retry: 2
-  allow_failure: true
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "containerd"
-
-.test:crio:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "crio"
-
-# Define the test targets
-test-toolkit-ubuntu18.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-containerd-ubuntu18.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-crio-ubuntu18.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-docker-ubuntu18.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-toolkit-ubuntu20.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-containerd-ubuntu20.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-crio-ubuntu20.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-docker-ubuntu20.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-

.gitmodules

@@ -1,9 +1,4 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
-	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
-[submodule "third_party/nvidia-container-runtime"]
-	path = third_party/nvidia-container-runtime
-	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
-[submodule "third_party/nvidia-docker"]
-	path = third_party/nvidia-docker
-	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
+	url = https://github.com/NVIDIA/libnvidia-container.git
+	branch = main

.golangci.yml

@@ -0,0 +1,72 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2"
+linters:
+  enable:
+    - contextcheck
+    - gocritic
+    - gosec
+    - misspell
+    - unconvert
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      # Exclude the gocritic dupSubExpr issue for cgo files.
+      - linters:
+          - gocritic
+        path: internal/dxcore/dxcore.go
+        text: dupSubExpr
+      # Exclude the checks for usage of returns to config.Delete(Path) in the
+      # crio and containerd config packages.
+      - linters:
+          - errcheck
+        path: pkg/config/engine/
+        text: config.Delete
+      # RENDERD refers to the Render Device and not the past tense of render.
+      - linters:
+          - misspell
+        path: .*.go
+        text: '`RENDERD` is a misspelling of `RENDERED`'
+      # The legacy hook relies on spec.Hooks.Prestart, which is deprecated as of
+      # the v1.2.0 OCI runtime spec.
+      - path: (.+)\.go$
+        text: SA1019:(.+).Prestart is deprecated(.+)
+      # TODO: We should address each of the following integer overflows.
+      - path: (.+)\.go$
+        text: 'G115: integer overflow conversion(.+)'
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  settings:
+    goimports:
+      local-prefixes:
+        - github.com/NVIDIA/nvidia-container-toolkit
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,74 +27,82 @@ default:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_TLS_CERTDIR: "/certs"
-  # Release "devel"-tagged images off the master branch
-  RELEASE_DEVEL_BRANCH: "master"
+  # Release "devel"-tagged images off the main branch
+  RELEASE_DEVEL_BRANCH: "main"
   DEVEL_RELEASE_IMAGE_VERSION: "devel"
   # On the multi-arch builder we don't need the qemu setup.
   SKIP_QEMU_SETUP: "1"
   # Define the public staging registry
-  STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
+  STAGING_REGISTRY: ghcr.io/nvidia
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  KITMAKER_RELEASE_FOLDER: "kitmaker"
+  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
-.image-pull:
-  stage: image-build
+# .copy-images copies the required application and packaging images from the
+# IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
+# to
+# OUT_IMAGE="${OUT_IMAGE_NAME}:${OUT_IMAGE_TAG}${TAG_SUFFIX}"
+# The script also logs into IN_REGISTRY and OUT_REGISTRY using the supplied
+# username and tokens.
+.copy-images:
+  parallel:
+    matrix:
+      - TAG_SUFFIX: ["", "-packaging"]
+  before_script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache make bash
+  variables:
+    REGCTL: regctl
+  script:
+    - |
+      if [ -n ${IN_REGISTRY} ] && [ -n ${IN_REGISTRY_USER} ]; then
+        echo "Logging in to ${IN_REGISTRY}"
+        ${REGCTL} registry login "${IN_REGISTRY}" -u "${IN_REGISTRY_USER}" -p "${IN_REGISTRY_TOKEN}" || exit 1
+      fi
+
+      if [ -n ${OUT_REGISTRY} ] && [ -n ${OUT_REGISTRY_USER} ] && [ "${IN_REGISTRY}" != "${OUT_REGISTRY}" ]; then
+        echo "Logging in to ${OUT_REGISTRY}"
+        ${REGCTL} registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" || exit 1
+      fi
+
+      export IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
+      export OUT_IMAGE="${OUT_IMAGE_NAME}:${OUT_IMAGE_TAG}${TAG_SUFFIX}"
+
+      echo "Copying ${IN_IMAGE} to ${OUT_IMAGE}"
+      ${REGCTL} image copy ${IN_IMAGE} ${OUT_IMAGE}
+
+# pull-images pulls images from the public CI registry to the internal CI registry.
+pull-images:
+  extends:
+    - .copy-images
+  stage: pull
   variables:
     IN_REGISTRY: "${STAGING_REGISTRY}"
-    IN_IMAGE_NAME: container-toolkit
-    IN_VERSION: "${STAGING_VERSION}"
+    IN_IMAGE_NAME: ${STAGING_REGISTRY}/container-toolkit
+    IN_IMAGE_TAG: "${STAGING_VERSION}"
+
+    OUT_REGISTRY: "${CI_REGISTRY}"
     OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
     OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    OUT_REGISTRY: "${CI_REGISTRY}"
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    PUSH_MULTIPLE_TAGS: "false"
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
   # We delay the job start to allow the public pipeline to generate the required images.
-  when: delayed
-  start_in: 30 minutes
+  rules:
+    # If the pipeline is triggered from a tag or the WEB UI we don't delay the
+    # start of the pipeline.
+    - if: $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "web"
+    # If the pipeline is triggered through other means (i.e. a branch or MR)
+    # we add a 30 minute delay to ensure that the images are available in the
+    # public CI registry.
+    - when: delayed
+      start_in: 30 minutes
   timeout: 30 minutes
   retry:
     max: 2
     when:
       - job_execution_timeout
       - stuck_or_timeout_failure
-  before_script:
-    - !reference [.regctl-setup, before_script]
-    - apk add --no-cache make bash
-    - >
-      regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
-  script:
-    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
-    - make -f build/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
-
-image-centos7:
-  extends:
-    - .image-pull
-    - .dist-centos7
-
-image-centos8:
-  extends:
-    - .image-pull
-    - .dist-centos8
-
-image-ubi8:
-  extends:
-    - .image-pull
-    - .dist-ubi8
-
-image-ubuntu18.04:
-  extends:
-    - .image-pull
-    - .dist-ubuntu18.04
-
-image-ubuntu20.04:
-  extends:
-    - .image-pull
-    - .dist-ubuntu20.04
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .image-pull
-    - .dist-packaging
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -106,26 +114,37 @@ image-packaging:
 
 # The .scan step forms the base of the image scan operation performed before releasing
 # images.
-.scan:
+scan-images:
   stage: scan
+  needs:
+  - pull-images
   image: "${PULSE_IMAGE}"
+  parallel:
+    matrix:
+      - TAG_SUFFIX: [""]
+        PLATFORM: ["linux/amd64", "linux/arm64"]
+      - TAG_SUFFIX: "-packaging"
+        PLATFORM: "linux/amd64"
   variables:
-    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit.tar"
-  except:
-    variables:
-      - $SKIP_SCANS && $SKIP_SCANS == "yes"
-  before_script:
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    # TODO: We should specify the architecture here and scan all architectures
-    - docker pull --platform="${PLATFORM}" "${IMAGE}"
-    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
-    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
-    - >
-      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
-    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}"
+    IMAGE_ARCHIVE: "container-toolkit-${CI_JOB_ID}.tar"
+  rules:
+    - if: $IGNORE_SCANS == "yes"
+      allow_failure: true
+    - when: on_success
   script:
-    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+    - |
+      docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+      export SCAN_IMAGE=${IMAGE}${TAG_SUFFIX}
+      echo "Scanning image ${SCAN_IMAGE} for ${PLATFORM}"
+      docker pull --platform="${PLATFORM}" "${SCAN_IMAGE}"
+      docker save "${SCAN_IMAGE}" -o "${IMAGE_ARCHIVE}"
+      AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+      if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+
+      pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+      rm -f "${IMAGE_ARCHIVE}"
   artifacts:
     when: always
     expire_in: 1 week
@@ -136,174 +155,138 @@ image-packaging:
       - vulns.json
       - policy_evaluation.json
 
-# Define the scan targets
-scan-centos7-amd64:
-  extends:
-    - .scan
-    - .dist-centos7
-    - .platform-amd64
+upload-kitmaker-packages:
+  stage: release
   needs:
-    - image-centos7
-
-scan-centos7-arm64:
-  extends:
-    - .scan
-    - .dist-centos7
-    - .platform-arm64
-  needs:
-    - image-centos7
-    - scan-centos7-amd64
-
-scan-centos8-amd64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-amd64
-  needs:
-    - image-centos8
-
-scan-centos8-arm64:
-  extends:
-    - .scan
-    - .dist-centos8
-    - .platform-arm64
-  needs:
-    - image-centos8
-    - scan-centos8-amd64
-
-scan-ubuntu18.04-amd64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-amd64
-  needs:
-    - image-ubuntu18.04
-
-scan-ubuntu18.04-arm64:
-  extends:
-    - .scan
-    - .dist-ubuntu18.04
-    - .platform-arm64
-  needs:
-    - image-ubuntu18.04
-    - scan-ubuntu18.04-amd64
-
-scan-ubuntu20.04-amd64:
-  extends:
-    - .scan
-    - .dist-ubuntu20.04
-    - .platform-amd64
-  needs:
-    - image-ubuntu20.04
-
-scan-ubuntu20.04-arm64:
-  extends:
-    - .scan
-    - .dist-ubuntu20.04
-    - .platform-arm64
-  needs:
-    - image-ubuntu20.04
-    - scan-ubuntu20.04-amd64
-
-scan-ubi8-amd64:
-  extends:
-    - .scan
-    - .dist-ubi8
-    - .platform-amd64
-  needs:
-    - image-ubi8
-
-scan-ubi8-arm64:
-  extends:
-    - .scan
-    - .dist-ubi8
-    - .platform-arm64
-  needs:
-    - image-ubi8
-    - scan-ubi8-amd64
-
-# Define external release helpers
-.release:ngc:
-  extends:
-    - .release:external
+    - pull-images
   variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+    ARTIFACTS_DIR: "${CI_PROJECT_DIR}/artifacts"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash git
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+    - rm -rf ${ARTIFACTS_DIR}
+
+push-images-to-staging:
+  extends:
+    - .copy-images
+  stage: release
+  needs:
+    - scan-images
+  variables:
+    IN_REGISTRY: "${CI_REGISTRY}"
+    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
+
+    OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
     OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    OUT_IMAGE_NAME: "${NGC_STAGING_REGISTRY}/container-toolkit"
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
 
-.release:dockerhub:
+.release-images:
   extends:
-    - .release:external
+    - .copy-images
+  stage: release
+  needs:
+    - scan-images
+    - push-images-to-staging
   variables:
-    OUT_REGISTRY_USER: "${REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${DOCKERHUB_REGISTRY}"
-    OUT_IMAGE_NAME: "${REGISTRY_IMAGE}"
+    IN_REGISTRY: "${CI_REGISTRY}"
+    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
 
-release:staging-ubuntu18.04:
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    OUT_IMAGE_TAG: "${CI_COMMIT_TAG}"
+
+release-images-to-ngc:
   extends:
-    - .release:staging
-    - .dist-ubuntu18.04
+    - .release-images
+  rules:
+    - if: $CI_COMMIT_TAG
+
+release-images-dummy:
+  extends:
+    - .release-images
+  variables:
+    REGCTL: "echo [DUMMY] regctl"
+  rules:
+    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
+
+# .sign-images forms the base of the jobs which sign images in the NGC registry.
+.sign-images:
+  stage: sign
+  image: ubuntu:latest
+  parallel:
+    matrix:
+      - TAG_SUFFIX: ["", "-packaging"]
+  variables:
+    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    IMAGE_TAG: "${CI_COMMIT_TAG}"
+    NGC_CLI: "ngc-cli/ngc"
+  before_script:
+    - !reference [.ngccli-setup, before_script]
+  script:
+    - |
+      # We ensure that the IMAGE_NAME and IMAGE_TAG is set
+      echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1
+      echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1
+
+      export IMAGE=${IMAGE_NAME}:${IMAGE_TAG}${TAG_SUFFIX}
+      echo "Signing the image ${IMAGE}"
+      ${NGC_CLI} registry image publish --source ${IMAGE} ${IMAGE} --public --discoverable --allow-guest --sign --org nvidia
+
+# Define the external image signing steps for NGC
+# Download the ngc cli binary for use in the sign steps
+.ngccli-setup:
+  before_script:
+    - apt-get update && apt-get install -y curl unzip jq
+    - |
+      if [ -z "${NGCCLI_VERSION}" ]; then
+        NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions"
+        # Extract the latest version from the JSON data using jq
+        export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr')
+      fi
+      echo "NGCCLI_VERSION ${NGCCLI_VERSION}"
+    - curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip
+    - unzip ngccli_linux.zip
+    - chmod u+x ngc-cli/ngc
+
+sign-ngc-images:
+  extends:
+    - .sign-images
   needs:
-    - image-ubuntu18.04
+    - release-images-to-ngc
+  rules:
+    - if: $CI_COMMIT_TAG
+  variables:
+    NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
+  retry:
+    max: 2
 
-release:staging-ubuntu20.04:
+sign-images-dummy:
   extends:
-    - .release:staging
-    - .dist-ubuntu20.04
+    - .sign-images
   needs:
-    - image-ubuntu20.04
-
-# Define the external release targets
-# Release to NGC
-release:ngc-centos7:
-  extends:
-    - .release:ngc
-    - .dist-centos7
-
-release:ngc-centos8:
-  extends:
-    - .release:ngc
-    - .dist-centos8
-
-release:ngc-ubuntu18.04:
-  extends:
-    - .release:ngc
-    - .dist-ubuntu18.04
-
-release:ngc-ubuntu20.04:
-  extends:
-    - .release:ngc
-    - .dist-ubuntu20.04
-
-release:ngc-ubi8:
-  extends:
-    - .release:ngc
-    - .dist-ubi8
-
-# Release to Dockerhub
-release:dockerhub-centos7:
-  extends:
-    - .release:dockerhub
-    - .dist-centos7
-
-release:dockerhub-centos8:
-  extends:
-    - .release:dockerhub
-    - .dist-centos8
-
-release:dockerhub-ubuntu18.04:
-  extends:
-    - .release:dockerhub
-    - .dist-ubuntu18.04
-
-release:dockerhub-ubuntu20.04:
-  extends:
-    - .release:dockerhub
-    - .dist-ubuntu20.04
-
-release:dockerhub-ubi8:
-  extends:
-    - .release:dockerhub
-    - .dist-ubi8
+    - release-images-dummy
+  variables:
+    NGC_CLI: "echo [DUMMY] ngc-cli/ngc"
+    IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
+  rules:
+    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""

CHANGELOG.md

@@ -0,0 +1,646 @@
+# NVIDIA Container Toolkit Changelog
+
+## v1.18.0-rc.1
+
+- Add create-soname-symlinks hook
+- Require matching version of libnvidia-container-tools
+- Add envvar for libcuda.so parent dir to CDI spec
+- Add EnvVar to Discover interface
+- Resolve to legacy by default in nvidia-container-runtime-hook
+- Default to jit-cdi mode in the nvidia runtime
+- Use functional options to construct runtime mode resolver
+- Add NVIDIA_CTK_CONFIG_FILE_PATH envvar
+- Switch to cuda ubi9 base image
+- Use single version tag for image
+- BUGFIX: modifier: respect GPU volume-mount device requests
+- Ensure consistent sorting of annotation devices
+- Extract deb and rpm packages to single image
+- Remove docker-run as default runtime candidate
+- Return annotation devices from VisibleDevices
+- Make CDI device requests consistent with other methods
+- Construct container info once
+- Add logic to extract annotation device requests to image type
+- Add IsPrivileged function to CUDA container type
+- Add device IDs to nvcdi.GetSpec API
+- Refactor extracting requested devices from the container image
+- Add EnvVars option for all nvidia-ctk cdi commands
+- Add nvidia-cdi-refresh service
+- Add discovery of arch-specific vulkan ICD
+- Add disabled-device-node-modification hook to CDI spec
+- Add a hook to disable device node creation in a container
+- Remove redundant deduplication of search paths for WSL
+- Added ability to disable specific (or all) CDI hooks
+- Consolidate HookName functionality on internal/discover pkg
+- Add envvar to control debug logging in CDI hooks
+- Add FeatureFlags to the nvcdi API
+- Reenable nvsandboxutils for driver discovery
+- Edit discover.mounts to have a deterministic output
+- Refactor the way we create CDI Hooks
+- Issue warning on unsupported CDI hook
+- Run update-ldcache in isolated namespaces
+- Add cuda-compat-mode config option
+- Fix mode detection on Thor-based systems
+- Add rprivate to CDI mount options
+- Skip nil discoverers in merge
+- bump runc go dep to v1.3.0
+- Fix resolution of libs in LDCache on ARM
+- Updated .release:staging to stage images in nvstaging
+- Refactor toolkit installer
+- Allow container runtime executable path to be specified
+- Add support for building ubuntu22.04 on arm64
+- Fix race condition in mounts cache
+- Add support for building ubuntu22.04 on amd64
+- Fix update-ldcache arguments
+- Remove positional arguments from nvidia-ctk-installer
+- Remove deprecated --runtime-args from nvidia-ctk-installer
+- Add version info to nvidia-ctk-installer
+- Update nvidia-ctk-installer app name to match binary name
+- Allow nvidia-ctk config --set to accept comma-separated lists
+- Disable enable-cuda-compat hook for management containers
+- Allow enable-cuda-compat hook to be disabled in CDI spec generation
+- Add disable-cuda-compat-lib-hook feature flag
+- Add basic integration tests for forward compat
+- Ensure that mode hook is executed last
+- Add enable-cuda-compat hook to CDI spec generation
+- Add ldconfig hook in legacy mode
+- Add enable-cuda-compat hook if required
+- Add enable-cuda-compat hook to allow compat libs to be discovered
+- Use libcontainer execseal to run ldconfig
+- Add ignore-imex-channel-requests feature flag
+- Disable nvsandboxutils in nvcdi API
+- Allow cdi mode to work with --gpus flag
+- Add E2E GitHub Action for Container Toolkit
+- Add remote-test option for E2E
+- Enable CDI in runtime if CDI_ENABLED is set
+- Fix overwriting docker feature flags
+- Add option in toolkit container to enable CDI in runtime
+- Remove Set from engine config API
+- Add EnableCDI() method to engine.Interface
+- Add IMEX binaries to CDI discovery
+- Rename test folder to tests
+- Add allow-cuda-compat-libs-from-container feature flag
+- Disable mounting of compat libs from container
+- Skip graphics modifier in CSV mode
+- Move nvidia-toolkit to nvidia-ctk-installer
+- Automated regression testing for the NVIDIA Container Toolkit
+- Add support for containerd version 3 config
+- Remove watch option from create-dev-char-symlinks
+- Add string TOML source
+- Improve the implementation for UseLegacyConfig
+- Properly pass configSearchPaths to a Driver constructor
+- Fix create-device-node test when devices exist
+- Add imex mode to CDI spec generation
+- Only allow host-relative LDConfig paths
+- Fix NVIDIA_IMEX_CHANNELS handling on legacy images
+- Fix bug in default config file path
+- Fix fsnotify.Remove logic function.
+- Force symlink creation in create-symlink hook
+
+### Changes in the Toolkit Container
+
+- Create /work/nvidia-toolkit symlink
+- Use Apache license for images
+- Switch to golang distroless image
+- Switch to cuda ubi9 base image
+- Use single version tag for image
+- Extract deb and rpm packages to single image
+- Bump nvidia/cuda in /deployments/container
+- Bump nvidia/cuda in /deployments/container
+- Add E2E GitHub Action for Container Toolkit
+- Bump nvidia/cuda in /deployments/container
+- Move nvidia-toolkit to nvidia-ctk-installer
+- Add support for containerd version 3 config
+- Improve the implementation for UseLegacyConfig
+- Bump nvidia/cuda in /deployments/container
+- Add imex mode to CDI spec generation
+- Only allow host-relative LDConfig paths
+- Fallback to file for runtime config
+
+### Changes in libnvidia-container
+
+- Fix pointer accessing local variable out of scope
+- Require version match between libnvidia-container-tools and libnvidia-container1
+- Add libnvidia-gpucomp.so to the list of compute libs
+- Use VERSION_ prefix for version parts in makefiles
+- Add additional logging
+- Do not discard container flags when --cuda-compat-mode is not specified
+- Remove unneeded --no-cntlibs argument from list command
+- Add cuda-compat-mode flag to configure command
+- Skip files when user has insufficient permissions
+- Fix building with Go 1.24
+- Add no-cntlibs CLI option to nvidia-container-cli
+- Fix always using fallback
+- Add fallback for systems without memfd_create()
+- Create virtual copy of host ldconfig binary before calling fexecve()
+- Fix some typos in text.
+
+
+## v1.17.0
+- Promote v1.17.0-rc.2 to v1.17.0
+- Fix bug when using just-in-time CDI spec generation
+- Check for valid paths in create-symlinks hook
+
+## v1.17.0-rc.2
+- Fix bug in locating libcuda.so from ldcache
+- Fix bug in sorting of symlink chain
+- Remove unsupported print-ldcache command
+- Remove csv-filename support from create-symlinks
+
+### Changes in the Toolkit Container
+- Fallback to `crio-status` if `crio status` does not work when configuring the crio runtime
+
+## v1.17.0-rc.1
+- Allow IMEX channels to be requested as volume mounts
+- Fix typo in error message
+- Add disable-imex-channel-creation feature flag
+- Add -z,lazy to LDFLAGS
+- Add imex channels to management CDI spec
+- Add support to fetch current container runtime config from the command line.
+- Add creation of select driver symlinks to CDI spec generation.
+- Remove support for config overrides when configuring runtimes.
+- Skip explicit creation of libnvidia-allocator.so.1 symlink
+- Add vdpau as as a driver library search path.
+- Add support for using libnvsandboxutils to generate CDI specifications.
+
+### Changes in the Toolkit Container
+
+- Allow opt-in features to be selected when deploying the toolkit-container.
+- Bump CUDA base image version to 12.6.2
+- Remove support for config overrides when configuring runtimes.
+
+### Changes in libnvidia-container
+
+- Add no-create-imex-channels command line option.
+
+## v1.16.2
+- Exclude libnvidia-allocator from graphics mounts. This fixes a bug that leaks mounts when a container is started with bi-directional mount propagation.
+- Use empty string for default runtime-config-override. This removes a redundant warning for runtimes (e.g. Docker) where this is not applicable.
+
+### Changes in the Toolkit Container
+- Bump CUDA base image version to 12.6.0
+
+### Changes in libnvidia-container
+- Add no-gsp-firmware command line option
+- Add no-fabricmanager command line option
+- Add no-persistenced command line option
+- Skip directories and symlinks when mounting libraries.
+
+## v1.16.1
+- Fix bug with processing errors during CDI spec generation for MIG devices
+
+## v1.16.0
+- Promote v1.16.0-rc.2 to v1.16.0
+
+### Changes in the Toolkit Container
+- Bump CUDA base image version to 12.5.1
+
+## v1.16.0-rc.2
+- Use relative path to locate driver libraries
+- Add RelativeToRoot function to Driver
+- Inject additional libraries for full X11 functionality
+- Extract options from default runtime if runc does not exist
+- Avoid using map pointers as maps are always passed by reference
+- Reduce logging for the NVIDIA Container runtime
+- Fix bug in argument parsing for logger creation
+
+## v1.16.0-rc.1
+
+- Support vulkan ICD files directly in a driver root. This allows for the discovery of vulkan files in GKE driver installations.
+- Increase priority of ld.so.conf.d config file injected into container. This ensures that injected libraries are preferred over libraries present in the container.
+- Set default CDI spec permissions to 644. This fixes permission issues when using the `nvidia-ctk cdi transform` functions.
+- Add `dev-root` option to `nvidia-ctk system create-device-nodes` command.
+- Fix location of `libnvidia-ml.so.1` when a non-standard driver root is used. This enabled CDI spec generation when using the driver container on a host.
+- Recalculate minimum required CDI spec version on save.
+- Move `nvidia-ctk hook` commands to a separate `nvidia-cdi-hook` binary. The same subcommands are supported.
+- Use `:` as an `nvidia-ctk config --set` list separator. This fixes a bug when trying to set config options that are lists.
+
+- [toolkit-container] Bump CUDA base image version to 12.5.0
+- [toolkit-container] Allow the path to `toolkit.pid` to be specified directly.
+- [toolkit-container] Remove provenance information from image manifests.
+- [toolkit-container] Add `dev-root` option when configuring the toolkit. This adds support for GKE driver installations.
+
+## v1.15.0
+
+* Remove `nvidia-container-runtime` and `nvidia-docker2` packages.
+* Use `XDG_DATA_DIRS` environment variable when locating config files such as graphics config files.
+* Add support for v0.7.0 Container Device Interface (CDI) specification.
+* Add `--config-search-path` option to `nvidia-ctk cdi generate` command. These paths are used when locating driver files such as graphics config files.
+* Use D3DKMTEnumAdapters3 to enumerate adpaters on WSL2 if available.
+* Add support for v1.2.0 OCI Runtime specification.
+* Explicitly set `NVIDIA_VISIBLE_DEVICES=void` in generated CDI specifications. This prevents the NVIDIA Container Runtime from making additional modifications.
+
+* [libnvidia-container] Use D3DKMTEnumAdapters3 to enumerate adpaters on WSL2 if available.
+
+* [toolkit-container] Bump CUDA base image version to 12.4.1
+
+## v1.15.0-rc.4
+* Add a `--spec-dir` option to the `nvidia-ctk cdi generate` command. This allows specs outside of `/etc/cdi` and `/var/run/cdi` to be processed.
+* Add support for extracting device major number from `/proc/devices` if `nvidia` is used as a device name over `nvidia-frontend`.
+* Allow multiple device naming strategies for `nvidia-ctk cdi generate` command. This allows a single
+  CDI spec to be generated that includes GPUs by index and UUID.
+* Set the default `--device-name-strategy` for the `nvidia-ctk cdi generate` command to `[index, uuid]`.
+* Remove `libnvidia-container0` jetpack dependency included for legacy Tegra-based systems.
+* Add `NVIDIA_VISIBLE_DEVICES=void` to generated CDI specifications.
+
+* [toolkit-container] Remove centos7 image. The ubi8 image can be used on all RPM-based platforms.
+* [toolkit-container] Bump CUDA base image version to 12.3.2
+
+## v1.15.0-rc.3
+* Fix bug in `nvidia-ctk hook update-ldcache` where default `--ldconfig-path` value was not applied.
+
+## v1.15.0-rc.2
+* Extend the `runtime.nvidia.com/gpu` CDI kind to support full-GPUs and MIG devices specified by index or UUID.
+* Fix bug when specifying `--dev-root` for Tegra-based systems.
+* Log explicitly requested runtime mode.
+* Remove package dependency on libseccomp.
+* Added detection of libnvdxgdmal.so.1 on WSL2
+* Use devRoot to resolve MIG device nodes.
+* Fix bug in determining default nvidia-container-runtime.user config value on SUSE-based systems.
+* Add `crun` to the list of configured low-level runtimes.
+* Added support for `--ldconfig-path` to `nvidia-ctk cdi generate` command.
+* Fix `nvidia-ctk runtime configure --cdi.enabled` for Docker.
+* Add discovery of the GDRCopy device (`gdrdrv`) if the `NVIDIA_GDRCOPY` environment variable of the container is set to `enabled`
+
+* [toolkit-container] Bump CUDA base image version to 12.3.1.
+
+## v1.15.0-rc.1
+* Skip update of ldcache in containers without ldconfig. The .so.SONAME symlinks are still created.
+* Normalize ldconfig path on use. This automatically adjust the ldconfig setting applied to ldconfig.real on systems where this exists.
+* Include `nvidia/nvoptix.bin` in list of graphics mounts.
+* Include `vulkan/icd.d/nvidia_layers.json` in list of graphics mounts.
+* Add support for `--library-search-paths` to `nvidia-ctk cdi generate` command.
+* Add support for injecting /dev/nvidia-nvswitch* devices if the NVIDIA_NVSWITCH=enabled envvar is specified.
+* Added support for `nvidia-ctk runtime configure --enable-cdi` for the `docker` runtime. Note that this requires Docker >= 25.
+* Fixed bug in `nvidia-ctk config` command when using `--set`. The types of applied config options are now applied correctly.
+* Add `--relative-to` option to `nvidia-ctk transform root` command. This controls whether the root transformation is applied to host or container paths.
+* Added automatic CDI spec generation when the `runtime.nvidia.com/gpu=all` device is requested by a container.
+
+* [libnvidia-container] Fix device permission check when using cgroupv2 (fixes #227)
+
+## v1.14.3
+* [toolkit-container] Bump CUDA base image version to 12.2.2.
+
+## v1.14.2
+* Fix bug on Tegra-based systems where symlinks were not created in containers.
+* Add --csv.ignore-pattern command line option to nvidia-ctk cdi generate command.
+
+## v1.14.1
+* Fixed bug where contents of `/etc/nvidia-container-runtime/config.toml` is ignored by the NVIDIA Container Runtime Hook.
+
+* [libnvidia-container] Use libelf.so on RPM-based systems due to removed mageia repositories hosting pmake and bmake.
+
+## v1.14.0
+* Promote v1.14.0-rc.3 to v1.14.0
+
+## v1.14.0-rc.3
+* Added support for generating OCI hook JSON file to `nvidia-ctk runtime configure` command.
+* Remove installation of OCI hook JSON from RPM package.
+* Refactored config for `nvidia-container-runtime-hook`.
+* Added a `nvidia-ctk config` command which supports setting config options using a `--set` flag.
+* Added `--library-search-path` option to `nvidia-ctk cdi generate` command in `csv` mode. This allows folders where
+  libraries are located to be specified explicitly.
+* Updated go-nvlib to support devices which are not present in the PCI device database. This allows the creation of dev/char symlinks on systems with such devices installed.
+* Added `UsesNVGPUModule` info function for more robust platform detection. This is required on Tegra-based systems where libnvidia-ml.so is also supported.
+
+* [toolkit-container] Set `NVIDIA_VISIBLE_DEVICES=void` to prevent injection of NVIDIA devices and drivers into the NVIDIA Container Toolkit container.
+
+## v1.14.0-rc.2
+* Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
+* Remove dependency on coreutils when installing package on RPM-based systems.
+* Create output folders if required when running `nvidia-ctk runtime configure`
+* Generate default config as post-install step.
+* Added support for detecting GSP firmware at custom paths when generating CDI specifications.
+* Added logic to skip the extraction of image requirements if `NVIDIA_DISABLE_REQUIRES` is set to `true`.
+
+* [libnvidia-container] Include Shared Compiler Library (libnvidia-gpucomp.so) in the list of compute libaries.
+
+* [toolkit-container] Ensure that common envvars have higher priority when configuring the container engines.
+* [toolkit-container] Bump CUDA base image version to 12.2.0.
+* [toolkit-container] Remove installation of nvidia-experimental runtime. This is superceded by the NVIDIA Container Runtime in CDI mode.
+
+## v1.14.0-rc.1
+
+* Add support for updating containerd configs to the `nvidia-ctk runtime configure` command.
+* Create file in `etc/ld.so.conf.d` with permissions `644` to support non-root containers.
+* Generate CDI specification files with `644` permissions to allow rootless applications (e.g. podman)
+* Add `nvidia-ctk cdi list` command to show the known CDI devices.
+* Add support for generating merged devices (e.g. `all` device) to the nvcdi API.
+* Use *.* pattern to locate libcuda.so when generating a CDI specification to support platforms where a patch version is not specified.
+* Update go-nvlib to skip devices that are not MIG capable when generating CDI specifications.
+* Add `nvidia-container-runtime-hook.path` config option to specify NVIDIA Container Runtime Hook path explicitly.
+* Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
+* Add option to load kernel modules when creating device nodes
+* Add option to create device nodes when creating `/dev/char` symlinks
+
+* [libnvidia-container] Support OpenSSL 3 with the Encrypt/Decrypt library
+
+* [toolkit-container] Allow same envars for all runtime configs
+
+## v1.13.1
+
+* Update `update-ldcache` hook to only update ldcache if it exists.
+* Update `update-ldcache` hook to create `/etc/ld.so.conf.d` folder if it doesn't exist.
+* Fix failure when libcuda cannot be located during XOrg library discovery.
+* Fix CDI spec generation on systems that use `/etc/alternatives` (e.g. Debian)
+
+## v1.13.0
+
+* Promote 1.13.0-rc.3 to 1.13.0
+
+## v1.13.0-rc.3
+
+* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
+* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
+* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
+* Add transformers to deduplicate and simplify CDI specifications.
+* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
+* Also return an error from the nvcdi.New constructor instead of panicing.
+* Detect XOrg libraries for injection and CDI spec generation.
+* Add `nvidia-ctk system create-device-nodes` command to create control devices.
+* Add `nvidia-ctk cdi transform` command to apply transforms to CDI specifications.
+* Add `--vendor` and `--class` options to `nvidia-ctk cdi generate`
+
+* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
+* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
+* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
+
+* [toolkit-container] Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
+* [toolkit-container] Create device nodes when generating CDI specification for management containers.
+* [toolkit-container] Add `nvidia-container-runtime.runtimes` config option to set the low-level runtime for the NVIDIA Container Runtime
+
+## v1.13.0-rc.2
+
+* Don't fail chmod hook if paths are not injected
+* Only create `by-path` symlinks if CDI devices are actually requested.
+* Fix possible blank `nvidia-ctk` path in generated CDI specifications
+* Fix error in postun scriplet on RPM-based systems
+* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
+* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
+* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
+* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
+* Add support for generating CDI specifications for GDS and MOFED devices
+* Ensure CDI specification is validated on save when generating a spec
+* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
+* [libnvidia-container] Fix segfault on WSL2 systems
+* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
+* [toolkit-container] Install `nvidia-ctk` from toolkit container
+* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
+* [toolkit-container] Bump CUDA base images to 12.1.0
+* [toolkit-container] Set `nvidia-ctk` path in the
+* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
+* [toolkit-container] Generate CDI specification for use in management containers
+* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
+* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
+
+## v1.13.0-rc.1
+
+* Include MIG-enabled devices as GPUs when generating CDI specification
+* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
+* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
+* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
+* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
+* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
+* Align `.deb` and `.rpm` release candidate package versions
+* Remove `fedora35` packaging targets
+* [libnvidia-container] Include all `gsp*.bin` firmware files if present
+* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
+* [libnvidia-container] Remove `fedora35` packaging targets
+* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
+* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
+
+## v1.12.0
+
+* Promote `v1.12.0-rc.5` to `v1.12.0`
+* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
+* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
+* [toolkit-container] Bump CUDA base images to 12.0.1
+
+## v1.12.0-rc.5
+
+* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
+## v1.12.0-rc.1
+
+* Add support for multiple Docker Swarm resources
+* Improve injection of Vulkan configurations and libraries
+* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
+* [libnvidia-container] Include NVVM compiler library in compute libs
+
+## v1.11.0
+
+* Promote v1.11.0-rc.3 to v1.11.0
+
+## v1.11.0-rc.3
+
+* Build fedora35 packages
+* Introduce an `nvidia-container-toolkit-base` package for better dependency management
+* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
+* Inject platform files into container on Tegra-based systems
+* [toolkit container] Update CUDA base images to 11.7.1
+* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
+
+## v1.11.0-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.0-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
+## v1.10.0
+
+* Promote v1.10.0-rc.3 to v1.10.0
+
+## v1.10.0-rc.3
+
+* Use default config instead of raising an error if config file cannot be found
+* Ignore NVIDIA_REQUIRE_JETPACK* environment variables for requirement checks
+* Fix bug in detection of Tegra systems where `/sys/devices/soc0/family` is ignored
+* Fix bug where links to devices were detected as devices
+* [libnvida-container] Fix bug introduced when adding libcudadebugger.so to list of libraries
+
+## v1.10.0-rc.2
+
+* Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
+* Switch to debug logging to reduce log verbosity
+* Support logging to logs requested in command line
+* Fix bug when launching containers with relative root path (e.g. using containerd)
+* Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
+* Fix failure to locate low-level runtime if PATH envvar is unset
+* Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
+* Use csv as default mode on Tegra systems without NVML
+* Add --version flag to all CLIs
+* [libnvidia-container] Bump libtirpc to 1.3.2
+* [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
+* [libnvidia-container] Add libcudadebugger.so to list of compute libraries
+
+## v1.10.0-rc.1
+
+* Include nvidia-ctk CLI in installed binaries
+* Add experimental option to NVIDIA Container Runtime
+
+## v1.9.0
+
+* [libnvidia-container] Add additional check for Tegra in /sys/.../family file in CLI
+* [libnvidia-container] Update jetpack-specific CLI option to only load Base CSV files by default
+* [libnvidia-container] Fix bug (from 1.8.0) when mounting GSP firmware into containers without /lib to /usr/lib symlinks
+* [libnvidia-container] Update nvml.h to CUDA 11.6.1 nvML_DEV 11.6.55
+* [libnvidia-container] Update switch statement to include new brands from latest nvml.h
+* [libnvidia-container] Process all --require flags on Jetson platforms
+* [libnvidia-container] Fix long-standing issue with running ldconfig on Debian systems
+
+## v1.8.1
+
+* [libnvidia-container] Fix bug in determining cgroup root when running in nested containers
+* [libnvidia-container] Fix permission issue when determining cgroup version
+
+## v1.8.0
+
+* Promote 1.8.0-rc.2-1 to 1.8.0
+
+## v1.8.0-rc.2
+
+* Remove support for building amazonlinux1 packages
+
+## v1.8.0-rc.1
+
+* [libnvidia-container] Add support for cgroupv2
+* Release toolkit-container images from nvidia-container-toolkit repository
+
+## v1.7.0
+
+* Promote 1.7.0-rc.1-1 to 1.7.0
+ * Bump Golang version to 1.16.4
+
+## v1.7.0-rc.1
+
+* Specify containerd runtime type as string in config tools to remove dependency on containerd package
+* Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
+
+## v1.6.0
+
+* Promote 1.6.0-rc.3-1 to 1.6.0
+ * Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
+
+## v1.6.0-rc.3
+
+* Add supported-driver-capabilities config option to the nvidia-container-toolkit
+* Move OCI and command line checks for runtime to internal oci package
+
+## v1.6.0-rc.2
+
+* Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+## v1.6.0-rc.1
+
+* Add AARCH64 package for Amazon Linux 2
+* Include nvidia-container-runtime into nvidia-container-toolkit package
+
+## v1.5.1
+
+* Fix bug where Docker Swarm device selection is ignored if NVIDIA_VISIBLE_DEVICES is also set
+* Improve unit testing by using require package and adding coverage reports
+* Remove unneeded go dependencies by running go mod tidy
+* Move contents of pkg directory to cmd for CLI tools
+* Ensure make binary target explicitly sets GOOS
+
+## v1.5.0
+
+* Add dependence on libnvidia-container-tools >= 1.4.0
+* Add golang check targets to Makefile
+* Add Jenkinsfile definition for build targets
+* Move docker.mk to docker folder
+
+## v1.4.2
+
+* Add dependence on libnvidia-container-tools >= 1.3.3
+
+## v1.4.1
+
+* Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
+* Add dependence on libnvidia-container-tools >= 1.3.2
+
+## v1.4.0
+
+* Add 'compute' capability to list of defaults
+* Add dependence on libnvidia-container-tools >= 1.3.1
+
+## v1.3.0
+
+* Promote 1.3.0-rc.2-1 to 1.3.0
+* Add dependence on libnvidia-container-tools >= 1.3.0
+
+## v1.3.0-rc.2
+
+* 2c180947 Add more tests for new semantics with device list from volume mounts
+* 7c003857 Refactor accepting device lists from volume mounts as a boolean
+
+## v1.3.0-rc.1
+
+* b50d86c1 Update build system to accept a TAG variable for things like rc.x
+* fe65573b Add common CI tests for things like golint, gofmt, unit tests, etc.
+* da6fbb34 Revert "Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*"
+* a7fb3330 Flip build-all targets to run automatically on merge requests
+* 8b248b66 Rename github.com/NVIDIA/container-toolkit to nvidia-container-toolkit
+* da36874e Add new config options to pull device list from mounted files instead of ENVVAR
+
+## v1.2.1
+
+* 4e6e0ed4 Add 'ngx' to list of*all* driver capabilities
+* 2f4af743 List config.toml as a config file in the RPM SPEC
+
+## v1.2.0
+
+*  8e0aab46 Fix repo listed in changelog for debian distributions
+*  320bb6e4 Update dependence on libnvidia-container to 1.2.0
+*  6cfc8097 Update package license to match source license
+*  e7dc3cbb Fix debian copyright file
+*  d3aee3e0 Add the 'ngx' driver capability
+
+## v1.1.2
+
+* c32237f3 Add support for parsing Linux Capabilities for older OCI specs
+
+## v1.1.1
+
+* d202aded Update dependence to libnvidia-container 1.1.1
+
+## v1.1.0
+
+* 4e4de762 Update build system to support multi-arch builds
+* fcc1d116 Add support for MIG (Multi-Instance GPUs)
+* d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
+* 60f165ad Add no-pivot option to toolkit
+
+## v1.0.5
+
+* Initial release. Replaces older package nvidia-container-runtime-hook. (Closes: #XXXXXX)

DEVELOPMENT.md

@@ -13,13 +13,15 @@ The `nvidia-container-toolkit` resides in this repo directly.
 
 In oder to build the packages, the following command is executed
 ```sh
-./scripts/build-all-components.sh TARGET
+./scripts/build-packages.sh TARGET
 ```
 where `TARGET` is a make target that is valid for each of the sub-components.
 
 These include:
 * `ubuntu18.04-amd64`
-* `centos8-x86_64`
+* `centos7-x86_64`
+
+If no `TARGET` is specified, all valid release targets are built.
 
 The packages are generated in the `dist` folder.
 
@@ -32,14 +34,28 @@ environment variables.
 
 ## Testing packages locally
 
-The [test/release](./test/release/) folder contains documentation on how the installation of local or staged packages can be tested.
+The [tests/release](./tests/release/) folder contains documentation on how the installation of local or staged packages can be tested.
 
 
 ## Releasing
 
-A utility script [`scripts/release.sh`](./scripts/release.sh) is provided to build
-packages required for release. If run without arguments, all supported distribution-architecture combinations are built. A specific distribution-architecture pair can also be provided
-```sh
-./scripts/release.sh ubuntu18.04-amd64
+In order to release packages required for a release, a utility script
+[`scripts/release-packages.sh`](./scripts/release-packages.sh) is provided.
+This script can be executed as follows:
+
+```bash
+GPG_LOCAL_USER="GPG_USER" \
+MASTER_KEY_PATH=/path/to/gpg-master.key \
+SUB_KEY_PATH=/path/to/gpg-subkey.key \
+    ./scripts/release-packages.sh REPO PACKAGE_REPO_ROOT [REFERENCE]
 ```
-where the `amd64` builds for `ubuntu18.04` are provided as an example.
+
+Where `REPO` is one of `stable` or `experimental`, `PACKAGE_REPO_ROOT` is the local path to the `libnvidia-container` repository checked out to the `gh-pages` branch, and `REFERENCE` is the git SHA that is to be released. If reference is not specified `HEAD` is assumed.
+
+This scripts performs the following basic functions:
+* Pulls the package image defined by the `REFERENCE` git SHA from the staging registry,
+* Copies the required packages to the package repository at `PACKAGE_REPO_ROOT/REPO`,
+* Signs the packages using the specified GPG keys
+
+While the last two are performed, commits are added to the package repository. These can be pushed to the relevant repository.
+

Jenkinsfile

@@ -1,142 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-podTemplate (cloud:'sw-gpu-cloudnative',
-    containers: [
-    containerTemplate(name: 'docker', image: 'docker:dind', ttyEnabled: true, privileged: true),
-    containerTemplate(name: 'golang', image: 'golang:1.16.3', ttyEnabled: true)
-  ]) {
-    node(POD_LABEL) {
-        def scmInfo
-
-        stage('checkout') {
-            scmInfo = checkout(scm)
-        }
-
-        stage('dependencies') {
-            container('golang') {
-                sh 'GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell'
-                sh 'GO111MODULE=off go get -u github.com/gordonklaus/ineffassign'
-                sh 'GO111MODULE=off go get -u golang.org/x/lint/golint'
-            }
-            container('docker') {
-                sh 'apk add --no-cache make bash git'
-            }
-        }
-        stage('check') {
-            parallel (
-                getGolangStages(["assert-fmt", "lint", "vet", "ineffassign", "misspell"])
-            )
-        }
-        stage('test') {
-            parallel (
-                getGolangStages(["test"])
-            )
-        }
-
-        def versionInfo
-        stage('version') {
-            container('docker') {
-                versionInfo = getVersionInfo(scmInfo)
-                println "versionInfo=${versionInfo}"
-            }
-        }
-
-        def dist = 'ubuntu20.04'
-        def arch = 'amd64'
-        def stageLabel = "${dist}-${arch}"
-
-        stage('build-one') {
-            container('docker') {
-                stage (stageLabel) {
-                    sh "make ${dist}-${arch}"
-                }
-            }
-        }
-
-        stage('release') {
-            container('docker') {
-                stage (stageLabel) {
-
-                    def component = 'main'
-                    def repository = 'sw-gpu-cloudnative-debian-local/pool/main/'
-
-                    def uploadSpec = """{
-                                        "files":
-                                        [  {
-                                                "pattern": "./dist/${dist}/${arch}/*.deb",
-                                                "target": "${repository}",
-                                                "props": "deb.distribution=${dist};deb.component=${component};deb.architecture=${arch}"
-                                            }
-                                        ]
-                                    }"""
-
-                    sh "echo starting release with versionInfo=${versionInfo}"
-                    if (versionInfo.isTag) {
-                        // upload to artifactory repository
-                        def server = Artifactory.server 'sw-gpu-artifactory'
-                        server.upload spec: uploadSpec
-                    } else {
-                        sh "echo skipping release for non-tagged build"
-                    }
-                }
-            }
-        }
-    }
-}
-
-def getGolangStages(def targets) {
-    stages = [:]
-
-    for (t in targets) {
-        stages[t] = getLintClosure(t)
-    }
-
-    return stages
-}
-
-def getLintClosure(def target) {
-    return {
-        container('golang') {
-            stage(target) {
-                sh "make ${target}"
-            }
-        }
-    }
-}
-
-// getVersionInfo returns a hash of version info
-def getVersionInfo(def scmInfo) {
-    def versionInfo = [
-        isTag: isTag(scmInfo.GIT_BRANCH)
-    ]
-
-    scmInfo.each { k, v -> versionInfo[k] = v }
-    return versionInfo
-}
-
-def isTag(def branch) {
-    if (!branch.startsWith('v')) {
-        return false
-    }
-
-    def version = shOutput('git describe --all --exact-match --always')
-    return version == "tags/${branch}"
-}
-
-def shOuptut(def script) {
-    return sh(script: script, returnStdout: true).trim()
-}

Makefile

@@ -38,32 +38,41 @@ EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES))
 CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
-$(info CMD_TARGETS=$(CMD_TARGETS))
-
-CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+CHECK_TARGETS := lint
+MAKE_TARGETS := binaries build check fmt test examples cmds coverage generate licenses vendor check-vendor $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
 DOCKER_TARGETS := $(patsubst %,docker-%, $(TARGETS))
 .PHONY: $(TARGETS) $(DOCKER_TARGETS)
 
-GOOS ?= linux
+ifeq ($(VERSION),)
+CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+else
+CLI_VERSION = $(VERSION)
+endif
+CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
 
 binaries: cmds
 ifneq ($(PREFIX),)
 cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
+
+ifneq ($(shell uname),Darwin)
+EXTLDFLAGS = -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files -Wl,-z,lazy
+else
+EXTLDFLAGS = -Wl,-undefined,dynamic_lookup
+endif
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-s -w" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	go build -ldflags "-s -w '-extldflags=$(EXTLDFLAGS)' -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
-	GOOS=$(GOOS) go build ./...
+	go build ./...
 
 examples: $(EXAMPLE_TARGETS)
 $(EXAMPLE_TARGETS): example-%:
-	GOOS=$(GOOS) go build ./examples/$(*)
+	go build ./examples/$(*)
 
 all: check test build binary
 check: $(CHECK_TARGETS)
@@ -73,68 +82,75 @@ fmt:
 	go list -f '{{.Dir}}' $(MODULE)/... \
 		| xargs gofmt -s -l -w
 
-assert-fmt:
-	go list -f '{{.Dir}}' $(MODULE)/... \
-		| xargs gofmt -s -l > fmt.out
-	@if [ -s fmt.out ]; then \
-		echo "\nERROR: The following files are not formatted:\n"; \
-		cat fmt.out; \
-		rm fmt.out; \
-		exit 1; \
-	else \
-		rm fmt.out; \
-	fi
-
-ineffassign:
-	ineffassign $(MODULE)/...
+# Apply goimports -local github.com/NVIDIA/container-toolkit to the codebase
+goimports:
+	go list -f {{.Dir}} $(MODULE)/... \
+		| xargs goimports -local $(MODULE) -w
 
 lint:
-# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
-	go list -f '{{.Dir}}' $(MODULE)/... | xargs golint -set_exit_status
+	golangci-lint run ./...
 
-misspell:
-	misspell $(MODULE)/...
+vendor:  | mod-tidy mod-vendor mod-verify
 
-vet:
-	go vet $(MODULE)/...
+mod-tidy:
+	@for mod in $$(find . -name go.mod -not -path "./testdata/*" -not -path "./third_party/*"); do \
+	    echo "Tidying $$mod..."; ( \
+	        cd $$(dirname $$mod) && go mod tidy \
+            ) || exit 1; \
+	done
+
+mod-vendor:
+	@for mod in $$(find . -name go.mod -not -path "./testdata/*" -not -path "./third_party/*" -not -path "./deployments/*"); do \
+		echo "Vendoring $$mod..."; ( \
+			cd $$(dirname $$mod) && go mod vendor \
+			) || exit 1; \
+	done
+
+mod-verify:
+	@for mod in $$(find . -name go.mod -not -path "./testdata/*" -not -path "./third_party/*"); do \
+	    echo "Verifying $$mod..."; ( \
+	        cd $$(dirname $$mod) && go mod verify | sed 's/^/  /g' \
+	    ) || exit 1; \
+	done
+
+
+check-vendor: vendor
+	git diff --exit-code HEAD -- go.mod go.sum vendor
+
+licenses:
+	go-licenses csv $(MODULE)/...
 
 COVERAGE_FILE := coverage.out
 test: build cmds
-	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...
+	go test -coverprofile=$(COVERAGE_FILE).with-mocks $(MODULE)/...
 
 coverage: test
-	cat $(COVERAGE_FILE) | grep -v "_mock.go" > $(COVERAGE_FILE).no-mocks
-	go tool cover -func=$(COVERAGE_FILE).no-mocks
+	cat $(COVERAGE_FILE).with-mocks | grep -v "_mock.go" > $(COVERAGE_FILE)
+	go tool cover -func=$(COVERAGE_FILE)
 
 generate:
 	go generate $(MODULE)/...
 
 # Generate an image for containerized builds
 # Note: This image is local only
-.PHONY: .build-image .pull-build-image .push-build-image
-.build-image: docker/Dockerfile.devel
-	if [ x"$(SKIP_IMAGE_BUILD)" = x"" ]; then \
-		$(DOCKER) build \
-			--progress=plain \
-			--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-			--tag $(BUILDIMAGE) \
-			-f $(^) \
-			docker; \
-	fi
+.PHONY: .build-image
+.build-image:
+	make -f deployments/devel/Makefile .build-image
 
-.pull-build-image:
-	$(DOCKER) pull $(BUILDIMAGE)
+ifeq ($(BUILD_DEVEL_IMAGE),yes)
+$(DOCKER_TARGETS): .build-image
+.shell: .build-image
+endif
 
-.push-build-image:
-	$(DOCKER) push $(BUILDIMAGE)
-
-$(DOCKER_TARGETS): docker-%: .build-image
-	@echo "Running 'make $(*)' in docker container $(BUILDIMAGE)"
+$(DOCKER_TARGETS): docker-%:
+	@echo "Running 'make $(*)' in container image $(BUILDIMAGE)"
 	$(DOCKER) run \
 		--rm \
-		-e GOCACHE=/tmp/.cache \
-		-v $(PWD):$(PWD) \
-		-w $(PWD) \
+		-e GOCACHE=/tmp/.cache/go \
+		-e GOMODCACHE=/tmp/.cache/gomod \
+		-e GOLANGCI_LINT_CACHE=/tmp/.cache/golangci-lint \
+		-v $(PWD):/work \
+		-w /work \
 		--user $$(id -u):$$(id -g) \
 		$(BUILDIMAGE) \
 			make $(*)
@@ -145,8 +161,10 @@ PHONY: .shell
 	$(DOCKER) run \
 		--rm \
 		-ti \
-		-e GOCACHE=/tmp/.cache \
-		-v $(PWD):$(PWD) \
-		-w $(PWD) \
+		-e GOCACHE=/tmp/.cache/go \
+		-e GOMODCACHE=/tmp/.cache/gomod \
+		-e GOLANGCI_LINT_CACHE=/tmp/.cache/golangci-lint \
+		-v $(PWD):/work \
+		-w /work \
 		--user $$(id -u):$$(id -g) \
 		$(BUILDIMAGE)

README.md

@@ -1,6 +1,6 @@
 # NVIDIA Container Toolkit
 
-[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/master/LICENSE)
+[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/main/LICENSE)
 [![Documentation](https://img.shields.io/badge/documentation-wiki-blue.svg?style=flat-square)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html)
 [![Package repository](https://img.shields.io/badge/packages-repository-b956e8.svg?style=flat-square)](https://nvidia.github.io/libnvidia-container)
 
@@ -28,4 +28,4 @@ The [user guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolk
 [Checkout the Contributing document!](CONTRIBUTING.md)
 
 * Please let us know by [filing a new issue](https://github.com/NVIDIA/nvidia-container-toolkit/issues/new)
-* You can contribute by creating a [merge request](https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/merge_requests/new) to our public GitLab repository
+* You can contribute by creating a [pull request](https://github.com/NVIDIA/nvidia-container-toolkit/compare) to our public GitHub repository

RELEASE.md

@@ -0,0 +1,36 @@
+# Release Process
+
+The NVIDIA Container Toolkit consists of the following artifacts:
+- The NVIDIA Container Toolkit container
+- Packages for debian-based systems
+- Packages for rpm-based systems
+
+# Release Process Checklist:
+- [ ] Create a release PR:
+    - [ ] Run the `./hack/prepare-release.sh` script to update the version in all the needed files. This also creates a [release issue](https://github.com/NVIDIA/cloud-native-team/issues?q=is%3Aissue+is%3Aopen+label%3Arelease)
+    - [ ] Run the `./hack/generate-changelog.sh` script to generate the a draft changelog and update `CHANGELOG.md` with the changes.
+    - [ ] Create a PR from the created `bump-release-{{ .VERSION }}` branch.
+- [ ] Merge the release PR
+- [ ] Tag the release and push the tag to the `internal` mirror:
+    - [ ] Image release pipeline: https://gitlab-master.nvidia.com/dl/container-dev/container-toolkit/-/pipelines/16466098
+- [ ] Wait for the image release to complete.
+- [ ] Push the tag to the the upstream GitHub repo.
+- [ ] Wait for the [`Release`](https://github.com/NVIDIA/k8s-device-plugin/actions/workflows/release.yaml) GitHub Action to complete
+- [ ] Publish the [draft release](https://github.com/NVIDIA/k8s-device-plugin/releases) created by the GitHub Action
+- [ ] Publish the packages to the gh-pages branch of the libnvidia-container repo
+- [ ] Create a KitPick
+
+## Troubleshooting
+
+*Note*: This assumes that we have the release tag checked out locally.
+
+- If the `Release` GitHub Action fails:
+    - Check the logs for the error first.
+    - Create the helm packages locally by running:
+      ```bash
+      ./hack/prepare-artifacts.sh {{ .VERSION }}
+      ```
+    - Create the draft release by running:
+      ```bash
+      ./hack/create-release.sh {{ .VERSION }}
+      ```

SECURITY.md

@@ -0,0 +1,24 @@
+# Security
+
+NVIDIA is dedicated to the security and trust of our software products and services, including all source code repositories managed through our organization.
+
+If you need to report a security issue, please use the appropriate contact points outlined below. **Please do not report security vulnerabilities through GitHub.**
+
+## Reporting Potential Security Vulnerability in an NVIDIA Product
+
+To report a potential security vulnerability in any NVIDIA product:
+- Web: [Security Vulnerability Submission Form](https://www.nvidia.com/object/submit-security-vulnerability.html)
+- E-Mail: psirt@nvidia.com
+    - We encourage you to use the following PGP key for secure email communication: [NVIDIA public PGP Key for communication](https://www.nvidia.com/en-us/security/pgp-key)
+    - Please include the following information:
+        - Product/Driver name and version/branch that contains the vulnerability
+     - Type of vulnerability (code execution, denial of service, buffer overflow, etc.)
+        - Instructions to reproduce the vulnerability
+        - Proof-of-concept or exploit code
+        - Potential impact of the vulnerability, including how an attacker could exploit the vulnerability
+
+While NVIDIA currently does not have a bug bounty program, we do offer acknowledgement when an externally reported security issue is addressed under our coordinated vulnerability disclosure policy. Please visit our [Product Security Incident Response Team (PSIRT)](https://www.nvidia.com/en-us/security/psirt-policies/) policies page for more information.
+
+## NVIDIA Product Security
+
+For all security-related concerns, please visit NVIDIA's Product Security portal at https://www.nvidia.com/en-us/security

build/container/Dockerfile.centos

@@ -1,97 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG BASE_DIST
-ARG CUDA_VERSION
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
-
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
-
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
-
-WORKDIR /build
-COPY . .
-
-# NOTE: Until the config utilities are properly integrated into the
-# nvidia-container-toolkit repository, these are built from the `tools` folder
-# and not `cmd`.
-RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
-
-
-FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
-
-ARG BASE_DIST
-# See https://www.centos.org/centos-linux-eol/
-# and https://stackoverflow.com/a/70930049 for move to vault.centos.org
-# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud
-RUN [[ "${BASE_DIST}" != "centos8" ]] || \
-    ( \
-      sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
-      sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \
-    )
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH ${TARGETARCH}
-RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
-    yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
-
-WORKDIR /work
-
-COPY --from=build /artifacts/bin /work
-
-ENV PATH=/work:$PATH
-
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-COPY ./LICENSE /licenses/LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        yum update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
-
-ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.ubuntu

@@ -1,102 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG BASE_DIST
-ARG CUDA_VERSION
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-# NOTE: In cases where the libc version is a concern, we would have to use an
-# image based on the target OS to build the golang executables here -- especially
-# if cgo code is included.
-FROM golang:${GOLANG_VERSION} as build
-
-# We override the GOPATH to ensure that the binaries are installed to
-# /artifacts/bin
-ARG GOPATH=/artifacts
-
-# Install the experiemental nvidia-container-runtime
-# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
-ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
-
-WORKDIR /build
-COPY . .
-
-# NOTE: Until the config utilities are properly integrated into the
-# nvidia-container-toolkit repository, these are built from the `tools` folder
-# and not `cmd`.
-RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
-
-
-FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libcap2 \
-    curl \
-        && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH ${TARGETARCH}
-
-ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
-ARG LIBNVIDIA_CONTAINER0_VERSION
-RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
-        curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
-            --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \
-        dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \
-    fi
-
-RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
-
-WORKDIR /work
-
-COPY --from=build /artifacts/bin /work/
-
-ENV PATH=/work:$PATH
-
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-COPY ./LICENSE /licenses/LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
-
-ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -1,152 +0,0 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-BUILD_MULTI_ARCH_IMAGES ?= false
-DOCKER ?= docker
-
-BUILDX =
-ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
-BUILDX = buildx
-endif
-
-MKDIR    ?= mkdir
-DIST_DIR ?= $(CURDIR)/dist
-
-##### Global variables #####
-include $(CURDIR)/versions.mk
-
-ifeq ($(IMAGE_NAME),)
-REGISTRY ?= nvidia
-IMAGE_NAME := $(REGISTRY)/container-toolkit
-endif
-
-VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
-IMAGE_VERSION := $(VERSION)
-
-IMAGE_TAG ?= $(VERSION)-$(DIST)
-IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
-
-OUT_IMAGE_NAME ?= $(IMAGE_NAME)
-OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
-OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
-OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
-
-##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu18.04
-DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
-
-META_TARGETS := packaging
-
-BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
-PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
-
-.PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
-
-ifneq ($(BUILD_MULTI_ARCH_IMAGES),true)
-include $(CURDIR)/build/container/native-only.mk
-else
-include $(CURDIR)/build/container/multi-arch.mk
-endif
-
-# For the default push target we also push a short tag equal to the version.
-# We skip this for the development release
-DEVEL_RELEASE_IMAGE_VERSION ?= devel
-PUSH_MULTIPLE_TAGS ?= true
-ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
-PUSH_MULTIPLE_TAGS = false
-endif
-ifeq ($(PUSH_MULTIPLE_TAGS),true)
-push-$(DEFAULT_PUSH_TARGET): push-short
-endif
-
-push-%: DIST = $(*)
-push-short: DIST = $(DEFAULT_PUSH_TARGET)
-
-build-%: DIST = $(*)
-build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
-
-ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
-
-# Use a generic build target to build the relevant images
-$(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
-	DOCKER_BUILDKIT=1 \
-		$(DOCKER) $(BUILDX) build --pull \
-		$(DOCKER_BUILD_OPTIONS) \
-		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
-		--tag $(IMAGE) \
-		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
-		--build-arg BASE_DIST="$(BASE_DIST)" \
-		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
-		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-		--build-arg LIBNVIDIA_CONTAINER0_VERSION="$(LIBNVIDIA_CONTAINER0_DEPENDENCY)" \
-		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
-		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
-		--build-arg VERSION="$(VERSION)" \
-		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
-		-f $(DOCKERFILE) \
-		$(CURDIR)
-
-
-build-ubuntu%: BASE_DIST = $(*)
-build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
-build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
-build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
-
-build-ubi8: BASE_DIST := ubi8
-build-ubi8: DOCKERFILE_SUFFIX := centos
-build-ubi8: PACKAGE_DIST = centos8
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
-
-build-centos%: BASE_DIST = $(*)
-build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: PACKAGE_DIST = $(BASE_DIST)
-build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
-
-build-packaging: BASE_DIST := ubuntu20.04
-build-packaging: DOCKERFILE_SUFFIX := packaging
-build-packaging: PACKAGE_ARCH := amd64
-build-packaging: PACKAGE_DIST = all
-build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
-
-# Test targets
-test-%: DIST = $(*)
-
-TEST_CASES ?= toolkit docker crio containerd
-$(TEST_TARGETS): test-%:
-	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \
-		$(CURDIR)/shared-$(*) \
-		$(IMAGE) \
-			--no-cleanup-on-error
-
-.PHONY: test-packaging
-test-packaging: DIST = packaging
-test-packaging:
-	@echo "Testing package image contents"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
-	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

cmd/nvidia-cdi-hook/README.md

@@ -0,0 +1,31 @@
+# NVIDIA CDI Hook
+
+The CLI `nvidia-cdi-hook` provides container device runtime hook capabilities when
+called by a container runtime, as specific in a
+[Container Device Interface](https://tags.cncf.io/container-device-interface/blob/main/SPEC.md)
+file.
+
+## Generating a CDI
+
+The CDI itself is created for an NVIDIA-capable device using the
+[`nvidia-ctk cdi generate`](../nvidia-ctk/) command.
+
+When `nvidia-ctk cdi generate` is run, the CDI specification is generated as a yaml file.
+The CDI specification provides instructions for a container runtime to set up devices, files and
+other resources for the container prior to starting it. Those instructions
+may include executing command-line tools to prepare the filesystem. The execution
+of such command-line tools is called a hook.
+
+`nvidia-cdi-hook` is the CLI tool that is expected to be called by the container runtime,
+when specified by the CDI file.
+
+See the [`nvidia-ctk` documentation](../nvidia-ctk/README.md) for more information
+on generating a CDI file.
+
+## Functionality
+
+The `nvidia-cdi-hook` CLI provides the following functionality:
+
+* `chmod` - Change the permissions of a file or directory inside the directory path to be mounted into a container.
+* `create-symlinks` - Create symlinks inside the directory path to be mounted into a container.
+* `update-ldcache` - Update the dynamic linker cache inside the directory path to be mounted into a container.

cmd/nvidia-cdi-hook/chmod/chmod.go

@@ -0,0 +1,160 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	paths         cli.StringSlice
+	modeStr       string
+	mode          fs.FileMode
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specify a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.modeStr,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.modeStr) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	modeInt, err := strconv.ParseUint(cfg.modeStr, 8, 32)
+	if err != nil {
+		return fmt.Errorf("failed to parse mode as octal: %v", err)
+	}
+	cfg.mode = fs.FileMode(modeInt)
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value(), cfg.mode)
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	for _, path := range paths {
+		err = os.Chmod(path, cfg.mode)
+		// in some cases this is not an issue (e.g. whole /dev mounted), see #143
+		if errors.Is(err, fs.ErrPermission) {
+			m.logger.Debugf("Ignoring permission error with chmod: %v", err)
+			err = nil
+		}
+	}
+
+	return err
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string, desiredMode fs.FileMode) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		path := filepath.Join(root, f)
+		stat, err := os.Stat(path)
+		if err != nil {
+			m.logger.Debugf("Skipping path %q: %v", path, err)
+			continue
+		}
+		if (stat.Mode()&(fs.ModePerm|fs.ModeSetuid|fs.ModeSetgid|fs.ModeSticky))^desiredMode == 0 {
+			m.logger.Debugf("Skipping path %q: already desired mode", path)
+			continue
+		}
+		pathsInRoot = append(pathsInRoot, path)
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-cdi-hook/commands/commands.go

@@ -0,0 +1,55 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package commands
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
+	createsonamesymlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-soname-symlinks"
+	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat"
+	disabledevicenodemodification "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/disable-device-node-modification"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+// New creates the commands associated with supported CDI hooks.
+// These are shared by the nvidia-cdi-hook and nvidia-ctk hook commands.
+func New(logger logger.Interface) []*cli.Command {
+	return []*cli.Command{
+		ldcache.NewCommand(logger),
+		symlinks.NewCommand(logger),
+		chmod.NewCommand(logger),
+		cudacompat.NewCommand(logger),
+		createsonamesymlinks.NewCommand(logger),
+		disabledevicenodemodification.NewCommand(logger),
+	}
+}
+
+// IssueUnsupportedHookWarning logs a warning that no hook or an unsupported
+// hook has been specified.
+// This happens if a subcommand is provided that does not match one of the
+// subcommands that has been explicitly specified.
+func IssueUnsupportedHookWarning(logger logger.Interface, c *cli.Context) {
+	args := c.Args().Slice()
+	if len(args) == 0 {
+		logger.Warningf("No CDI hook specified")
+	} else {
+		logger.Warningf("Unsupported CDI hook: %v", args[0])
+	}
+}

cmd/nvidia-cdi-hook/create-soname-symlinks/soname-symlinks.go

@@ -0,0 +1,166 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package create_soname_symlinks
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/moby/sys/reexec"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldconfig"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	reexecUpdateLdCacheCommandName = "reexec-create-soname-symlinks"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	folders       cli.StringSlice
+	ldconfigPath  string
+	containerSpec string
+}
+
+func init() {
+	reexec.Register(reexecUpdateLdCacheCommandName, createSonameSymlinksHandler)
+	if reexec.Init() {
+		os.Exit(0)
+	}
+}
+
+// NewCommand constructs an create-soname-symlinks command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the create-soname-symlinks command
+func (m command) build() *cli.Command {
+	cfg := options{}
+
+	// Create the 'create-soname-symlinks' command
+	c := cli.Command{
+		Name:  "create-soname-symlinks",
+		Usage: "Create soname symlinks libraries in specified directories",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "folder",
+			Usage:       "Specify a directory to generate soname symlinks in. Can be specified multiple times",
+			Destination: &cfg.folders,
+		},
+		&cli.StringFlag{
+			Name:        "ldconfig-path",
+			Usage:       "Specify the path to ldconfig on the host",
+			Destination: &cfg.ldconfigPath,
+			Value:       "/sbin/ldconfig",
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *options) error {
+	if cfg.ldconfigPath == "" {
+		return errors.New("ldconfig-path must be specified")
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *options) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRootDir, err := s.GetContainerRoot()
+	if err != nil || containerRootDir == "" || containerRootDir == "/" {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	cmd, err := ldconfig.NewRunner(
+		reexecUpdateLdCacheCommandName,
+		cfg.ldconfigPath,
+		containerRootDir,
+		cfg.folders.Value()...,
+	)
+	if err != nil {
+		return err
+	}
+
+	return cmd.Run()
+}
+
+// createSonameSymlinksHandler wraps createSonameSymlinks with error handling.
+func createSonameSymlinksHandler() {
+	if err := createSonameSymlinks(os.Args); err != nil {
+		log.Printf("Error updating ldcache: %v", err)
+		os.Exit(1)
+	}
+}
+
+// createSonameSymlinks ensures that soname symlinks are created in the
+// specified directories.
+// It is invoked from a reexec'd handler and provides namespace isolation for
+// the operations performed by this hook. At the point where this is invoked,
+// we are in a new mount namespace that is cloned from the parent.
+//
+// args[0] is the reexec initializer function name
+// args[1] is the path of the ldconfig binary on the host
+// args[2] is the container root directory
+// The remaining args are directories where soname symlinks need to be created.
+func createSonameSymlinks(args []string) error {
+	if len(args) < 3 {
+		return fmt.Errorf("incorrect arguments: %v", args)
+	}
+	hostLdconfigPath := args[1]
+	containerRootDirPath := args[2]
+
+	ldconfig, err := ldconfig.New(
+		hostLdconfigPath,
+		containerRootDirPath,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to construct ldconfig runner: %w", err)
+	}
+
+	return ldconfig.CreateSonameSymlinks(args[3:]...)
+}

cmd/nvidia-cdi-hook/create-symlinks/create-symlinks.go

@@ -0,0 +1,172 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package symlinks
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/moby/sys/symlink"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	links         cli.StringSlice
+	containerSpec string
+}
+
+// NewCommand constructs a hook command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the create-symlink command.
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	c := cli.Command{
+		Name:  "create-symlinks",
+		Usage: "A hook to create symlinks in the container.",
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "link",
+			Usage:       "Specify a specific link to create. The link is specified as target::link. If the link exists in the container root, it is removed.",
+			Destination: &cfg.links,
+		},
+		// The following flags are testing-only flags.
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN. This is only intended for testing.",
+			Destination: &cfg.containerSpec,
+			Hidden:      true,
+		},
+	}
+
+	return &c
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	created := make(map[string]bool)
+	for _, l := range cfg.links.Value() {
+		if created[l] {
+			m.logger.Debugf("Link %v already processed", l)
+			continue
+		}
+		parts := strings.Split(l, "::")
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid symlink specification %v", l)
+		}
+
+		err := m.createLink(containerRoot, parts[0], parts[1])
+		if err != nil {
+			return fmt.Errorf("failed to create link %v: %w", parts, err)
+		}
+		created[l] = true
+	}
+	return nil
+}
+
+// createLink creates a symbolic link in the specified container root.
+// This is equivalent to:
+//
+//	chroot {{ .containerRoot }} ln -f -s {{ .target }} {{ .link }}
+//
+// If the specified link already exists and points to the same target, this
+// operation is a no-op.
+// If a file exists at the link path or the link points to a different target
+// this file is removed before creating the link.
+//
+// Note that if the link path resolves to an absolute path oudside of the
+// specified root, this is treated as an absolute path in this root.
+func (m command) createLink(containerRoot string, targetPath string, link string) error {
+	linkPath := filepath.Join(containerRoot, link)
+
+	exists, err := linkExists(targetPath, linkPath)
+	if err != nil {
+		return fmt.Errorf("failed to check if link exists: %w", err)
+	}
+	if exists {
+		m.logger.Debugf("Link %s already exists", linkPath)
+		return nil
+	}
+
+	// We resolve the parent of the symlink that we're creating in the container root.
+	// If we resolve the full link path, an existing link at the location itself
+	// is also resolved here and we are unable to force create the link.
+	resolvedLinkParent, err := symlink.FollowSymlinkInScope(filepath.Dir(linkPath), containerRoot)
+	if err != nil {
+		return fmt.Errorf("failed to follow path for link %v relative to %v: %w", link, containerRoot, err)
+	}
+	resolvedLinkPath := filepath.Join(resolvedLinkParent, filepath.Base(linkPath))
+
+	m.logger.Infof("Symlinking %v to %v", resolvedLinkPath, targetPath)
+	err = os.MkdirAll(filepath.Dir(resolvedLinkPath), 0755)
+	if err != nil {
+		return fmt.Errorf("failed to create directory: %v", err)
+	}
+	err = symlinks.ForceCreate(targetPath, resolvedLinkPath)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+
+	return nil
+}
+
+// linkExists checks whether the specified link exists.
+// A link exists if the path exists, is a symlink, and points to the specified target.
+func linkExists(target string, link string) (bool, error) {
+	currentTarget, err := symlinks.Resolve(link)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("failed to resolve existing symlink %s: %w", link, err)
+	}
+	if currentTarget == target {
+		return true, nil
+	}
+	return false, nil
+}

cmd/nvidia-cdi-hook/create-symlinks/create-symlinks_test.go

@@ -0,0 +1,297 @@
+package symlinks
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+)
+
+func TestLinkExist(t *testing.T) {
+	tmpDir := t.TempDir()
+	require.NoError(
+		t,
+		makeFs(tmpDir,
+			dirOrLink{path: "/a/b/c", target: "d"},
+			dirOrLink{path: "/a/b/e", target: "/a/b/f"},
+		),
+	)
+
+	exists, err := linkExists("d", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.True(t, exists)
+
+	exists, err = linkExists("/a/b/f", filepath.Join(tmpDir, "/a/b/e"))
+	require.NoError(t, err)
+	require.True(t, exists)
+
+	exists, err = linkExists("different-target", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.False(t, exists)
+
+	exists, err = linkExists("/a/b/d", filepath.Join(tmpDir, "/a/b/c"))
+	require.NoError(t, err)
+	require.False(t, exists)
+
+	exists, err = linkExists("foo", filepath.Join(tmpDir, "/a/b/does-not-exist"))
+	require.NoError(t, err)
+	require.False(t, exists)
+}
+
+func TestCreateLink(t *testing.T) {
+	type link struct {
+		path   string
+		target string
+	}
+	type expectedLink struct {
+		link
+		err error
+	}
+
+	testCases := []struct {
+		description         string
+		containerContents   []dirOrLink
+		link                link
+		expectedCreateError error
+		expectedLinks       []expectedLink
+	}{
+		{
+			description: "link to / resolves to container root",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "libfoo.so.1",
+					},
+				},
+			},
+		},
+		{
+			description: "link to / resolves to container root; parent relative link",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "../libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "../libfoo.so.1",
+					},
+				},
+			},
+		},
+		{
+			description: "link to / resolves to container root; absolute link",
+			containerContents: []dirOrLink{
+				{path: "/lib/foo", target: "/"},
+			},
+			link: link{
+				path:   "/lib/foo/libfoo.so",
+				target: "/a-path-in-container/foo/libfoo.so.1",
+			},
+			expectedLinks: []expectedLink{
+				{
+					link: link{
+						path:   "{{ .containerRoot }}/libfoo.so",
+						target: "/a-path-in-container/foo/libfoo.so.1",
+					},
+				},
+				{
+					// We also check that the target is NOT created.
+					link: link{
+						path: "{{ .containerRoot }}/a-path-in-container/foo/libfoo.so.1",
+					},
+					err: os.ErrNotExist,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			hostRoot := filepath.Join(tmpDir, "/host-root/")
+			containerRoot := filepath.Join(tmpDir, "/container-root")
+
+			require.NoError(t, makeFs(hostRoot))
+			require.NoError(t, makeFs(containerRoot, tc.containerContents...))
+
+			// nvidia-cdi-hook create-symlinks --link linkSpec
+			err := getTestCommand().createLink(containerRoot, tc.link.target, tc.link.path)
+			// TODO: We may be able to replace this with require.ErrorIs.
+			if tc.expectedCreateError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			for _, expectedLink := range tc.expectedLinks {
+				path := strings.ReplaceAll(expectedLink.path, "{{ .containerRoot }}", containerRoot)
+				path = strings.ReplaceAll(path, "{{ .hostRoot }}", hostRoot)
+				if expectedLink.target != "" {
+					target, err := symlinks.Resolve(path)
+					require.ErrorIs(t, err, expectedLink.err)
+					require.Equal(t, expectedLink.target, target)
+				} else {
+					_, err := os.Stat(path)
+					require.ErrorIs(t, err, expectedLink.err)
+				}
+			}
+		})
+	}
+}
+
+func TestCreateLinkRelativePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root/")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t, makeFs(hostRoot))
+	require.NoError(t, makeFs(containerRoot, dirOrLink{path: "/lib/"}))
+
+	// nvidia-cdi-hook create-symlinks --link libfoo.so.1::/lib/libfoo.so
+	err := getTestCommand().createLink(containerRoot, "libfoo.so.1", "/lib/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "libfoo.so.1", target)
+}
+
+func TestCreateLinkAbsolutePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root/")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t, makeFs(hostRoot))
+	require.NoError(t, makeFs(containerRoot, dirOrLink{path: "/lib/"}))
+
+	// nvidia-cdi-hook create-symlinks --link /lib/libfoo.so.1::/lib/libfoo.so
+	err := getTestCommand().createLink(containerRoot, "/lib/libfoo.so.1", "/lib/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "/lib/libfoo.so.1", target)
+}
+
+func TestCreateLinkAlreadyExists(t *testing.T) {
+	testCases := []struct {
+		description       string
+		containerContents []dirOrLink
+		shouldExist       []string
+	}{
+		{
+			description:       "link already exists with correct target",
+			containerContents: []dirOrLink{{path: "/lib/libfoo.so", target: "libfoo.so.1"}},
+			shouldExist:       []string{},
+		},
+		{
+			description:       "link already exists with different target",
+			containerContents: []dirOrLink{{path: "/lib/libfoo.so", target: "different-target"}, {path: "different-target"}},
+			shouldExist:       []string{"{{ .containerRoot }}/different-target"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			hostRoot := filepath.Join(tmpDir, "/host-root/")
+			containerRoot := filepath.Join(tmpDir, "/container-root")
+			require.NoError(t, makeFs(hostRoot))
+			require.NoError(t, makeFs(containerRoot, tc.containerContents...))
+
+			// nvidia-cdi-hook create-symlinks --link libfoo.so.1::/lib/libfoo.so
+			err := getTestCommand().createLink(containerRoot, "libfoo.so.1", "/lib/libfoo.so")
+			require.NoError(t, err)
+			target, err := symlinks.Resolve(filepath.Join(containerRoot, "lib/libfoo.so"))
+			require.NoError(t, err)
+			require.Equal(t, "libfoo.so.1", target)
+
+			for _, p := range tc.shouldExist {
+				require.DirExists(t, strings.ReplaceAll(p, "{{ .containerRoot }}", containerRoot))
+			}
+		})
+	}
+}
+
+func TestCreateLinkOutOfBounds(t *testing.T) {
+	tmpDir := t.TempDir()
+	hostRoot := filepath.Join(tmpDir, "/host-root")
+	containerRoot := filepath.Join(tmpDir, "/container-root")
+
+	require.NoError(t,
+		makeFs(hostRoot,
+			dirOrLink{path: "libfoo.so"},
+		),
+	)
+	require.NoError(t,
+		makeFs(containerRoot,
+			dirOrLink{path: "/lib"},
+			dirOrLink{path: "/lib/foo", target: hostRoot},
+		),
+	)
+
+	path, err := symlinks.Resolve(filepath.Join(containerRoot, "/lib/foo"))
+	require.NoError(t, err)
+	require.Equal(t, hostRoot, path)
+
+	// nvidia-cdi-hook create-symlinks --link ../libfoo.so.1::/lib/foo/libfoo.so
+	_ = getTestCommand().createLink(containerRoot, "../libfoo.so.1", "/lib/foo/libfoo.so")
+	require.NoError(t, err)
+
+	target, err := symlinks.Resolve(filepath.Join(containerRoot, hostRoot, "libfoo.so"))
+	require.NoError(t, err)
+	require.Equal(t, "../libfoo.so.1", target)
+
+	require.DirExists(t, filepath.Join(hostRoot, "libfoo.so"))
+}
+
+type dirOrLink struct {
+	path   string
+	target string
+}
+
+func makeFs(tmpdir string, fs ...dirOrLink) error {
+	if err := os.MkdirAll(tmpdir, 0o755); err != nil {
+		return err
+	}
+	for _, s := range fs {
+		s.path = filepath.Join(tmpdir, s.path)
+		if s.target == "" {
+			_ = os.MkdirAll(s.path, 0o755)
+			continue
+		}
+		if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
+			return err
+		}
+		if err := os.Symlink(s.target, s.path); err != nil && !os.IsExist(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// getTestCommand creates a command for running tests against.
+func getTestCommand() *command {
+	logger, _ := testlog.NewNullLogger()
+	return &command{
+		logger: logger,
+	}
+}

cmd/nvidia-cdi-hook/cudacompat/container-root.go

@@ -0,0 +1,76 @@
+/**
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cudacompat
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/moby/sys/symlink"
+)
+
+// A containerRoot represents the root filesystem of a container.
+type containerRoot string
+
+// hasPath checks whether the specified path exists in the root.
+func (r containerRoot) hasPath(path string) bool {
+	resolved, err := r.resolve(path)
+	if err != nil {
+		return false
+	}
+	if _, err := os.Stat(resolved); err != nil && os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+// globFiles matches the specified pattern in the root.
+// The files that match must be regular files.
+func (r containerRoot) globFiles(pattern string) ([]string, error) {
+	patternPath, err := r.resolve(pattern)
+	if err != nil {
+		return nil, err
+	}
+	matches, err := filepath.Glob(patternPath)
+	if err != nil {
+		return nil, err
+	}
+	var files []string
+	for _, match := range matches {
+		info, err := os.Lstat(match)
+		if err != nil {
+			return nil, err
+		}
+		// Ignore symlinks.
+		if info.Mode()&os.ModeSymlink != 0 {
+			continue
+		}
+		// Ignore directories.
+		if info.IsDir() {
+			continue
+		}
+		files = append(files, match)
+	}
+	return files, nil
+}
+
+// resolve returns the absolute path including root path.
+// Symlinks are resolved, but are guaranteed to resolve in the root.
+func (r containerRoot) resolve(path string) (string, error) {
+	absolute := filepath.Clean(filepath.Join(string(r), path))
+	return symlink.FollowSymlinkInScope(absolute, string(r))
+}

cmd/nvidia-cdi-hook/cudacompat/cudacompat.go

@@ -0,0 +1,221 @@
+/**
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cudacompat
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	cudaCompatPath = "/usr/local/cuda/compat"
+	// cudaCompatLdsoconfdFilenamePattern specifies the pattern for the filename
+	// in ld.so.conf.d that includes a reference to the CUDA compat path.
+	// The 00-compat prefix is chosen to ensure that these libraries have a
+	// higher precedence than other libraries on the system.
+	cudaCompatLdsoconfdFilenamePattern = "00-compat-*.conf"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	hostDriverVersion string
+	containerSpec     string
+}
+
+// NewCommand constructs a cuda-compat command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the enable-cuda-compat command
+func (m command) build() *cli.Command {
+	cfg := options{}
+
+	// Create the 'enable-cuda-compat' command
+	c := cli.Command{
+		Name:  "enable-cuda-compat",
+		Usage: "This hook ensures that the folder containing the CUDA compat libraries is added to the ldconfig search path if required.",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "host-driver-version",
+			Usage:       "Specify the host driver version. If the CUDA compat libraries detected in the container do not have a higher MAJOR version, the hook is a no-op.",
+			Destination: &cfg.hostDriverVersion,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Hidden:      true,
+			Category:    "testing-only",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(_ *cli.Context, cfg *options) error {
+	return nil
+}
+
+func (m command) run(_ *cli.Context, cfg *options) error {
+	if cfg.hostDriverVersion == "" {
+		return nil
+	}
+
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %w", err)
+	}
+
+	containerRootDir, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %w", err)
+	}
+
+	containerForwardCompatDir, err := m.getContainerForwardCompatDir(containerRoot(containerRootDir), cfg.hostDriverVersion)
+	if err != nil {
+		return fmt.Errorf("failed to get container forward compat directory: %w", err)
+	}
+	if containerForwardCompatDir == "" {
+		return nil
+	}
+
+	return m.createLdsoconfdFile(containerRoot(containerRootDir), cudaCompatLdsoconfdFilenamePattern, containerForwardCompatDir)
+}
+
+func (m command) getContainerForwardCompatDir(containerRoot containerRoot, hostDriverVersion string) (string, error) {
+	if hostDriverVersion == "" {
+		m.logger.Debugf("Host driver version not specified")
+		return "", nil
+	}
+	if !containerRoot.hasPath(cudaCompatPath) {
+		m.logger.Debugf("No CUDA forward compatibility libraries directory in container")
+		return "", nil
+	}
+	if !containerRoot.hasPath("/etc/ld.so.cache") {
+		m.logger.Debugf("The container does not have an LDCache")
+		return "", nil
+	}
+
+	libs, err := containerRoot.globFiles(filepath.Join(cudaCompatPath, "libcuda.so.*.*"))
+	if err != nil {
+		m.logger.Warningf("Failed to find CUDA compat library: %w", err)
+		return "", nil
+	}
+
+	if len(libs) == 0 {
+		m.logger.Debugf("No CUDA forward compatibility libraries container")
+		return "", nil
+	}
+
+	if len(libs) != 1 {
+		m.logger.Warningf("Unexpected number of CUDA compat libraries in container: %v", libs)
+		return "", nil
+	}
+
+	compatDriverVersion := strings.TrimPrefix(filepath.Base(libs[0]), "libcuda.so.")
+	compatMajor, err := extractMajorVersion(compatDriverVersion)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract major version from %q: %v", compatDriverVersion, err)
+	}
+
+	driverMajor, err := extractMajorVersion(hostDriverVersion)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract major version from %q: %v", hostDriverVersion, err)
+	}
+
+	if driverMajor >= compatMajor {
+		m.logger.Debugf("Compat major version is not greater than the host driver major version (%v >= %v)", hostDriverVersion, compatDriverVersion)
+		return "", nil
+	}
+
+	resolvedCompatDir := strings.TrimPrefix(filepath.Dir(libs[0]), string(containerRoot))
+	return resolvedCompatDir, nil
+}
+
+// createLdsoconfdFile creates a file at /etc/ld.so.conf.d/ in the specified root.
+// The file is created at /etc/ld.so.conf.d/{{ .pattern }} using `CreateTemp` and
+// contains the specified directories on each line.
+func (m command) createLdsoconfdFile(in containerRoot, pattern string, dirs ...string) error {
+	if len(dirs) == 0 {
+		m.logger.Debugf("No directories to add to /etc/ld.so.conf")
+		return nil
+	}
+
+	ldsoconfdDir, err := in.resolve("/etc/ld.so.conf.d")
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(ldsoconfdDir, 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %w", err)
+	}
+
+	configFile, err := os.CreateTemp(ldsoconfdDir, pattern)
+	if err != nil {
+		return fmt.Errorf("failed to create config file: %w", err)
+	}
+	defer configFile.Close()
+
+	m.logger.Debugf("Adding directories %v to %v", dirs, configFile.Name())
+
+	added := make(map[string]bool)
+	for _, dir := range dirs {
+		if added[dir] {
+			continue
+		}
+		_, err = fmt.Fprintf(configFile, "%s\n", dir)
+		if err != nil {
+			return fmt.Errorf("failed to update config file: %w", err)
+		}
+		added[dir] = true
+	}
+
+	// The created file needs to be world readable for the cases where the container is run as a non-root user.
+	if err := configFile.Chmod(0644); err != nil {
+		return fmt.Errorf("failed to chmod config file: %w", err)
+	}
+
+	return nil
+}
+
+// extractMajorVersion parses a version string and returns the major version as an int.
+func extractMajorVersion(version string) (int, error) {
+	majorString := strings.SplitN(version, ".", 2)[0]
+	return strconv.Atoi(majorString)
+}

cmd/nvidia-cdi-hook/cudacompat/cudacompat_test.go

@@ -0,0 +1,182 @@
+/*
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package cudacompat
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompatLibs(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description                       string
+		contents                          map[string]string
+		hostDriverVersion                 string
+		expectedContainerForwardCompatDir string
+	}{
+		{
+			description:       "empty root",
+			hostDriverVersion: "222.55.66",
+		},
+		{
+			description: "compat lib is newer; no ldcache",
+			contents: map[string]string{
+				"/usr/local/cuda/compat/libcuda.so.333.88.99": "",
+			},
+			hostDriverVersion: "222.55.66",
+		},
+		{
+			description: "compat lib is newer; ldcache",
+			contents: map[string]string{
+				"/etc/ld.so.cache": "",
+				"/usr/local/cuda/compat/libcuda.so.333.88.99": "",
+			},
+			hostDriverVersion:                 "222.55.66",
+			expectedContainerForwardCompatDir: "/usr/local/cuda/compat",
+		},
+		{
+			description: "compat lib is older; ldcache",
+			contents: map[string]string{
+				"/etc/ld.so.cache": "",
+				"/usr/local/cuda/compat/libcuda.so.111.88.99": "",
+			},
+			hostDriverVersion:                 "222.55.66",
+			expectedContainerForwardCompatDir: "",
+		},
+		{
+			description: "compat lib has same major version; ldcache",
+			contents: map[string]string{
+				"/etc/ld.so.cache": "",
+				"/usr/local/cuda/compat/libcuda.so.222.88.99": "",
+			},
+			hostDriverVersion:                 "222.55.66",
+			expectedContainerForwardCompatDir: "",
+		},
+		{
+			description: "numeric comparison is used; ldcache",
+			contents: map[string]string{
+				"/etc/ld.so.cache": "",
+				"/usr/local/cuda/compat/libcuda.so.222.88.99": "",
+			},
+			hostDriverVersion:                 "99.55.66",
+			expectedContainerForwardCompatDir: "/usr/local/cuda/compat",
+		},
+		{
+			description: "driver version empty; ldcache",
+			contents: map[string]string{
+				"/etc/ld.so.cache": "",
+				"/usr/local/cuda/compat/libcuda.so.222.88.99": "",
+			},
+			hostDriverVersion: "",
+		},
+		{
+			description: "symlinks are followed",
+			contents: map[string]string{
+				"/etc/ld.so.cache": "",
+				"/etc/alternatives/cuda/compat/libcuda.so.333.88.99": "",
+				"/usr/local/cuda": "symlink=/etc/alternatives/cuda",
+			},
+			hostDriverVersion:                 "222.55.66",
+			expectedContainerForwardCompatDir: "/etc/alternatives/cuda/compat",
+		},
+		{
+			description: "symlinks stay in container",
+			contents: map[string]string{
+				"/etc/ld.so.cache":             "",
+				"/compat/libcuda.so.333.88.99": "",
+				"/usr/local/cuda":              "symlink=../../../../../../",
+			},
+			hostDriverVersion:                 "222.55.66",
+			expectedContainerForwardCompatDir: "/compat",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			containerRootDir := t.TempDir()
+			for name, contents := range tc.contents {
+				target := filepath.Join(containerRootDir, name)
+				require.NoError(t, os.MkdirAll(filepath.Dir(target), 0755))
+
+				if strings.HasPrefix(contents, "symlink=") {
+					require.NoError(t, os.Symlink(strings.TrimPrefix(contents, "symlink="), target))
+					continue
+				}
+
+				require.NoError(t, os.WriteFile(target, []byte(contents), 0600))
+			}
+
+			c := command{
+				logger: logger,
+			}
+			containerForwardCompatDir, err := c.getContainerForwardCompatDir(containerRoot(containerRootDir), tc.hostDriverVersion)
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expectedContainerForwardCompatDir, containerForwardCompatDir)
+		})
+	}
+}
+
+func TestUpdateLdconfig(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	testCases := []struct {
+		description      string
+		folders          []string
+		expectedContents string
+	}{
+		{
+			description: "no folders; have no contents",
+		},
+		{
+			description:      "single folder is added",
+			folders:          []string{"/usr/local/cuda/compat"},
+			expectedContents: "/usr/local/cuda/compat\n",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			containerRootDir := t.TempDir()
+			c := command{
+				logger: logger,
+			}
+			err := c.createLdsoconfdFile(containerRoot(containerRootDir), cudaCompatLdsoconfdFilenamePattern, tc.folders...)
+			require.NoError(t, err)
+
+			matches, err := filepath.Glob(filepath.Join(containerRootDir, "/etc/ld.so.conf.d/00-compat-*.conf"))
+			require.NoError(t, err)
+
+			if tc.expectedContents == "" {
+				require.Empty(t, matches)
+				return
+			}
+
+			require.Len(t, matches, 1)
+			contents, err := os.ReadFile(matches[0])
+			require.NoError(t, err)
+
+			require.EqualValues(t, tc.expectedContents, string(contents))
+		})
+	}
+
+}

cmd/nvidia-cdi-hook/disable-device-node-modification/disable-device-node-modification.go

@@ -0,0 +1,144 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package disabledevicenodemodification
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	nvidiaDriverParamsPath = "/proc/driver/nvidia/params"
+)
+
+type options struct {
+	containerSpec string
+}
+
+// NewCommand constructs an disable-device-node-modification subcommand with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	cfg := options{}
+
+	c := cli.Command{
+		Name:  "disable-device-node-modification",
+		Usage: "Ensure that the /proc/driver/nvidia/params file present in the container does not allow device node modifications.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Hidden:      true,
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *options) error {
+	return nil
+}
+
+func run(_ *cli.Context, cfg *options) error {
+	modifiedParamsFileContents, err := getModifiedNVIDIAParamsContents()
+	if err != nil {
+		return fmt.Errorf("failed to get modified params file contents: %w", err)
+	}
+	if len(modifiedParamsFileContents) == 0 {
+		return nil
+	}
+
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %w", err)
+	}
+
+	containerRootDirPath, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %w", err)
+	}
+
+	return createParamsFileInContainer(containerRootDirPath, modifiedParamsFileContents)
+}
+
+func getModifiedNVIDIAParamsContents() ([]byte, error) {
+	hostNvidiaParamsFile, err := os.Open(nvidiaDriverParamsPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to load params file: %w", err)
+	}
+	defer hostNvidiaParamsFile.Close()
+
+	modifiedContents, err := getModifiedParamsFileContentsFromReader(hostNvidiaParamsFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get modfied params file contents: %w", err)
+	}
+
+	return modifiedContents, nil
+}
+
+// getModifiedParamsFileContentsFromReader returns the contents of a modified params file from the specified reader.
+func getModifiedParamsFileContentsFromReader(r io.Reader) ([]byte, error) {
+	var modified bytes.Buffer
+	scanner := bufio.NewScanner(r)
+
+	var requiresModification bool
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "ModifyDeviceFiles: ") {
+			if line == "ModifyDeviceFiles: 0" {
+				return nil, nil
+			}
+			if line == "ModifyDeviceFiles: 1" {
+				line = "ModifyDeviceFiles: 0"
+				requiresModification = true
+			}
+		}
+		if _, err := modified.WriteString(line + "\n"); err != nil {
+			return nil, fmt.Errorf("failed to create output buffer: %w", err)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("failed to read params file: %w", err)
+	}
+
+	if !requiresModification {
+		return nil, nil
+	}
+
+	return modified.Bytes(), nil
+}

cmd/nvidia-cdi-hook/disable-device-node-modification/disable-device-node-modification_test.go

@@ -0,0 +1,91 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package disabledevicenodemodification
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetModifiedParamsFileContentsFromReader(t *testing.T) {
+	testCases := map[string]struct {
+		contents         []byte
+		expectedError    error
+		expectedContents []byte
+	}{
+		"no contents": {
+			contents:         nil,
+			expectedError:    nil,
+			expectedContents: nil,
+		},
+		"other contents are ignored": {
+			contents: []byte(`# Some other content
+			that we don't care about
+			`),
+			expectedError:    nil,
+			expectedContents: nil,
+		},
+		"already zero requires no modification": {
+			contents:         []byte("ModifyDeviceFiles: 0"),
+			expectedError:    nil,
+			expectedContents: nil,
+		},
+		"leading spaces require no modification": {
+			contents: []byte("  ModifyDeviceFiles: 1"),
+		},
+		"Trailing spaces require no modification": {
+			contents: []byte("ModifyDeviceFiles: 1  "),
+		},
+		"Not 1 require no modification": {
+			contents: []byte("ModifyDeviceFiles: 11"),
+		},
+		"single line requires modification": {
+			contents:         []byte("ModifyDeviceFiles: 1"),
+			expectedError:    nil,
+			expectedContents: []byte("ModifyDeviceFiles: 0\n"),
+		},
+		"single line with trailing newline requires modification": {
+			contents:         []byte("ModifyDeviceFiles: 1\n"),
+			expectedError:    nil,
+			expectedContents: []byte("ModifyDeviceFiles: 0\n"),
+		},
+		"other content is maintained": {
+			contents: []byte(`ModifyDeviceFiles: 1
+			other content
+			that
+			is maintained`),
+			expectedError: nil,
+			expectedContents: []byte(`ModifyDeviceFiles: 0
+			other content
+			that
+			is maintained
+`),
+		},
+	}
+
+	for description, tc := range testCases {
+		t.Run(description, func(t *testing.T) {
+			contents, err := getModifiedParamsFileContentsFromReader(bytes.NewReader(tc.contents))
+			require.EqualValues(t, tc.expectedError, err)
+			require.EqualValues(t, string(tc.expectedContents), string(contents))
+		})
+	}
+
+}

cmd/nvidia-cdi-hook/disable-device-node-modification/params_linux.go

@@ -0,0 +1,63 @@
+//go:build linux
+
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package disabledevicenodemodification
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runc/libcontainer/utils"
+	"golang.org/x/sys/unix"
+)
+
+func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
+	tmpRoot, err := os.MkdirTemp("", "nvct-empty-dir*")
+	if err != nil {
+		return fmt.Errorf("failed to create temp root: %w", err)
+	}
+
+	if err := createTmpFs(tmpRoot, len(contents)); err != nil {
+		return fmt.Errorf("failed to create tmpfs mount for params file: %w", err)
+	}
+
+	modifiedParamsFile, err := os.OpenFile(filepath.Join(tmpRoot, "nvct-params"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0444)
+	if err != nil {
+		return fmt.Errorf("failed to open modified params file: %w", err)
+	}
+	defer modifiedParamsFile.Close()
+
+	if _, err := modifiedParamsFile.Write(contents); err != nil {
+		return fmt.Errorf("failed to write temporary params file: %w", err)
+	}
+
+	err = utils.WithProcfd(containerRootDirPath, nvidiaDriverParamsPath, func(nvidiaDriverParamsFdPath string) error {
+		return unix.Mount(modifiedParamsFile.Name(), nvidiaDriverParamsFdPath, "", unix.MS_BIND|unix.MS_RDONLY|unix.MS_NODEV|unix.MS_PRIVATE|unix.MS_NOSYMFOLLOW, "")
+	})
+	if err != nil {
+		return fmt.Errorf("failed to mount modified params file: %w", err)
+	}
+
+	return nil
+}
+
+func createTmpFs(target string, size int) error {
+	return unix.Mount("tmpfs", target, "tmpfs", 0, fmt.Sprintf("size=%d", size))
+}

cmd/nvidia-cdi-hook/disable-device-node-modification/params_other.go

@@ -0,0 +1,27 @@
+//go:build !linux
+// +build !linux
+
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package disabledevicenodemodification
+
+import "fmt"
+
+func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
+	return fmt.Errorf("not supported")
+}

cmd/nvidia-cdi-hook/main.go

@@ -0,0 +1,107 @@
+/**
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/commands"
+)
+
+// options defines the options that can be set for the CLI through config files,
+// environment variables, or command line flags
+type options struct {
+	// Debug indicates whether the CLI is started in "debug" mode
+	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
+}
+
+func main() {
+	logger := logrus.New()
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "NVIDIA CDI Hook"
+	c.UseShortOptionHandling = true
+	c.EnableBashCompletion = true
+	c.Usage = "Command to structure files for usage inside a container, called as hooks from a container runtime, defined in a CDI yaml file"
+	c.Version = info.GetVersionString()
+
+	// We set the default action for the `nvidia-cdi-hook` command to issue a
+	// warning and exit with no error.
+	// This means that if an unsupported hook is run, a container will not fail
+	// to launch. An unsupported hook could be the result of a CDI specification
+	// referring to a new hook that is not yet supported by an older NVIDIA
+	// Container Toolkit version or a hook that has been removed in newer
+	// version.
+	c.Action = func(ctx *cli.Context) error {
+		commands.IssueUnsupportedHookWarning(logger, ctx)
+		return nil
+	}
+
+	// Setup the flags for this command
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "debug",
+			Aliases:     []string{"d"},
+			Usage:       "Enable debug-level logging",
+			Destination: &opts.Debug,
+			// TODO: Support for NVIDIA_CDI_DEBUG is deprecated and NVIDIA_CTK_DEBUG should be used instead.
+			EnvVars: []string{"NVIDIA_CTK_DEBUG", "NVIDIA_CDI_DEBUG"},
+		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			// TODO: Support for NVIDIA_CDI_QUIET is deprecated and NVIDIA_CTK_QUIET should be used instead.
+			EnvVars: []string{"NVDIA_CTK_QUIET", "NVIDIA_CDI_QUIET"},
+		},
+	}
+
+	// Set log-level for all subcommands
+	c.Before = func(c *cli.Context) error {
+		logLevel := logrus.InfoLevel
+		if opts.Debug {
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
+		}
+		logger.SetLevel(logLevel)
+		return nil
+	}
+
+	// Define the subcommands
+	c.Commands = commands.New(logger)
+
+	// Run the CLI
+	err := c.Run(os.Args)
+	if err != nil {
+		logger.Errorf("%v", err)
+		os.Exit(1)
+	}
+}

cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go

@@ -0,0 +1,164 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package ldcache
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/moby/sys/reexec"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldconfig"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	reexecUpdateLdCacheCommandName = "reexec-update-ldcache"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	folders       cli.StringSlice
+	ldconfigPath  string
+	containerSpec string
+}
+
+func init() {
+	reexec.Register(reexecUpdateLdCacheCommandName, updateLdCacheHandler)
+	if reexec.Init() {
+		os.Exit(0)
+	}
+}
+
+// NewCommand constructs an update-ldcache command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the update-ldcache command
+func (m command) build() *cli.Command {
+	cfg := options{}
+
+	// Create the 'update-ldcache' command
+	c := cli.Command{
+		Name:  "update-ldcache",
+		Usage: "Update ldcache in a container by running ldconfig",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "folder",
+			Usage:       "Specify a folder to add to /etc/ld.so.conf before updating the ld cache",
+			Destination: &cfg.folders,
+		},
+		&cli.StringFlag{
+			Name:        "ldconfig-path",
+			Usage:       "Specify the path to the ldconfig program",
+			Destination: &cfg.ldconfigPath,
+			Value:       "/sbin/ldconfig",
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *options) error {
+	if cfg.ldconfigPath == "" {
+		return errors.New("ldconfig-path must be specified")
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *options) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRootDir, err := s.GetContainerRoot()
+	if err != nil || containerRootDir == "" || containerRootDir == "/" {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	cmd, err := ldconfig.NewRunner(
+		reexecUpdateLdCacheCommandName,
+		cfg.ldconfigPath,
+		containerRootDir,
+		cfg.folders.Value()...,
+	)
+	if err != nil {
+		return err
+	}
+	return cmd.Run()
+}
+
+// updateLdCacheHandler wraps updateLdCache with error handling.
+func updateLdCacheHandler() {
+	if err := updateLdCache(os.Args); err != nil {
+		log.Printf("Error updating ldcache: %v", err)
+		os.Exit(1)
+	}
+}
+
+// updateLdCache ensures that the ldcache in the container is updated to include
+// libraries that are mounted from the host.
+// It is invoked from a reexec'd handler and provides namespace isolation for
+// the operations performed by this hook. At the point where this is invoked,
+// we are in a new mount namespace that is cloned from the parent.
+//
+// args[0] is the reexec initializer function name
+// args[1] is the path of the ldconfig binary on the host
+// args[2] is the container root directory
+// The remaining args are folders where soname symlinks need to be created.
+func updateLdCache(args []string) error {
+	if len(args) < 3 {
+		return fmt.Errorf("incorrect arguments: %v", args)
+	}
+	hostLdconfigPath := args[1]
+	containerRootDirPath := args[2]
+
+	ldconfig, err := ldconfig.New(
+		hostLdconfigPath,
+		containerRootDirPath,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to construct ldconfig runner: %w", err)
+	}
+
+	return ldconfig.UpdateLDCache(args[3:]...)
+}

cmd/nvidia-container-runtime-hook/capabilities.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"log"
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -0,0 +1,290 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"golang.org/x/mod/semver"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
+
+type nvidiaConfig struct {
+	Devices            []string
+	MigConfigDevices   string
+	MigMonitorDevices  string
+	ImexChannels       []string
+	DriverCapabilities string
+	// Requirements defines the requirements DSL for the container to run.
+	// This is empty if no specific requirements are needed, or if requirements are
+	// explicitly disabled.
+	Requirements []string
+}
+
+type containerConfig struct {
+	Pid    int
+	Rootfs string
+	Image  image.CUDA
+	Nvidia *nvidiaConfig
+}
+
+// Root from OCI runtime spec
+// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L94-L100
+type Root struct {
+	Path string `json:"path"`
+}
+
+// Process from OCI runtime spec
+// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L57
+type Process struct {
+	Env          []string         `json:"env,omitempty"`
+	Capabilities *json.RawMessage `json:"capabilities,omitempty" platform:"linux"`
+}
+
+// LinuxCapabilities from OCI runtime spec
+// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L61
+type LinuxCapabilities struct {
+	Bounding    []string `json:"bounding,omitempty" platform:"linux"`
+	Effective   []string `json:"effective,omitempty" platform:"linux"`
+	Inheritable []string `json:"inheritable,omitempty" platform:"linux"`
+	Permitted   []string `json:"permitted,omitempty" platform:"linux"`
+	Ambient     []string `json:"ambient,omitempty" platform:"linux"`
+}
+
+// Spec from OCI runtime spec
+// We use pointers to structs, similarly to the latest version of runtime-spec:
+// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L5-L28
+type Spec struct {
+	Version *string       `json:"ociVersion"`
+	Process *Process      `json:"process,omitempty"`
+	Root    *Root         `json:"root,omitempty"`
+	Mounts  []specs.Mount `json:"mounts,omitempty"`
+}
+
+// HookState holds state information about the hook
+type HookState struct {
+	Pid int `json:"pid,omitempty"`
+	// After 17.06, runc is using the runtime spec:
+	// github.com/docker/runc/blob/17.06/libcontainer/configs/config.go#L262-L263
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/state.go#L3-L17
+	Bundle string `json:"bundle"`
+	// Before 17.06, runc used a custom struct that didn't conform to the spec:
+	// github.com/docker/runc/blob/17.03.x/libcontainer/configs/config.go#L245-L252
+	BundlePath string `json:"bundlePath"`
+}
+
+func loadSpec(path string) (spec *Spec) {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Panicln("could not open OCI spec:", err)
+	}
+	defer f.Close()
+
+	if err = json.NewDecoder(f).Decode(&spec); err != nil {
+		log.Panicln("could not decode OCI spec:", err)
+	}
+	if spec.Version == nil {
+		log.Panicln("Version is empty in OCI spec")
+	}
+	if spec.Process == nil {
+		log.Panicln("Process is empty in OCI spec")
+	}
+	if spec.Root == nil {
+		log.Panicln("Root is empty in OCI spec")
+	}
+	return
+}
+
+func (s *Spec) GetCapabilities() []string {
+	if s == nil || s.Process == nil || s.Process.Capabilities == nil {
+		return nil
+	}
+
+	var caps []string
+	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
+	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
+	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
+	if (rc1cmp == 1 || rc1cmp == 0) && (rc5cmp == -1) {
+		err := json.Unmarshal(*s.Process.Capabilities, &caps)
+		if err != nil {
+			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+		}
+		return caps
+	}
+
+	// Otherwise, parse s.Process.Capabilities as:
+	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	capabilities := specs.LinuxCapabilities{}
+	err := json.Unmarshal(*s.Process.Capabilities, &capabilities)
+	if err != nil {
+		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
+	}
+
+	return image.OCISpecCapabilities(capabilities).GetCapabilities()
+}
+
+func isPrivileged(s *Spec) bool {
+	return image.IsPrivileged(s)
+}
+
+func getMigConfigDevices(i image.CUDA) *string {
+	return getMigDevices(i, image.EnvVarNvidiaMigConfigDevices)
+}
+
+func getMigMonitorDevices(i image.CUDA) *string {
+	return getMigDevices(i, image.EnvVarNvidiaMigMonitorDevices)
+}
+
+func getMigDevices(image image.CUDA, envvar string) *string {
+	if !image.HasEnvvar(envvar) {
+		return nil
+	}
+	devices := image.Getenv(envvar)
+	return &devices
+}
+
+func (hookConfig *hookConfig) getImexChannels(image image.CUDA, privileged bool) []string {
+	if hookConfig.Features.IgnoreImexChannelRequests.IsEnabled() {
+		return nil
+	}
+
+	// If enabled, try and get the device list from volume mounts first
+	if hookConfig.AcceptDeviceListAsVolumeMounts {
+		devices := image.ImexChannelsFromMounts()
+		if len(devices) > 0 {
+			return devices
+		}
+	}
+	devices := image.ImexChannelsFromEnvVar()
+	if len(devices) == 0 {
+		return nil
+	}
+
+	if privileged || hookConfig.AcceptEnvvarUnprivileged {
+		return devices
+	}
+
+	return nil
+}
+
+func (hookConfig *hookConfig) getDriverCapabilities(cudaImage image.CUDA, legacyImage bool) image.DriverCapabilities {
+	// We use the default driver capabilities by default. This is filtered to only include the
+	// supported capabilities
+	supportedDriverCapabilities := image.NewDriverCapabilities(hookConfig.SupportedDriverCapabilities)
+	capabilities := supportedDriverCapabilities.Intersection(image.DefaultDriverCapabilities)
+
+	capsEnvSpecified := cudaImage.HasEnvvar(image.EnvVarNvidiaDriverCapabilities)
+	capsEnv := cudaImage.Getenv(image.EnvVarNvidiaDriverCapabilities)
+
+	if !capsEnvSpecified && legacyImage {
+		// Environment variable unset with legacy image: set all capabilities.
+		return supportedDriverCapabilities
+	}
+
+	if capsEnvSpecified && len(capsEnv) > 0 {
+		// If the envvironment variable is specified and is non-empty, use the capabilities value
+		envCapabilities := image.NewDriverCapabilities(capsEnv)
+		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
+		if !envCapabilities.IsAll() && len(capabilities) != len(envCapabilities) {
+			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
+		}
+	}
+
+	return capabilities
+}
+
+func (hookConfig *hookConfig) getNvidiaConfig(image image.CUDA, privileged bool) *nvidiaConfig {
+	legacyImage := image.IsLegacy()
+
+	devices := image.VisibleDevices()
+	if len(devices) == 0 {
+		// empty devices means this is not a GPU container.
+		return nil
+	}
+
+	var migConfigDevices string
+	if d := getMigConfigDevices(image); d != nil {
+		migConfigDevices = *d
+	}
+	if !privileged && migConfigDevices != "" {
+		log.Panicln("cannot set MIG_CONFIG_DEVICES in non privileged container")
+	}
+
+	var migMonitorDevices string
+	if d := getMigMonitorDevices(image); d != nil {
+		migMonitorDevices = *d
+	}
+	if !privileged && migMonitorDevices != "" {
+		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
+	}
+
+	imexChannels := hookConfig.getImexChannels(image, privileged)
+
+	driverCapabilities := hookConfig.getDriverCapabilities(image, legacyImage).String()
+
+	requirements, err := image.GetRequirements()
+	if err != nil {
+		log.Panicln("failed to get requirements", err)
+	}
+
+	return &nvidiaConfig{
+		Devices:            devices,
+		MigConfigDevices:   migConfigDevices,
+		MigMonitorDevices:  migMonitorDevices,
+		ImexChannels:       imexChannels,
+		DriverCapabilities: driverCapabilities,
+		Requirements:       requirements,
+	}
+}
+
+func (hookConfig *hookConfig) getContainerConfig() (config *containerConfig) {
+	hookConfig.Lock()
+	defer hookConfig.Unlock()
+
+	if hookConfig.containerConfig != nil {
+		return hookConfig.containerConfig
+	}
+
+	var h HookState
+	d := json.NewDecoder(os.Stdin)
+	if err := d.Decode(&h); err != nil {
+		log.Panicln("could not decode container state:", err)
+	}
+
+	b := h.Bundle
+	if len(b) == 0 {
+		b = h.BundlePath
+	}
+
+	s := loadSpec(path.Join(b, "config.json"))
+
+	privileged := isPrivileged(s)
+
+	i, err := image.New(
+		image.WithEnv(s.Process.Env),
+		image.WithMounts(s.Mounts),
+		image.WithPrivileged(privileged),
+		image.WithDisableRequire(hookConfig.DisableRequire),
+		image.WithAcceptDeviceListAsVolumeMounts(hookConfig.AcceptDeviceListAsVolumeMounts),
+		image.WithAcceptEnvvarUnprivileged(hookConfig.AcceptEnvvarUnprivileged),
+		image.WithPreferredVisibleDevicesEnvVars(hookConfig.getSwarmResourceEnvvars()...),
+	)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	cc := containerConfig{
+		Pid:    h.Pid,
+		Rootfs: s.Root.Path,
+		Image:  i,
+		Nvidia: hookConfig.getNvidiaConfig(i, privileged),
+	}
+	hookConfig.containerConfig = &cc
+
+	return hookConfig.containerConfig
+}

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -0,0 +1,644 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+)
+
+func TestGetNvidiaConfig(t *testing.T) {
+	var tests = []struct {
+		description    string
+		env            map[string]string
+		privileged     bool
+		hookConfig     *hookConfig
+		expectedConfig *nvidiaConfig
+		expectedPanic  bool
+	}{
+		{
+			description:    "No environment, unprivileged",
+			env:            map[string]string{},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description:    "No environment, privileged",
+			env:            map[string]string{},
+			privileged:     true,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, no devices, no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion: "9.0",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices 'all', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices 'empty', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, devices 'void', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "void",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Legacy image, devices 'none', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "none",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{""},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:          "9.0",
+				image.EnvVarNvidiaVisibleDevices: "gpu0,gpu1",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities 'empty', no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities 'all', no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, no requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, requirements set",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+			},
+		},
+		{
+			description: "Legacy image, devices set, capabilities set, requirements set, disable requirements",
+			env: map[string]string{
+				image.EnvVarCudaVersion:              "9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+				image.EnvVarNvidiaDisableRequire:     "true",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
+			},
+		},
+		{
+			description: "Modern image, no devices, no capabilities, no requirements, no image.EnvVarCudaVersion",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda: "cuda>=9.0",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, no devices, no capabilities, no requirement, image.EnvVarCudaVersion set",
+			env: map[string]string{
+				image.EnvVarCudaVersion:       "9.0",
+				image.EnvVarNvidiaRequireCuda: "cuda>=9.0",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'all', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices 'empty', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'void', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "void",
+			},
+			privileged:     false,
+			expectedConfig: nil,
+		},
+		{
+			description: "Modern image, devices 'none', no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "none",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{""},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, no capabilities, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:    "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices: "gpu0,gpu1",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities 'empty', no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities 'all', no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: image.SupportedDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, no requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, requirements set",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
+			},
+		},
+		{
+			description: "Modern image, devices set, capabilities set, requirements set, disable requirements",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:        "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:     "gpu0,gpu1",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+				image.NvidiaRequirePrefix + "REQ0":   "req0=true",
+				image.NvidiaRequirePrefix + "REQ1":   "req1=false",
+				image.EnvVarNvidiaDisableRequire:     "true",
+			},
+			privileged: false,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"gpu0", "gpu1"},
+				DriverCapabilities: "display,video",
+				Requirements:       []string{},
+			},
+		},
+		{
+			description: "No cuda envs, devices 'all'",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: false,
+
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{},
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migConfig set, privileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:      "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:   "all",
+				image.EnvVarNvidiaMigConfigDevices: "mig0,mig1",
+			},
+			privileged: true,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				MigConfigDevices:   "mig0,mig1",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migConfig set, unprivileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:      "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:   "all",
+				image.EnvVarNvidiaMigConfigDevices: "mig0,mig1",
+			},
+			privileged:    false,
+			expectedPanic: true,
+		},
+		{
+			description: "Modern image, devices 'all', migMonitor set, privileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:       "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:    "all",
+				image.EnvVarNvidiaMigMonitorDevices: "mig0,mig1",
+			},
+			privileged: true,
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				MigMonitorDevices:  "mig0,mig1",
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+				Requirements:       []string{"cuda>=9.0"},
+			},
+		},
+		{
+			description: "Modern image, devices 'all', migMonitor set, unprivileged",
+			env: map[string]string{
+				image.EnvVarNvidiaRequireCuda:       "cuda>=9.0",
+				image.EnvVarNvidiaVisibleDevices:    "all",
+				image.EnvVarNvidiaMigMonitorDevices: "mig0,mig1",
+			},
+			privileged:    false,
+			expectedPanic: true,
+		},
+		{
+			description: "Hook config set as driver-capabilities-all",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:     "all",
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: "video,display",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: "display,video",
+			},
+		},
+		{
+			description: "Hook config set, envvar sets driver-capabilities",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices:     "all",
+				image.EnvVarNvidiaDriverCapabilities: "video,display",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: "video,display,compute,utility",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: "display,video",
+			},
+		},
+		{
+			description: "Hook config set, envvar unset sets default driver-capabilities",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: "video,display,utility,compute",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"all"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, swarmResource overrides device selection",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+				"DOCKER_SWARM_RESOURCE":          "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SwarmResource:               "DOCKER_SWARM_RESOURCE",
+					SupportedDriverCapabilities: "video,display,utility,compute",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"GPU1", "GPU2"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+		{
+			description: "Hook config set, comma separated swarmResource is split and overrides device selection",
+			env: map[string]string{
+				image.EnvVarNvidiaVisibleDevices: "all",
+				"DOCKER_SWARM_RESOURCE":          "GPU1,GPU2",
+			},
+			privileged: true,
+			hookConfig: &hookConfig{
+				Config: &config.Config{
+					SwarmResource:               "NOT_DOCKER_SWARM_RESOURCE,DOCKER_SWARM_RESOURCE",
+					SupportedDriverCapabilities: "video,display,utility,compute",
+				},
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            []string{"GPU1", "GPU2"},
+				DriverCapabilities: image.DefaultDriverCapabilities.String(),
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			image, _ := image.New(
+				image.WithEnvMap(tc.env),
+				image.WithPrivileged(tc.privileged),
+				image.WithPreferredVisibleDevicesEnvVars(tc.hookConfig.getSwarmResourceEnvvars()...),
+			)
+
+			// Wrap the call to getNvidiaConfig() in a closure.
+			var cfg *nvidiaConfig
+			getConfig := func() {
+				hookCfg := tc.hookConfig
+				if hookCfg == nil {
+					defaultConfig, _ := config.GetDefault()
+					hookCfg = &hookConfig{Config: defaultConfig}
+				}
+				cfg = hookCfg.getNvidiaConfig(image, tc.privileged)
+			}
+
+			// For any tests that are expected to panic, make sure they do.
+			if tc.expectedPanic {
+				require.Panics(t, getConfig)
+				return
+			}
+
+			// For all other tests, just grab the config
+			getConfig()
+
+			// And start comparing the test results to the expected results.
+			if tc.expectedConfig == nil {
+				require.Nil(t, cfg, tc.description)
+				return
+			}
+
+			require.NotNil(t, cfg, tc.description)
+
+			require.Equal(t, tc.expectedConfig.Devices, cfg.Devices)
+			require.Equal(t, tc.expectedConfig.MigConfigDevices, cfg.MigConfigDevices)
+			require.Equal(t, tc.expectedConfig.MigMonitorDevices, cfg.MigMonitorDevices)
+			require.Equal(t, tc.expectedConfig.DriverCapabilities, cfg.DriverCapabilities)
+
+			require.ElementsMatch(t, tc.expectedConfig.Requirements, cfg.Requirements)
+		})
+	}
+}
+
+func TestGetDriverCapabilities(t *testing.T) {
+
+	supportedCapabilities := "compute,display,utility,video"
+
+	testCases := []struct {
+		description           string
+		env                   map[string]string
+		legacyImage           bool
+		supportedCapabilities string
+		expectedPanic         bool
+		expectedCapabilities  string
+	}{
+		{
+			description: "Env is set for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "display,video",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description: "Env is all for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for legacy image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
+		},
+		{
+			description:           "Env unset for legacy image is 'all'",
+			env:                   map[string]string{},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is set for modern image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "display,video",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description:           "Env unset for modern image is default",
+			env:                   map[string]string{},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
+		},
+		{
+			description: "Env is all for modern image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "all",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for modern image",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  image.DefaultDriverCapabilities.String(),
+		},
+		{
+			description: "Invalid capabilities panic",
+			env: map[string]string{
+				image.EnvVarNvidiaDriverCapabilities: "compute,utility",
+			},
+			supportedCapabilities: "not-compute,not-utility",
+			expectedPanic:         true,
+		},
+		{
+			description:           "Default is restricted for modern image",
+			legacyImage:           false,
+			supportedCapabilities: "compute",
+			expectedCapabilities:  "compute",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			var capabilities string
+
+			c := hookConfig{
+				Config: &config.Config{
+					SupportedDriverCapabilities: tc.supportedCapabilities,
+				},
+			}
+
+			image, _ := image.New(
+				image.WithEnvMap(tc.env),
+			)
+			getDriverCapabilities := func() {
+				capabilities = c.getDriverCapabilities(image, tc.legacyImage).String()
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getDriverCapabilities)
+				return
+			}
+
+			getDriverCapabilities()
+			require.EqualValues(t, tc.expectedCapabilities, capabilities)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"reflect"
+	"strings"
+	"sync"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+)
+
+// hookConfig wraps the toolkit config.
+// This allows for functions to be defined on the local type.
+type hookConfig struct {
+	sync.Mutex
+	*config.Config
+	containerConfig *containerConfig
+}
+
+// loadConfig loads the required paths for the hook config.
+func loadConfig() (*config.Config, error) {
+	configFilePath, required := getConfigFilePath()
+	cfg, err := config.New(
+		config.WithConfigFile(configFilePath),
+		config.WithRequired(true),
+	)
+	if err == nil {
+		return cfg.Config()
+	} else if os.IsNotExist(err) && !required {
+		return config.GetDefault()
+	}
+	return nil, fmt.Errorf("couldn't open required configuration file: %v", err)
+}
+
+func getConfigFilePath() (string, bool) {
+	if configFromFlag := *configflag; configFromFlag != "" {
+		return configFromFlag, true
+	}
+	if configFromEnvvar := os.Getenv(config.FilePathOverrideEnvVar); configFromEnvvar != "" {
+		return configFromEnvvar, true
+	}
+	return config.GetConfigFilePath(), false
+}
+
+func getHookConfig() (*hookConfig, error) {
+	cfg, err := loadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+	config := &hookConfig{Config: cfg}
+
+	allSupportedDriverCapabilities := image.SupportedDriverCapabilities
+	if config.SupportedDriverCapabilities == "all" {
+		config.SupportedDriverCapabilities = allSupportedDriverCapabilities.String()
+	}
+	configuredCapabilities := image.NewDriverCapabilities(config.SupportedDriverCapabilities)
+	// We ensure that the configured value is a subset of all supported capabilities
+	if !allSupportedDriverCapabilities.IsSuperset(configuredCapabilities) {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allSupportedDriverCapabilities.String())
+	}
+
+	return config, nil
+}
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c *hookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(&c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}
+
+// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
+func (c *hookConfig) getSwarmResourceEnvvars() []string {
+	if c == nil || c.SwarmResource == "" {
+		return nil
+	}
+
+	candidates := strings.Split(c.SwarmResource, ",")
+
+	var envvars []string
+	for _, c := range candidates {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) > 0 {
+			envvars = append(envvars, trimmed)
+		}
+	}
+
+	return envvars
+}
+
+// nvidiaContainerCliCUDACompatModeFlags returns required --cuda-compat-mode
+// flag(s) depending on the hook and runtime configurations.
+func (c *hookConfig) nvidiaContainerCliCUDACompatModeFlags() []string {
+	var flag string
+	switch c.NVIDIAContainerRuntimeConfig.Modes.Legacy.CUDACompatMode {
+	case config.CUDACompatModeLdconfig:
+		flag = "--cuda-compat-mode=ldconfig"
+	case config.CUDACompatModeMount:
+		flag = "--cuda-compat-mode=mount"
+	case config.CUDACompatModeDisabled, config.CUDACompatModeHook:
+		flag = "--cuda-compat-mode=disabled"
+	default:
+		if !c.Features.AllowCUDACompatLibsFromContainer.IsEnabled() {
+			flag = "--cuda-compat-mode=disabled"
+		}
+	}
+
+	if flag == "" {
+		return nil
+	}
+	return []string{flag}
+}
+
+func (c *hookConfig) assertModeIsLegacy() error {
+	if c.NVIDIAContainerRuntimeHookConfig.SkipModeDetection {
+		return nil
+	}
+
+	mr := info.NewRuntimeModeResolver(
+		info.WithLogger(&logInterceptor{}),
+		info.WithImage(&c.containerConfig.Image),
+		info.WithDefaultMode(info.LegacyRuntimeMode),
+	)
+
+	mode := mr.ResolveRuntimeMode(c.NVIDIAContainerRuntimeConfig.Mode)
+	if mode == "legacy" {
+		return nil
+	}
+	return fmt.Errorf("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead")
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -22,22 +22,25 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 )
 
 func TestGetHookConfig(t *testing.T) {
 	testCases := []struct {
 		lines                      []string
 		expectedPanic              bool
-		expectedDriverCapabilities DriverCapabilities
+		expectedDriverCapabilities string
 	}{
 		{
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"all\"",
 			},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
@@ -47,19 +50,19 @@ func TestGetHookConfig(t *testing.T) {
 		},
 		{
 			lines:                      []string{},
-			expectedDriverCapabilities: allDriverCapabilities,
+			expectedDriverCapabilities: image.SupportedDriverCapabilities.String(),
 		},
 		{
 			lines: []string{
 				"supported-driver-capabilities = \"\"",
 			},
-			expectedDriverCapabilities: none,
+			expectedDriverCapabilities: "",
 		},
 		{
 			lines: []string{
-				"supported-driver-capabilities = \"utility,compute\"",
+				"supported-driver-capabilities = \"compute,utility\"",
 			},
-			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+			expectedDriverCapabilities: "compute,utility",
 		},
 	}
 
@@ -82,14 +85,15 @@ func TestGetHookConfig(t *testing.T) {
 				configflag = &filename
 
 				for _, line := range tc.lines {
-					_, err := configFile.WriteString(fmt.Sprintf("%s\n", line))
+					_, err := fmt.Fprintf(configFile, "%s\n", line)
 					require.NoError(t, err)
 				}
 			}
 
-			var config HookConfig
+			var cfg *hookConfig
 			getHookConfig := func() {
-				config = getHookConfig()
+				c, _ := getHookConfig()
+				cfg = c
 			}
 
 			if tc.expectedPanic {
@@ -99,7 +103,56 @@ func TestGetHookConfig(t *testing.T) {
 
 			getHookConfig()
 
-			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+			require.EqualValues(t, tc.expectedDriverCapabilities, cfg.SupportedDriverCapabilities)
+		})
+	}
+}
+
+func TestGetSwarmResourceEnvvars(t *testing.T) {
+	testCases := []struct {
+		value    string
+		expected []string
+	}{
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value:    " ",
+			expected: nil,
+		},
+		{
+			value:    "single",
+			expected: []string{"single"},
+		},
+		{
+			value:    "single ",
+			expected: []string{"single"},
+		},
+		{
+			value:    "one,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one ,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one, two",
+			expected: []string{"one", "two"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			c := &hookConfig{
+				Config: &config.Config{
+					SwarmResource: tc.value,
+				},
+			}
+
+			envvars := c.getSwarmResourceEnvvars()
+			require.EqualValues(t, tc.expected, envvars)
 		})
 	}
 }

cmd/nvidia-container-runtime-hook/hook_test.go

@@ -7,51 +7,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestParseCudaVersionValid(t *testing.T) {
-	var tests = []struct {
-		version  string
-		expected [3]uint32
-	}{
-		{"0", [3]uint32{0, 0, 0}},
-		{"8", [3]uint32{8, 0, 0}},
-		{"7.5", [3]uint32{7, 5, 0}},
-		{"9.0.116", [3]uint32{9, 0, 116}},
-		{"4294967295.4294967295.4294967295", [3]uint32{4294967295, 4294967295, 4294967295}},
-	}
-	for i, c := range tests {
-		vmaj, vmin, vpatch := parseCudaVersion(c.version)
-
-		version := [3]uint32{vmaj, vmin, vpatch}
-
-		require.Equal(t, c.expected, version, "%d: %v", i, c)
-	}
-}
-
-func TestParseCudaVersionInvalid(t *testing.T) {
-	var tests = []string{
-		"foo",
-		"foo.5.10",
-		"9.0.116.50",
-		"9.0.116foo",
-		"7.foo",
-		"9.0.bar",
-		"9.4294967296",
-		"9.0.116.",
-		"9..0",
-		"9.",
-		".5.10",
-		"-9",
-		"+9",
-		"-9.1.116",
-		"-9.-1.-116",
-	}
-	for _, c := range tests {
-		require.Panics(t, func() {
-			parseCudaVersion(c)
-		}, "parseCudaVersion(%v)", c)
-	}
-}
-
 func TestIsPrivileged(t *testing.T) {
 	var tests = []struct {
 		spec     string

cmd/nvidia-container-runtime-hook/main.go

@@ -6,21 +6,23 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
 	"strings"
 	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 var (
-	debugflag  = flag.Bool("debug", false, "enable debug output")
-	forceflag  = flag.Bool("force", false, "force execution of prestart hook in experimental mode")
-	configflag = flag.String("config", "", "configuration file")
-
-	defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
+	debugflag   = flag.Bool("debug", false, "enable debug output")
+	versionflag = flag.Bool("version", false, "enable version output")
+	configflag  = flag.String("config", "", "configuration file")
 )
 
 func exit() {
@@ -36,28 +38,12 @@ func exit() {
 	os.Exit(0)
 }
 
-func getPATH(config CLIConfig) string {
-	dirs := filepath.SplitList(os.Getenv("PATH"))
-	// directories from the hook environment have higher precedence
-	dirs = append(dirs, defaultPATH...)
-
-	if config.Root != nil {
-		rootDirs := []string{}
-		for _, dir := range dirs {
-			rootDirs = append(rootDirs, path.Join(*config.Root, dir))
-		}
-		// directories with the root prefix have higher precedence
-		dirs = append(rootDirs, dirs...)
-	}
-	return strings.Join(dirs, ":")
-}
-
-func getCLIPath(config CLIConfig) string {
-	if config.Path != nil {
-		return *config.Path
+func getCLIPath(config config.ContainerCLIConfig) string {
+	if config.Path != "" {
+		return config.Path
 	}
 
-	if err := os.Setenv("PATH", getPATH(config)); err != nil {
+	if err := os.Setenv("PATH", lookup.GetPath(config.Root)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -69,7 +55,7 @@ func getCLIPath(config CLIConfig) string {
 }
 
 // getRootfsPath returns an absolute path. We don't need to resolve symlinks for now.
-func getRootfsPath(config containerConfig) string {
+func getRootfsPath(config *containerConfig) string {
 	rootfs, err := filepath.Abs(config.Rootfs)
 	if err != nil {
 		log.Panicln(err)
@@ -83,53 +69,61 @@ func doPrestart() {
 	defer exit()
 	log.SetFlags(0)
 
-	hook := getHookConfig()
-	cli := hook.NvidiaContainerCLI
-
-	if hook.NVIDIAContainerRuntime.Experimental && !*forceflag {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+	hook, err := getHookConfig()
+	if err != nil || hook == nil {
+		log.Panicln("error getting hook config:", err)
 	}
+	cli := hook.NVIDIAContainerCLIConfig
 
-	container := getContainerConfig(hook)
+	container := hook.getContainerConfig()
 	nvidia := container.Nvidia
 	if nvidia == nil {
 		// Not a GPU container, nothing to do.
 		return
 	}
 
+	if err := hook.assertModeIsLegacy(); err != nil {
+		log.Panicf("%v", err)
+	}
+
 	rootfs := getRootfsPath(container)
 
 	args := []string{getCLIPath(cli)}
-	if cli.Root != nil {
-		args = append(args, fmt.Sprintf("--root=%s", *cli.Root))
+	if cli.Root != "" {
+		args = append(args, fmt.Sprintf("--root=%s", cli.Root))
 	}
 	if cli.LoadKmods {
 		args = append(args, "--load-kmods")
 	}
+	if hook.Features.DisableImexChannelCreation.IsEnabled() {
+		args = append(args, "--no-create-imex-channels")
+	}
 	if cli.NoPivot {
 		args = append(args, "--no-pivot")
 	}
 	if *debugflag {
 		args = append(args, "--debug=/dev/stderr")
-	} else if cli.Debug != nil {
-		args = append(args, fmt.Sprintf("--debug=%s", *cli.Debug))
+	} else if cli.Debug != "" {
+		args = append(args, fmt.Sprintf("--debug=%s", cli.Debug))
 	}
-	if cli.Ldcache != nil {
-		args = append(args, fmt.Sprintf("--ldcache=%s", *cli.Ldcache))
+	if cli.Ldcache != "" {
+		args = append(args, fmt.Sprintf("--ldcache=%s", cli.Ldcache))
 	}
-	if cli.User != nil {
-		args = append(args, fmt.Sprintf("--user=%s", *cli.User))
+	if cli.User != "" {
+		args = append(args, fmt.Sprintf("--user=%s", cli.User))
 	}
 	args = append(args, "configure")
 
-	if cli.Ldconfig != nil {
-		args = append(args, fmt.Sprintf("--ldconfig=%s", *cli.Ldconfig))
+	args = append(args, hook.nvidiaContainerCliCUDACompatModeFlags()...)
+
+	if ldconfigPath := cli.NormalizeLDConfigPath(); ldconfigPath != "" {
+		args = append(args, fmt.Sprintf("--ldconfig=%s", ldconfigPath))
 	}
 	if cli.NoCgroups {
 		args = append(args, "--no-cgroups")
 	}
-	if len(nvidia.Devices) > 0 {
-		args = append(args, fmt.Sprintf("--device=%s", nvidia.Devices))
+	if devicesString := strings.Join(nvidia.Devices, ","); len(devicesString) > 0 {
+		args = append(args, fmt.Sprintf("--device=%s", devicesString))
 	}
 	if len(nvidia.MigConfigDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-config=%s", nvidia.MigConfigDevices))
@@ -137,6 +131,9 @@ func doPrestart() {
 	if len(nvidia.MigMonitorDevices) > 0 {
 		args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
 	}
+	if imexString := strings.Join(nvidia.ImexChannels, ","); len(imexString) > 0 {
+		args = append(args, fmt.Sprintf("--imex-channel=%s", imexString))
+	}
 
 	for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
 		if len(cap) == 0 {
@@ -145,16 +142,15 @@ func doPrestart() {
 		args = append(args, capabilityToCLI(cap))
 	}
 
-	if !hook.DisableRequire && !nvidia.DisableRequire {
-		for _, req := range nvidia.Requirements {
-			args = append(args, fmt.Sprintf("--require=%s", req))
-		}
+	for _, req := range nvidia.Requirements {
+		args = append(args, fmt.Sprintf("--require=%s", req))
 	}
 
 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
 	args = append(args, rootfs)
 
 	env := append(os.Environ(), cli.Environment...)
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection?
 	err = syscall.Exec(args[0], args, env)
 	log.Panicln("exec failed:", err)
 }
@@ -172,6 +168,11 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	if *versionflag {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime Hook", info.GetVersionString())
+		return
+	}
+
 	args := flag.Args()
 	if len(args) == 0 {
 		flag.Usage()
@@ -191,3 +192,12 @@ func main() {
 		os.Exit(2)
 	}
 }
+
+// logInterceptor implements the logger.Interface to allow for logging from executable.
+type logInterceptor struct {
+	logger.NullLogger
+}
+
+func (l *logInterceptor) Infof(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}

cmd/nvidia-container-runtime.cdi/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("cdi"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime.legacy/main.go

@@ -0,0 +1,34 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+)
+
+func main() {
+	rt := runtime.New(
+		runtime.WithModeOverride("legacy"),
+	)
+
+	err := rt.Run(os.Args)
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/nvidia-container-runtime/README.md

@@ -2,24 +2,209 @@
 
 The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
 
-## Standard Mode
+## Configuration
 
-In the standard mode configuration, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from project [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+The NVIDIA Container Runtime uses file-based configuration, with the config stored in `/etc/nvidia-container-runtime/config.toml`. The `/etc` path can be overridden using the `XDG_CONFIG_HOME` environment variable with the `${XDG_CONFIG_HOME}/nvidia-container-runtime/config.toml` file used instead if this environment variable is set.
 
-## Experimental Mode
+This config file may contain options for other components of the NVIDIA container stack and for the NVIDIA Container Runtime, the relevant config section is `nvidia-container-runtime`
 
-The NVIDIA Container Runtime can be configured in an experimental mode by setting the following options in the runtime's `config.toml` file:
+### Logging
+
+The `log-level` config option (default: `"info"`) specifies the log level to use and the `debug` option, if set, specifies a log file to which logs for the NVIDIA Container Runtime must be written.
+
+In addition to this, the NVIDIA Container Runtime considers the value of `--log` and `--log-format` flags that may be passed to it by a container runtime such as docker or containerd. If the `--debug` flag is present the log-level specified in the config file is overridden as `"debug"`.
+
+### Low-level Runtime Path
+
+The `runtimes` config option allows for the low-level runtime to be specified. The first entry in this list that is an existing executable file is used as the low-level runtime. If the entry is not a path, the `PATH` is searched for a matching executable. If the entry is a path this is checked instead.
+
+The default value for this setting is:
+```toml
+runtimes = [
+    "runc",
+    "crun",
+]
+```
+
+and if, for example, `crun` is to be used instead this can be changed to:
+```toml
+runtimes = [
+    "crun",
+]
+```
+
+### Runtime Mode
+
+The `mode` config option (default `"auto"`) controls the high-level behaviour of the runtime.
+
+#### Auto Mode
+
+When `mode` is set to `"auto"`, the runtime employs heuristics to determine which mode to use based on, for example, the platform where the runtime is being run.
+
+#### Legacy Mode
+
+When `mode` is set to `"legacy"`, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from the [libnvidia-container](https://github.com/NVIDIA/libnvidia-container) project.
+
+#### CSV Mode
+
+When `mode` is set to `"csv"`, CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` define the devices and mounts that are to be injected into a container when it is created. The search path for the files can be overridden by modifying the `nvidia-container-runtime.modes.csv.mount-spec-path` in the config as below:
 
 ```toml
 [nvidia-container-runtime]
-experimental = true
+    [nvidia-container-runtime.modes.csv]
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
 ```
 
-When this setting is enabled, the modifications made to the OCI specification are controlled by the `nvidia-container-runtime.discover-mode` option, with the following mode supported:
-
-* `"legacy"`: This mode mirrors the behaviour of the standard mode, inserting the NVIDIA Container Runtime Hook as a `prestart` hook into the container's OCI specification.
-* `"csv"`: This mode uses CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` to define the devices and mounts that are to be injected into a container when it is created.
+This mode is primarily targeted at Tegra-based systems without NVML available.
 
 ### Notes on using the docker CLI
 
-The `docker` CLI supports the `--gpus` flag to select GPUs for inclusion in a container. Since specifying this flag inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification. When experimental mode is activated, the NVIDIA Container Runtime detects the presence of the hook and raises an error. This requirement will be relaxed in the near future.
+Note that only the `"legacy"` NVIDIA Container Runtime mode is directly compatible with the `--gpus` flag implemented by the `docker` CLI (assuming the NVIDIA Container Runtime is not used). The reason for this is that `docker` inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification.
+
+
+If a different mode is explicitly set or detected, the NVIDIA Container Runtime Hook will raise the following error when `--gpus` is set:
+```
+$ docker run --rm --gpus all ubuntu:18.04
+docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'csv'
+invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.: unknown.
+```
+Here NVIDIA Container Runtime must be used explicitly. The recommended way to do this is to specify the `--runtime=nvidia` command line argument as part of the `docker run` commmand as follows:
+```
+$ docker run --rm --gpus all --runtime=nvidia ubuntu:18.04
+```
+
+Alternatively the NVIDIA Container Runtime can be set as the default runtime for docker. This can be done by modifying the `/etc/docker/daemon.json` file as follows:
+```json
+{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}
+```
+
+## Environment variables (OCI spec)
+
+Each environment variable maps to an command-line argument for `nvidia-container-cli` from [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+These variables are already set in our [official CUDA images](https://hub.docker.com/r/nvidia/cuda/).
+
+### `NVIDIA_VISIBLE_DEVICES`
+This variable controls which GPUs will be made accessible inside the container.
+
+#### Possible values
+* `0,1,2`, `GPU-fef8089b` …: a comma-separated list of GPU UUID(s) or index(es).
+* `all`: all GPUs will be accessible, this is the default value in our container images.
+* `none`: no GPU will be accessible, but driver capabilities will be enabled.
+* `void` or *empty* or *unset*: `nvidia-container-runtime` will have the same behavior as `runc`.
+
+**Note**: When running on a MIG capable device, the following values will also be available:
+* `0:0,0:1,1:0`, `MIG-GPU-fef8089b/0/1` …: a comma-separated list of MIG Device UUID(s) or index(es).
+
+Where the MIG device indices have the form `<GPU Device Index>:<MIG Device Index>` as seen in the example output:
+```
+$ nvidia-smi -L
+GPU 0: Graphics Device (UUID: GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5)
+  MIG Device 0: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/0)
+  MIG Device 1: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/1)
+  MIG Device 2: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/11/0)
+```
+
+### `NVIDIA_MIG_CONFIG_DEVICES`
+This variable controls which of the visible GPUs can have their MIG
+configuration managed from within the container. This includes enabling and
+disabling MIG mode, creating and destroying GPU Instances and Compute
+Instances, etc.
+
+#### Possible values
+* `all`: Allow all MIG-capable GPUs in the visible device list to have their
+  MIG configurations managed.
+
+**Note**:
+* This feature is only available on MIG capable devices (e.g. the A100).
+* To use this feature, the container must be started with `CAP_SYS_ADMIN` privileges.
+* When not running as `root`, the container user must have read access to the
+  `/proc/driver/nvidia/capabilities/mig/config` file on the host.
+
+### `NVIDIA_MIG_MONITOR_DEVICES`
+This variable controls which of the visible GPUs can have aggregate information
+about all of their MIG devices monitored from within the container. This
+includes inspecting the aggregate memory usage, listing the aggregate running
+processes, etc.
+
+#### Possible values
+* `all`: Allow all MIG-capable GPUs in the visible device list to have their
+  MIG devices monitored.
+
+**Note**:
+* This feature is only available on MIG capable devices (e.g. the A100).
+* To use this feature, the container must be started with `CAP_SYS_ADMIN` privileges.
+* When not running as `root`, the container user must have read access to the
+  `/proc/driver/nvidia/capabilities/mig/monitor` file on the host.
+
+### `NVIDIA_DRIVER_CAPABILITIES`
+This option controls which driver libraries/binaries will be mounted inside the container.
+
+#### Possible values
+* `compute,video`, `graphics,utility` …: a comma-separated list of driver features the container needs.
+* `all`: enable all available driver capabilities.
+* *empty* or *unset*: use default driver capability: `utility,compute`.
+
+#### Supported driver capabilities
+* `compute`: required for CUDA and OpenCL applications.
+* `compat32`: required for running 32-bit applications.
+* `graphics`: required for running OpenGL and Vulkan applications.
+* `utility`: required for using `nvidia-smi` and NVML.
+* `video`: required for using the Video Codec SDK.
+* `display`: required for leveraging X11 display.
+
+### `NVIDIA_REQUIRE_*`
+A logical expression to define constraints on the configurations supported by the container.
+
+#### Supported constraints
+* `cuda`: constraint on the CUDA driver version.
+* `driver`: constraint on the driver version.
+* `arch`: constraint on the compute architectures of the selected GPUs.
+* `brand`: constraint on the brand of the selected GPUs (e.g. GeForce, Tesla, GRID).
+
+#### Expressions
+Multiple constraints can be expressed in a single environment variable: space-separated constraints are ORed, comma-separated constraints are ANDed.
+Multiple environment variables of the form `NVIDIA_REQUIRE_*` are ANDed together.
+
+### `NVIDIA_DISABLE_REQUIRE`
+Single switch to disable all the constraints of the form `NVIDIA_REQUIRE_*`.
+
+### `NVIDIA_REQUIRE_CUDA`
+
+The version of the CUDA toolkit used by the container. It is an instance of the generic `NVIDIA_REQUIRE_*` case and it is set by official CUDA images.
+If the version of the NVIDIA driver is insufficient to run this version of CUDA, the container will not be started.
+
+#### Possible values
+* `cuda>=7.5`, `cuda>=8.0`, `cuda>=9.0` …: any valid CUDA version in the form `major.minor`.
+
+### `CUDA_VERSION`
+Similar to `NVIDIA_REQUIRE_CUDA`, for legacy CUDA images.
+In addition, if `NVIDIA_REQUIRE_CUDA` is not set, `NVIDIA_VISIBLE_DEVICES` and `NVIDIA_DRIVER_CAPABILITIES` will default to `all`.
+
+## Usage example
+
+**NOTE:** The use of the `nvidia-container-runtime` as CLI replacement for `runc` is uncommon and is only provided for completeness.
+
+Although the `nvidia-container-runtime` is typically configured as a replacement for `runc` or `crun` in various container engines, it can also be
+invoked from the command line as `runc` would. For example:
+
+```sh
+# Setup a rootfs based on Ubuntu 16.04
+cd $(mktemp -d) && mkdir rootfs
+curl -sS http://cdimage.ubuntu.com/ubuntu-base/releases/16.04/release/ubuntu-base-16.04.6-base-amd64.tar.gz | tar --exclude 'dev/*' -C rootfs -xz
+
+# Create an OCI runtime spec
+nvidia-container-runtime spec
+sed -i 's;"sh";"nvidia-smi";' config.json
+sed -i 's;\("TERM=xterm"\);\1, "NVIDIA_VISIBLE_DEVICES=0";' config.json
+
+# Run the container
+sudo nvidia-container-runtime run nvidia_smi
+```

cmd/nvidia-container-runtime/logger.go

@@ -1,79 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/sirupsen/logrus"
-	"github.com/tsaikd/KDGoLib/logrusutil"
-)
-
-// Logger adds a way to manage output to a log file to a logrus.Logger
-type Logger struct {
-	*logrus.Logger
-	previousOutput io.Writer
-	logFile        *os.File
-}
-
-// NewLogger constructs a Logger with a preddefined formatter
-func NewLogger() *Logger {
-	logrusLogger := logrus.New()
-
-	formatter := &logrusutil.ConsoleLogFormatter{
-		TimestampFormat: "2006/01/02 15:04:07",
-		Flag:            logrusutil.Ltime,
-	}
-
-	logger := &Logger{
-		Logger: logrusLogger,
-	}
-	logger.SetFormatter(formatter)
-
-	return logger
-}
-
-// LogToFile opens the specified file for appending and sets the logger to
-// output to the opened file. A reference to the file pointer is stored to
-// allow this to be closed.
-func (l *Logger) LogToFile(filename string) error {
-	logFile, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return fmt.Errorf("error opening debug log file: %v", err)
-	}
-
-	l.logFile = logFile
-	l.previousOutput = l.Out
-	l.SetOutput(logFile)
-
-	return nil
-}
-
-// CloseFile closes the log file (if any) and resets the logger output to what it
-// was before LogToFile was called.
-func (l *Logger) CloseFile() error {
-	if l.logFile == nil {
-		return nil
-	}
-	logFile := l.logFile
-	l.SetOutput(l.previousOutput)
-	l.logFile = nil
-
-	return logFile.Close()
-}

cmd/nvidia-container-runtime/main.go

@@ -1,54 +1,15 @@
 package main
 
 import (
-	"fmt"
 	"os"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/sirupsen/logrus"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
 )
 
-var logger = NewLogger()
-
 func main() {
-	err := run(os.Args)
+	r := runtime.New()
+	err := r.Run(os.Args)
 	if err != nil {
-		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
-
-// run is an entry point that allows for idiomatic handling of errors
-// when calling from the main function.
-func run(argv []string) (rerr error) {
-	logger.Debugf("Running %v", argv)
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return fmt.Errorf("error loading config: %v", err)
-	}
-
-	err = logger.LogToFile(cfg.NVIDIAContainerRuntimeConfig.DebugFilePath)
-	if err != nil {
-		return fmt.Errorf("error opening debug log file: %v", err)
-	}
-	defer func() {
-		// We capture and log a returning error before closing the log file.
-		if rerr != nil {
-			logger.Errorf("%v", rerr)
-		}
-		logger.CloseFile()
-	}()
-
-	if logLevel, err := logrus.ParseLevel(cfg.NVIDIAContainerRuntimeConfig.LogLevel); err == nil {
-		logger.SetLevel(logLevel)
-	} else {
-		logger.Warnf("Invalid log-level '%v'; using '%v'", cfg.NVIDIAContainerRuntimeConfig.LogLevel, logger.Level.String())
-	}
-
-	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
-	if err != nil {
-		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
-	}
-
-	return runtime.Exec(argv)
-}

cmd/nvidia-container-runtime/main_test.go

@@ -3,25 +3,32 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"io/ioutil"
+	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 )
 
 const (
 	nvidiaRuntime            = "nvidia-container-runtime"
 	nvidiaHook               = "nvidia-container-runtime-hook"
-	bundlePathSuffix         = "test/output/bundle/"
+	bundlePathSuffix         = "tests/output/bundle/"
 	specFile                 = "config.json"
-	unmodifiedSpecFileSuffix = "test/input/test_spec.json"
+	unmodifiedSpecFileSuffix = "tests/input/test_spec.json"
+)
+
+const (
+	runcExecutableName = "runc"
 )
 
 type testConfig struct {
@@ -37,10 +44,10 @@ func TestMain(m *testing.M) {
 	var err error
 	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
-		logger.Fatalf("error in test setup: could not get module root: %v", err)
+		log.Fatalf("error in test setup: could not get module root: %v", err)
 	}
-	testBinPath := filepath.Join(moduleRoot, "test", "bin")
-	testInputPath := filepath.Join(moduleRoot, "test", "input")
+	testBinPath := filepath.Join(moduleRoot, "tests", "bin")
+	testInputPath := filepath.Join(moduleRoot, "tests", "input")
 
 	// Set the environment variables for the test
 	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
@@ -49,11 +56,11 @@ func TestMain(m *testing.M) {
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		log.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -73,18 +80,14 @@ func TestMain(m *testing.M) {
 
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//		- Confirm the runtime handles bad input correctly
+//   - Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle")
-	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
-	output, err := cmdRun.CombinedOutput()
-	require.Errorf(t, err, "runtime should return an error", "output=%v", string(output))
-
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	err = cmdCreate.Run()
@@ -92,15 +95,17 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime runs with no errors
+//   - Confirm the runtime runs with no errors
+//
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
 		t.Fatalf("error generating runtime spec: %v", err)
 	}
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle", cfg.bundlePath(), "testcontainer")
 	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
 	output, err := cmdRun.CombinedOutput()
@@ -111,16 +116,16 @@ func TestGoodInput(t *testing.T) {
 	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
 	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	err = cmdCreate.Run()
 	require.NoError(t, err, "runtime should not return an error")
 
-	// Check config.json for NVIDIA prestart hook
+	// Check config.json to ensure that the NVIDIA prestart was not inserted.
 	spec, err = cfg.getRuntimeSpec()
 	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
-	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
-	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
 }
 
 // NVIDIA prestart hook already present in config file
@@ -156,22 +161,23 @@ func TestDuplicateHook(t *testing.T) {
 	}
 
 	// Test how runtime handles already existing prestart hook in config.json
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	output, err := cmdCreate.CombinedOutput()
 	require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
 
-	// Check config.json for NVIDIA prestart hook
+	// Check config.json to ensure that the NVIDIA prestart hook was removed.
 	spec, err = cfg.getRuntimeSpec()
 	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
-	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
-	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
 }
 
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	logger, _ := testlog.NewNullLogger()
+	m := modifier.NewStableRuntimeModifier(logger, nvidiaHook)
 	return m.Modify(spec)
 }
 
@@ -185,15 +191,16 @@ func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
 	}
 	defer jsonFile.Close()
 
-	jsonContent, err := ioutil.ReadAll(jsonFile)
-	if err != nil {
+	jsonContent, err := io.ReadAll(jsonFile)
+	switch {
+	case err != nil:
 		return spec, err
-	} else if json.Valid(jsonContent) {
+	case json.Valid(jsonContent):
 		err = json.Unmarshal(jsonContent, &spec)
 		if err != nil {
 			return spec, err
 		}
-	} else {
+	default:
 		err = json.NewDecoder(bytes.NewReader(jsonContent)).Decode(&spec)
 		if err != nil {
 			return spec, err
@@ -223,6 +230,7 @@ func (c testConfig) generateNewRuntimeSpec() error {
 		return err
 	}
 
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
 	cmd := exec.Command("cp", c.unmodifiedSpecFile(), c.specFilePath())
 	err = cmd.Run()
 	if err != nil {
@@ -230,18 +238,3 @@ func (c testConfig) generateNewRuntimeSpec() error {
 	}
 	return nil
 }
-
-// Return number of valid NVIDIA prestart hooks in runtime spec
-func nvidiaHookCount(hooks *specs.Hooks) int {
-	if hooks == nil {
-		return 0
-	}
-
-	count := 0
-	for _, hook := range hooks.Prestart {
-		if strings.Contains(hook.Path, nvidiaHook) {
-			count++
-		}
-	}
-	return count
-}

cmd/nvidia-container-runtime/modifier/experimental.go

@@ -1,178 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
-)
-
-// experiemental represents the modifications required by the experimental runtime
-type experimental struct {
-	logger     *logrus.Logger
-	discoverer discover.Discover
-}
-
-const (
-	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
-	visibleDevicesVoid   = "void"
-
-	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
-)
-
-// NewExperimentalModifier creates a modifier that applies the experimental
-// modications to an OCI spec if required by the runtime wrapper.
-func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	if err := ociSpec.Load(); err != nil {
-		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
-	}
-
-	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
-	// if no modification is required.
-	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
-	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
-		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
-		return nil, nil
-	}
-	logger.Infof("Constructing modifier from config: %+v", cfg)
-
-	config := &discover.Config{
-		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
-		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
-	}
-
-	var d discover.Discover
-
-	switch resolveAutoDiscoverMode(logger, cfg.NVIDIAContainerRuntimeConfig.DiscoverMode) {
-	case "legacy":
-		legacyDiscoverer, err := discover.NewLegacyDiscoverer(logger, config)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create legacy discoverer: %v", err)
-		}
-		d = legacyDiscoverer
-	case "csv":
-		csvFiles, err := csv.GetFileList(csv.DefaultMountSpecPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
-		}
-
-		nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
-		if nvidiaRequireJetpack != "csv-mounts=all" {
-			csvFiles = csv.BaseFilesOnly(csvFiles)
-		}
-
-		csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
-		}
-
-		ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
-		}
-
-		createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, config)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
-		}
-
-		d = discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
-	default:
-		return nil, fmt.Errorf("invalid discover mode: %v", cfg.NVIDIAContainerRuntimeConfig.DiscoverMode)
-	}
-
-	return newExperimentalModifierFromDiscoverer(logger, d)
-}
-
-// newExperimentalModifierFromDiscoverer created a modifier that aplies the discovered
-// modifications to an OCI spec if require by the runtime wrapper.
-func newExperimentalModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
-	m := experimental{
-		logger:     logger,
-		discoverer: d,
-	}
-	return &m, nil
-}
-
-// Modify applies the required modifications to the incomming OCI spec. These modifications
-// are applied in-place.
-func (m experimental) Modify(spec *specs.Spec) error {
-	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
-	if err != nil {
-		return fmt.Errorf("failed to remove existing hooks: %v", err)
-	}
-
-	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
-	if err != nil {
-		return fmt.Errorf("failed to get required container edits: %v", err)
-	}
-
-	return specEdits.Modify(spec)
-}
-
-// resolveAutoDiscoverMode determines the correct discover mode for the specified platform if set to "auto"
-func resolveAutoDiscoverMode(logger *logrus.Logger, mode string) (rmode string) {
-	if mode != "auto" {
-		return mode
-	}
-	defer func() {
-		logger.Infof("Auto-detected discover mode as '%v'", rmode)
-	}()
-
-	isTegra, reason := isTegraSystem()
-	logger.Debugf("Is Tegra-based system? %v: %v", isTegra, reason)
-
-	if isTegra {
-		return "csv"
-	}
-
-	return "legacy"
-}
-
-// isTegraSystem returns true if the system is detected as a Tegra-based system
-func isTegraSystem() (bool, string) {
-	const tegraReleaseFile = "/etc/nv_tegra_release"
-	const tegraFamilyFile = "/sys/devices/soc0/family"
-
-	if info, err := os.Stat(tegraReleaseFile); err == nil && !info.IsDir() {
-		return true, fmt.Sprintf("%v found", tegraReleaseFile)
-	}
-
-	if info, err := os.Stat(tegraFamilyFile); err != nil || !info.IsDir() {
-		return false, fmt.Sprintf("%v not found", tegraFamilyFile)
-	}
-
-	contents, err := os.ReadFile(tegraFamilyFile)
-	if err != nil {
-		return false, fmt.Sprintf("could not read %v", tegraFamilyFile)
-	}
-
-	if strings.HasPrefix(strings.ToLower(string(contents)), "tegra") {
-		return true, fmt.Sprintf("%v has 'tegra' prefix", tegraFamilyFile)
-	}
-
-	return false, fmt.Sprintf("%v has no 'tegra' prefix", tegraFamilyFile)
-}

cmd/nvidia-container-runtime/modifier/experimental_test.go

@@ -1,349 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNewExperimentalModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description    string
-		cfg            *config.Config
-		spec           oci.Spec
-		visibleDevices string
-		expectedError  error
-		expectedNil    bool
-	}{
-		{
-			description: "spec load error returns error",
-			spec: &oci.SpecMock{
-				LoadFunc: func() error {
-					return fmt.Errorf("load failed")
-				},
-			},
-			expectedError: fmt.Errorf("load failed"),
-		},
-		{
-			description:    "visible devices not set returns nil",
-			visibleDevices: "NOT_SET",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices empty returns nil",
-			visibleDevices: "",
-			expectedNil:    true,
-		},
-		{
-			description:    "visible devices 'void' returns nil",
-			visibleDevices: "void",
-			expectedNil:    true,
-		},
-		{
-			description: "empty config raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
-			},
-			visibleDevices: "all",
-			expectedError:  fmt.Errorf("invalid discover mode"),
-		},
-		{
-			description: "non-legacy discover mode raises error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					DiscoverMode: "non-legacy",
-				},
-			},
-			visibleDevices: "all",
-			expectedError:  fmt.Errorf("invalid discover mode"),
-		},
-		{
-			description: "legacy discover mode returns modifier",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					DiscoverMode: "legacy",
-				},
-			},
-			visibleDevices: "all",
-		},
-		{
-			description: "csv discover mode returns modifier",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					DiscoverMode: "csv",
-				},
-			},
-			visibleDevices: "all",
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			spec := tc.spec
-			if spec == nil {
-				spec = &oci.SpecMock{
-					LookupEnvFunc: func(s string) (string, bool) {
-						if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
-							return tc.visibleDevices, true
-						}
-						return "", false
-					},
-				}
-			}
-
-			m, err := NewExperimentalModifier(logger, tc.cfg, spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			if tc.expectedNil || tc.expectedError != nil {
-				require.Nil(t, m)
-			} else {
-				require.NotNil(t, m)
-			}
-		})
-	}
-}
-
-func TestExperimentalModifier(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description   string
-		discover      *discover.DiscoverMock
-		spec          *specs.Spec
-		expectedError error
-		expectedSpec  *specs.Spec
-	}{
-		{
-			description: "empty discoverer does not modify spec",
-			discover:    &discover.DiscoverMock{},
-		},
-		{
-			description: "failed hooks discoverer returns error",
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					return nil, fmt.Errorf("discover.Hooks error")
-				},
-			},
-			expectedError: fmt.Errorf("discover.Hooks error"),
-		},
-		{
-			description: "discovered hooks are injected into spec",
-			spec:        &specs.Spec{},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/a",
-							Args:      []string{"/hook/a", "arga"},
-						},
-						{
-							Lifecycle: "createContainer",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-					CreateContainer: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "existing hooks are maintained",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/a",
-							Args: []string{"/hook/a", "arga"},
-						},
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-runtime-hook",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-runtime-hook",
-							Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-		{
-			description: "modification removes existing nvidia-container-toolkit",
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/path/to/nvidia-container-toolkit",
-							Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
-						},
-					},
-				},
-			},
-			discover: &discover.DiscoverMock{
-				HooksFunc: func() ([]discover.Hook, error) {
-					hooks := []discover.Hook{
-						{
-							Lifecycle: "prestart",
-							Path:      "/hook/b",
-							Args:      []string{"/hook/b", "argb"},
-						},
-					}
-					return hooks, nil
-				},
-			},
-			expectedSpec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{
-						{
-							Path: "/hook/b",
-							Args: []string{"/hook/b", "argb"},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			m, err := newExperimentalModifierFromDiscoverer(logger, tc.discover)
-			require.NoError(t, err)
-
-			err = m.Modify(tc.spec)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			require.EqualValues(t, tc.expectedSpec, tc.spec)
-		})
-	}
-}
-
-func TestResolveDiscoverMode(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description  string
-		mode         string
-		expectedMode string
-	}{
-		{
-			description:  "non-auto resolves to input",
-			mode:         "not-auto",
-			expectedMode: "not-auto",
-		},
-		// TODO: The following test is brittle in that it will break on Tegra-based systems.
-		// {
-		// 	description:  "auto resolves to legacy",
-		// 	mode:         "auto",
-		// 	expectedMode: "legacy",
-		// },
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			mode := resolveAutoDiscoverMode(logger, tc.mode)
-			require.EqualValues(t, tc.expectedMode, mode)
-		})
-	}
-}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -1,70 +0,0 @@
-/*
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	dockerRuncExecutableName = "docker-runc"
-	runcExecutableName       = "runc"
-)
-
-// newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger
-func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) {
-	ociSpec, err := oci.NewSpec(logger, argv)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
-	}
-
-	lowLevelRuntimeCandidates := []string{dockerRuncExecutableName, runcExecutableName}
-	lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, lowLevelRuntimeCandidates)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
-	}
-
-	specModifier, err := newSpecModifier(logger, cfg, ociSpec)
-	if err != nil {
-		return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
-	}
-
-	// Create the wrapping runtime with the specified modifier
-	r := runtime.NewModifyingRuntimeWrapper(
-		logger,
-		lowLevelRuntime,
-		ociSpec,
-		specModifier,
-	)
-
-	return r, nil
-}
-
-// newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
-func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
-	if !cfg.NVIDIAContainerRuntimeConfig.Experimental {
-		return modifier.NewStableRuntimeModifier(logger), nil
-	}
-
-	return modifier.NewExperimentalModifier(logger, cfg, ociSpec)
-}

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -1,82 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"encoding/json"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestFactoryMethod(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-
-	testCases := []struct {
-		description   string
-		cfg           *config.Config
-		spec          *specs.Spec
-		expectedError bool
-	}{
-		{
-			description: "empty config no error",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
-			},
-		},
-		{
-			description: "experimental flag supported",
-			cfg: &config.Config{
-				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
-					Experimental: true,
-					DiscoverMode: "legacy",
-				},
-			},
-			spec: &specs.Spec{
-				Process: &specs.Process{
-					Env: []string{
-						"NVIDIA_VISIBLE_DEVICES=all",
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			bundleDir := t.TempDir()
-
-			specFile, err := os.Create(filepath.Join(bundleDir, "config.json"))
-			require.NoError(t, err)
-			require.NoError(t, json.NewEncoder(specFile).Encode(tc.spec))
-
-			argv := []string{"--bundle", bundleDir}
-
-			_, err = newNVIDIAContainerRuntime(logger, tc.cfg, argv)
-			if tc.expectedError {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-		})
-	}
-}

cmd/nvidia-container-toolkit/capabilities.go

@@ -1,83 +0,0 @@
-package main
-
-import (
-	"log"
-	"strings"
-)
-
-const (
-	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
-	defaultDriverCapabilities = DriverCapabilities("utility,compute")
-
-	none = DriverCapabilities("")
-	all  = DriverCapabilities("all")
-)
-
-func capabilityToCLI(cap string) string {
-	switch cap {
-	case "compute":
-		return "--compute"
-	case "compat32":
-		return "--compat32"
-	case "graphics":
-		return "--graphics"
-	case "utility":
-		return "--utility"
-	case "video":
-		return "--video"
-	case "display":
-		return "--display"
-	case "ngx":
-		return "--ngx"
-	default:
-		log.Panicln("unknown driver capability:", cap)
-	}
-	return ""
-}
-
-// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
-// variable. Operations include default values, filtering, and handling meta values such as "all"
-type DriverCapabilities string
-
-// Intersection returns intersection between two sets of capabilities.
-func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
-	if capabilities == all {
-		return d
-	}
-	if d == all {
-		return capabilities
-	}
-
-	lookup := make(map[string]bool)
-	for _, c := range d.list() {
-		lookup[c] = true
-	}
-	var found []string
-	for _, c := range capabilities.list() {
-		if lookup[c] {
-			found = append(found, c)
-		}
-	}
-
-	intersection := DriverCapabilities(strings.Join(found, ","))
-	return intersection
-}
-
-// String returns the string representation of the driver capabilities
-func (d DriverCapabilities) String() string {
-	return string(d)
-}
-
-// list returns the driver capabilities as a list
-func (d DriverCapabilities) list() []string {
-	var caps []string
-	for _, c := range strings.Split(string(d), ",") {
-		trimmed := strings.TrimSpace(c)
-		if len(trimmed) == 0 {
-			continue
-		}
-		caps = append(caps, trimmed)
-	}
-
-	return caps
-}

cmd/nvidia-container-toolkit/capabilities_test.go

@@ -1,134 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestDriverCapabilitiesIntersection(t *testing.T) {
-	testCases := []struct {
-		capabilities          DriverCapabilities
-		supportedCapabilities DriverCapabilities
-		expectedIntersection  DriverCapabilities
-	}{
-		{
-			capabilities:          none,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          all,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          allDriverCapabilities,
-			supportedCapabilities: all,
-			expectedIntersection:  allDriverCapabilities,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: all,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          none,
-			supportedCapabilities: DriverCapabilities("cap1"),
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("cap0,cap1"),
-			supportedCapabilities: DriverCapabilities("cap1,cap0"),
-			expectedIntersection:  DriverCapabilities("cap0,cap1"),
-		},
-		{
-			capabilities:          defaultDriverCapabilities,
-			supportedCapabilities: allDriverCapabilities,
-			expectedIntersection:  defaultDriverCapabilities,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-		{
-			capabilities:          DriverCapabilities("cap1"),
-			supportedCapabilities: none,
-			expectedIntersection:  none,
-		},
-		{
-			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
-			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
-			require.EqualValues(t, tc.expectedIntersection, intersection)
-		})
-	}
-}
-
-func TestDriverCapabilitiesList(t *testing.T) {
-	testCases := []struct {
-		capabilities DriverCapabilities
-		expected     []string
-	}{
-		{
-			capabilities: DriverCapabilities(""),
-		},
-		{
-			capabilities: DriverCapabilities("  "),
-		},
-		{
-			capabilities: DriverCapabilities(","),
-		},
-		{
-			capabilities: DriverCapabilities(",cap"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap,"),
-			expected:     []string{"cap"},
-		},
-		{
-			capabilities: DriverCapabilities("cap0,,cap1"),
-			expected:     []string{"cap0", "cap1"},
-		},
-		{
-			capabilities: DriverCapabilities("cap1,cap0,cap3"),
-			expected:     []string{"cap1", "cap0", "cap3"},
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			require.EqualValues(t, tc.expected, tc.capabilities.list())
-		})
-	}
-}

cmd/nvidia-container-toolkit/container_config.go

@@ -1,421 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"os"
-	"path"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	"golang.org/x/mod/semver"
-)
-
-var envSwarmGPU *string
-
-const (
-	envCUDAVersion          = "CUDA_VERSION"
-	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
-	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
-	envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"
-	envNVMigConfigDevices   = "NVIDIA_MIG_CONFIG_DEVICES"
-	envNVMigMonitorDevices  = "NVIDIA_MIG_MONITOR_DEVICES"
-	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
-)
-
-const (
-	capSysAdmin = "CAP_SYS_ADMIN"
-)
-
-const (
-	deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
-)
-
-type nvidiaConfig struct {
-	Devices            string
-	MigConfigDevices   string
-	MigMonitorDevices  string
-	DriverCapabilities string
-	Requirements       []string
-	DisableRequire     bool
-}
-
-type containerConfig struct {
-	Pid    int
-	Rootfs string
-	Env    map[string]string
-	Nvidia *nvidiaConfig
-}
-
-// Root from OCI runtime spec
-// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L94-L100
-type Root struct {
-	Path string `json:"path"`
-}
-
-// Process from OCI runtime spec
-// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L57
-type Process struct {
-	Env          []string         `json:"env,omitempty"`
-	Capabilities *json.RawMessage `json:"capabilities,omitempty" platform:"linux"`
-}
-
-// LinuxCapabilities from OCI runtime spec
-// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L61
-type LinuxCapabilities struct {
-	Bounding    []string `json:"bounding,omitempty" platform:"linux"`
-	Effective   []string `json:"effective,omitempty" platform:"linux"`
-	Inheritable []string `json:"inheritable,omitempty" platform:"linux"`
-	Permitted   []string `json:"permitted,omitempty" platform:"linux"`
-	Ambient     []string `json:"ambient,omitempty" platform:"linux"`
-}
-
-// Mount from OCI runtime spec
-// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L103
-type Mount struct {
-	Destination string   `json:"destination"`
-	Type        string   `json:"type,omitempty" platform:"linux,solaris"`
-	Source      string   `json:"source,omitempty"`
-	Options     []string `json:"options,omitempty"`
-}
-
-// Spec from OCI runtime spec
-// We use pointers to structs, similarly to the latest version of runtime-spec:
-// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L5-L28
-type Spec struct {
-	Version *string  `json:"ociVersion"`
-	Process *Process `json:"process,omitempty"`
-	Root    *Root    `json:"root,omitempty"`
-	Mounts  []Mount  `json:"mounts,omitempty"`
-}
-
-// HookState holds state information about the hook
-type HookState struct {
-	Pid int `json:"pid,omitempty"`
-	// After 17.06, runc is using the runtime spec:
-	// github.com/docker/runc/blob/17.06/libcontainer/configs/config.go#L262-L263
-	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/state.go#L3-L17
-	Bundle string `json:"bundle"`
-	// Before 17.06, runc used a custom struct that didn't conform to the spec:
-	// github.com/docker/runc/blob/17.03.x/libcontainer/configs/config.go#L245-L252
-	BundlePath string `json:"bundlePath"`
-}
-
-func parseCudaVersion(cudaVersion string) (vmaj, vmin, vpatch uint32) {
-	if _, err := fmt.Sscanf(cudaVersion, "%d.%d.%d\n", &vmaj, &vmin, &vpatch); err != nil {
-		vpatch = 0
-		if _, err := fmt.Sscanf(cudaVersion, "%d.%d\n", &vmaj, &vmin); err != nil {
-			vmin = 0
-			if _, err := fmt.Sscanf(cudaVersion, "%d\n", &vmaj); err != nil {
-				log.Panicln("invalid CUDA version:", cudaVersion)
-			}
-		}
-	}
-
-	return
-}
-
-func getEnvMap(e []string) (m map[string]string) {
-	m = make(map[string]string)
-	for _, s := range e {
-		p := strings.SplitN(s, "=", 2)
-		if len(p) != 2 {
-			log.Panicln("environment error")
-		}
-		m[p[0]] = p[1]
-	}
-	return
-}
-
-func loadSpec(path string) (spec *Spec) {
-	f, err := os.Open(path)
-	if err != nil {
-		log.Panicln("could not open OCI spec:", err)
-	}
-	defer f.Close()
-
-	if err = json.NewDecoder(f).Decode(&spec); err != nil {
-		log.Panicln("could not decode OCI spec:", err)
-	}
-	if spec.Version == nil {
-		log.Panicln("Version is empty in OCI spec")
-	}
-	if spec.Process == nil {
-		log.Panicln("Process is empty in OCI spec")
-	}
-	if spec.Root == nil {
-		log.Panicln("Root is empty in OCI spec")
-	}
-	return
-}
-
-func isPrivileged(s *Spec) bool {
-	if s.Process.Capabilities == nil {
-		return false
-	}
-
-	var caps []string
-	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
-	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
-	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
-	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
-	if (rc1cmp == 1 || rc1cmp == 0) && (rc5cmp == -1) {
-		err := json.Unmarshal(*s.Process.Capabilities, &caps)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
-		}
-		// Otherwise, parse s.Process.Capabilities as:
-		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	} else {
-		var lc LinuxCapabilities
-		err := json.Unmarshal(*s.Process.Capabilities, &lc)
-		if err != nil {
-			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
-		}
-		// We only make sure that the bounding capabibility set has
-		// CAP_SYS_ADMIN. This allows us to make sure that the container was
-		// actually started as '--privileged', but also allow non-root users to
-		// access the privileged NVIDIA capabilities.
-		caps = lc.Bounding
-	}
-
-	for _, c := range caps {
-		if c == capSysAdmin {
-			return true
-		}
-	}
-
-	return false
-}
-
-func isLegacyCUDAImage(env map[string]string) bool {
-	legacyCudaVersion := env[envCUDAVersion]
-	cudaRequire := env[envNVRequireCUDA]
-	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
-}
-
-func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
-	// Build a list of envvars to consider.
-	envVars := []string{envNVVisibleDevices}
-	if envSwarmGPU != nil {
-		// The Swarm envvar has higher precedence.
-		envVars = append([]string{*envSwarmGPU}, envVars...)
-	}
-
-	// Grab a reference to devices from the first envvar
-	// in the list that actually exists in the environment.
-	var devices *string
-	for _, envVar := range envVars {
-		if devs, ok := env[envVar]; ok {
-			devices = &devs
-			break
-		}
-	}
-
-	// Environment variable unset with legacy image: default to "all".
-	if devices == nil && legacyImage {
-		all := "all"
-		return &all
-	}
-
-	// Environment variable unset or empty or "void": return nil
-	if devices == nil || len(*devices) == 0 || *devices == "void" {
-		return nil
-	}
-
-	// Environment variable set to "none": reset to "".
-	if *devices == "none" {
-		empty := ""
-		return &empty
-	}
-
-	// Any other value.
-	return devices
-}
-
-func getDevicesFromMounts(mounts []Mount) *string {
-	var devices []string
-	for _, m := range mounts {
-		root := filepath.Clean(deviceListAsVolumeMountsRoot)
-		source := filepath.Clean(m.Source)
-		destination := filepath.Clean(m.Destination)
-
-		// Only consider mounts who's host volume is /dev/null
-		if source != "/dev/null" {
-			continue
-		}
-		// Only consider container mount points that begin with 'root'
-		if len(destination) < len(root) {
-			continue
-		}
-		if destination[:len(root)] != root {
-			continue
-		}
-		// Grab the full path beyond 'root' and add it to the list of devices
-		device := destination[len(root):]
-		if len(device) > 0 && device[0] == '/' {
-			device = device[1:]
-		}
-		if len(device) == 0 {
-			continue
-		}
-		devices = append(devices, device)
-	}
-
-	if devices == nil {
-		return nil
-	}
-
-	ret := strings.Join(devices, ",")
-	return &ret
-}
-
-func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
-	// If enabled, try and get the device list from volume mounts first
-	if hookConfig.AcceptDeviceListAsVolumeMounts {
-		devices := getDevicesFromMounts(mounts)
-		if devices != nil {
-			return devices
-		}
-	}
-
-	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(env, legacyImage)
-	if devices == nil {
-		return nil
-	}
-	if privileged || hookConfig.AcceptEnvvarUnprivileged {
-		return devices
-	}
-
-	configName := hookConfig.getConfigOption("AcceptEnvvarUnprivileged")
-	log.Printf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES (privileged=%v, %v=%v) ", privileged, configName, hookConfig.AcceptEnvvarUnprivileged)
-
-	return nil
-}
-
-func getMigConfigDevices(env map[string]string) *string {
-	if devices, ok := env[envNVMigConfigDevices]; ok {
-		return &devices
-	}
-	return nil
-}
-
-func getMigMonitorDevices(env map[string]string) *string {
-	if devices, ok := env[envNVMigMonitorDevices]; ok {
-		return &devices
-	}
-	return nil
-}
-
-func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
-	// We use the default driver capabilities by default. This is filtered to only include the
-	// supported capabilities
-	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
-
-	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
-
-	if !capsEnvSpecified && legacyImage {
-		// Environment variable unset with legacy image: set all capabilities.
-		return supportedDriverCapabilities
-	}
-
-	if capsEnvSpecified && len(capsEnv) > 0 {
-		// If the envvironment variable is specified and is non-empty, use the capabilities value
-		envCapabilities := DriverCapabilities(capsEnv)
-		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
-		if envCapabilities != all && capabilities != envCapabilities {
-			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
-		}
-	}
-
-	return capabilities
-}
-
-func getRequirements(env map[string]string, legacyImage bool) []string {
-	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
-	var requirements []string
-	for name, value := range env {
-		if strings.HasPrefix(name, envNVRequirePrefix) {
-			requirements = append(requirements, value)
-		}
-	}
-	if legacyImage {
-		vmaj, vmin, _ := parseCudaVersion(env[envCUDAVersion])
-		cudaRequire := fmt.Sprintf("cuda>=%d.%d", vmaj, vmin)
-		requirements = append(requirements, cudaRequire)
-	}
-	return requirements
-}
-
-func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool) *nvidiaConfig {
-	legacyImage := isLegacyCUDAImage(env)
-
-	var devices string
-	if d := getDevices(hookConfig, env, mounts, privileged, legacyImage); d != nil {
-		devices = *d
-	} else {
-		// 'nil' devices means this is not a GPU container.
-		return nil
-	}
-
-	var migConfigDevices string
-	if d := getMigConfigDevices(env); d != nil {
-		migConfigDevices = *d
-	}
-	if !privileged && migConfigDevices != "" {
-		log.Panicln("cannot set MIG_CONFIG_DEVICES in non privileged container")
-	}
-
-	var migMonitorDevices string
-	if d := getMigMonitorDevices(env); d != nil {
-		migMonitorDevices = *d
-	}
-	if !privileged && migMonitorDevices != "" {
-		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
-	}
-
-	driverCapabilities := getDriverCapabilities(env, hookConfig.SupportedDriverCapabilities, legacyImage).String()
-
-	requirements := getRequirements(env, legacyImage)
-
-	// Don't fail on invalid values.
-	disableRequire, _ := strconv.ParseBool(env[envNVDisableRequire])
-
-	return &nvidiaConfig{
-		Devices:            devices,
-		MigConfigDevices:   migConfigDevices,
-		MigMonitorDevices:  migMonitorDevices,
-		DriverCapabilities: driverCapabilities,
-		Requirements:       requirements,
-		DisableRequire:     disableRequire,
-	}
-}
-
-func getContainerConfig(hook HookConfig) (config containerConfig) {
-	var h HookState
-	d := json.NewDecoder(os.Stdin)
-	if err := d.Decode(&h); err != nil {
-		log.Panicln("could not decode container state:", err)
-	}
-
-	b := h.Bundle
-	if len(b) == 0 {
-		b = h.BundlePath
-	}
-
-	s := loadSpec(path.Join(b, "config.json"))
-
-	env := getEnvMap(s.Process.Env)
-	privileged := isPrivileged(s)
-	envSwarmGPU = hook.SwarmResource
-	return containerConfig{
-		Pid:    h.Pid,
-		Rootfs: s.Root.Path,
-		Env:    env,
-		Nvidia: getNvidiaConfig(&hook, env, s.Mounts, privileged),
-	}
-}

cmd/nvidia-container-toolkit/container_config_test.go

@@ -1,989 +0,0 @@
-package main
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetNvidiaConfig(t *testing.T) {
-	var tests = []struct {
-		description    string
-		env            map[string]string
-		privileged     bool
-		hookConfig     *HookConfig
-		expectedConfig *nvidiaConfig
-		expectedPanic  bool
-	}{
-		{
-			description:    "No environment, unprivileged",
-			env:            map[string]string{},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description:    "No environment, privileged",
-			env:            map[string]string{},
-			privileged:     true,
-			expectedConfig: nil,
-		},
-		{
-			description: "Legacy image, no devices, no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion: "9.0",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices 'all', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices 'empty', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Legacy image, devices 'void', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Legacy image, devices 'none', no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "none",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, no capabilities, no requirements",
-			env: map[string]string{
-				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "gpu0,gpu1",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities 'empty', no requirements",
-			env: map[string]string{
-				envCUDAVersion:          "9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities 'all', no requirements",
-			env: map[string]string{
-				envCUDAVersion:          "9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities set, no requirements",
-			env: map[string]string{
-				envCUDAVersion:          "9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "video,display",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities set, requirements set",
-			env: map[string]string{
-				envCUDAVersion:              "9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Legacy image, devices set, capabilities set, requirements set, disable requirements",
-			env: map[string]string{
-				envCUDAVersion:              "9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-				envNVDisableRequire:         "true",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
-			},
-		},
-		{
-			description: "Modern image, no devices, no capabilities, no requirements, no envCUDAVersion",
-			env: map[string]string{
-				envNVRequireCUDA: "cuda>=9.0",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, no devices, no capabilities, no requirement, envCUDAVersion set",
-			env: map[string]string{
-				envCUDAVersion:   "9.0",
-				envNVRequireCUDA: "cuda>=9.0",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, devices 'all', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'empty', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, devices 'void', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "",
-			},
-			privileged:     false,
-			expectedConfig: nil,
-		},
-		{
-			description: "Modern image, devices 'none', no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "none",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, no capabilities, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "gpu0,gpu1",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities 'empty', no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:        "cuda>=9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities 'all', no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:        "cuda>=9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "all",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities set, no requirements",
-			env: map[string]string{
-				envNVRequireCUDA:        "cuda>=9.0",
-				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "video,display",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities set, requirements set",
-			env: map[string]string{
-				envNVRequireCUDA:            "cuda>=9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices set, capabilities set, requirements set, disable requirements",
-			env: map[string]string{
-				envNVRequireCUDA:            "cuda>=9.0",
-				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "video,display",
-				envNVRequirePrefix + "REQ0": "req0=true",
-				envNVRequirePrefix + "REQ1": "req1=false",
-				envNVDisableRequire:         "true",
-			},
-			privileged: false,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "video,display",
-				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
-				DisableRequire:     true,
-			},
-		},
-		{
-			description: "No cuda envs, devices 'all'",
-			env: map[string]string{
-				envNVVisibleDevices: "all",
-			},
-			privileged: false,
-
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'all', migConfig set, privileged",
-			env: map[string]string{
-				envNVRequireCUDA:      "cuda>=9.0",
-				envNVVisibleDevices:   "all",
-				envNVMigConfigDevices: "mig0,mig1",
-			},
-			privileged: true,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				MigConfigDevices:   "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'all', migConfig set, unprivileged",
-			env: map[string]string{
-				envNVRequireCUDA:      "cuda>=9.0",
-				envNVVisibleDevices:   "all",
-				envNVMigConfigDevices: "mig0,mig1",
-			},
-			privileged:    false,
-			expectedPanic: true,
-		},
-		{
-			description: "Modern image, devices 'all', migMonitor set, privileged",
-			env: map[string]string{
-				envNVRequireCUDA:       "cuda>=9.0",
-				envNVVisibleDevices:    "all",
-				envNVMigMonitorDevices: "mig0,mig1",
-			},
-			privileged: true,
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				MigMonitorDevices:  "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-				Requirements:       []string{"cuda>=9.0"},
-				DisableRequire:     false,
-			},
-		},
-		{
-			description: "Modern image, devices 'all', migMonitor set, unprivileged",
-			env: map[string]string{
-				envNVRequireCUDA:       "cuda>=9.0",
-				envNVVisibleDevices:    "all",
-				envNVMigMonitorDevices: "mig0,mig1",
-			},
-			privileged:    false,
-			expectedPanic: true,
-		},
-		{
-			description: "Hook config set as driver-capabilities-all",
-			env: map[string]string{
-				envNVVisibleDevices:     "all",
-				envNVDriverCapabilities: "all",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SupportedDriverCapabilities: "video,display",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: "video,display",
-			},
-		},
-		{
-			description: "Hook config set, envvar sets driver-capabilities",
-			env: map[string]string{
-				envNVVisibleDevices:     "all",
-				envNVDriverCapabilities: "video,display",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SupportedDriverCapabilities: "video,display,compute,utility",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: "video,display",
-			},
-		},
-		{
-			description: "Hook config set, envvar unset sets default driver-capabilities",
-			env: map[string]string{
-				envNVVisibleDevices: "all",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SupportedDriverCapabilities: "video,display,utility,compute",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-			},
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			// Wrap the call to getNvidiaConfig() in a closure.
-			var config *nvidiaConfig
-			getConfig := func() {
-				hookConfig := tc.hookConfig
-				if hookConfig == nil {
-					defaultConfig := getDefaultHookConfig()
-					hookConfig = &defaultConfig
-				}
-				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
-			}
-
-			// For any tests that are expected to panic, make sure they do.
-			if tc.expectedPanic {
-				require.Panics(t, getConfig)
-				return
-			}
-
-			// For all other tests, just grab the config
-			getConfig()
-
-			// And start comparing the test results to the expected results.
-			if tc.expectedConfig == nil {
-				require.Nil(t, config, tc.description)
-				return
-			}
-
-			require.NotNil(t, config, tc.description)
-
-			require.Equal(t, tc.expectedConfig.Devices, config.Devices)
-			require.Equal(t, tc.expectedConfig.MigConfigDevices, config.MigConfigDevices)
-			require.Equal(t, tc.expectedConfig.MigMonitorDevices, config.MigMonitorDevices)
-			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
-
-			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
-			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
-		})
-	}
-}
-
-func TestGetDevicesFromMounts(t *testing.T) {
-	var tests = []struct {
-		description     string
-		mounts          []Mount
-		expectedDevices *string
-	}{
-		{
-			description:     "No mounts",
-			mounts:          nil,
-			expectedDevices: nil,
-		},
-		{
-			description: "Host path is not /dev/null",
-			mounts: []Mount{
-				{
-					Source:      "/not/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-			},
-			expectedDevices: nil,
-		},
-		{
-			description: "Container path is not prefixed by 'root'",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join("/other/prefix", "GPU0"),
-				},
-			},
-			expectedDevices: nil,
-		},
-		{
-			description: "Container path is only 'root'",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: deviceListAsVolumeMountsRoot,
-				},
-			},
-			expectedDevices: nil,
-		},
-		{
-			description: "Discover 2 devices",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			expectedDevices: &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description: "Discover 2 devices with slashes in the name",
-			mounts: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0-MIG0/0/1"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1-MIG0/0/1"),
-				},
-			},
-			expectedDevices: &[]string{"GPU0-MIG0/0/1,GPU1-MIG0/0/1"}[0],
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			devices := getDevicesFromMounts(tc.mounts)
-			require.Equal(t, tc.expectedDevices, devices)
-		})
-	}
-}
-
-func TestDeviceListSourcePriority(t *testing.T) {
-	var tests = []struct {
-		description        string
-		mountDevices       []Mount
-		envvarDevices      string
-		privileged         bool
-		acceptUnprivileged bool
-		acceptMounts       bool
-		expectedDevices    *string
-	}{
-		{
-			description: "Mount devices, unprivileged, no accept unprivileged",
-			mountDevices: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			envvarDevices:      "GPU2,GPU3",
-			privileged:         false,
-			acceptUnprivileged: false,
-			acceptMounts:       true,
-			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description:        "No mount devices, unprivileged, no accept unprivileged",
-			mountDevices:       nil,
-			envvarDevices:      "GPU0,GPU1",
-			privileged:         false,
-			acceptUnprivileged: false,
-			acceptMounts:       true,
-			expectedDevices:    nil,
-		},
-		{
-			description:        "No mount devices, privileged, no accept unprivileged",
-			mountDevices:       nil,
-			envvarDevices:      "GPU0,GPU1",
-			privileged:         true,
-			acceptUnprivileged: false,
-			acceptMounts:       true,
-			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description:        "No mount devices, unprivileged, accept unprivileged",
-			mountDevices:       nil,
-			envvarDevices:      "GPU0,GPU1",
-			privileged:         false,
-			acceptUnprivileged: true,
-			acceptMounts:       true,
-			expectedDevices:    &[]string{"GPU0,GPU1"}[0],
-		},
-		{
-			description: "Mount devices, unprivileged, accept unprivileged, no accept mounts",
-			mountDevices: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			envvarDevices:      "GPU2,GPU3",
-			privileged:         false,
-			acceptUnprivileged: true,
-			acceptMounts:       false,
-			expectedDevices:    &[]string{"GPU2,GPU3"}[0],
-		},
-		{
-			description: "Mount devices, unprivileged, no accept unprivileged, no accept mounts",
-			mountDevices: []Mount{
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
-				},
-				{
-					Source:      "/dev/null",
-					Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
-				},
-			},
-			envvarDevices:      "GPU2,GPU3",
-			privileged:         false,
-			acceptUnprivileged: false,
-			acceptMounts:       false,
-			expectedDevices:    nil,
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			// Wrap the call to getDevices() in a closure.
-			var devices *string
-			getDevices := func() {
-				env := map[string]string{
-					envNVVisibleDevices: tc.envvarDevices,
-				}
-				hookConfig := getDefaultHookConfig()
-				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
-				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
-			}
-
-			// For all other tests, just grab the devices and check the results
-			getDevices()
-
-			require.Equal(t, tc.expectedDevices, devices)
-		})
-	}
-}
-
-func TestGetDevicesFromEnvvar(t *testing.T) {
-	all := "all"
-	empty := ""
-	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
-	gpuID := "GPU-12345"
-	anotherGPUID := "GPU-67890"
-
-	var tests = []struct {
-		description     string
-		envSwarmGPU     *string
-		env             map[string]string
-		legacyImage     bool
-		expectedDevices *string
-	}{
-		{
-			description: "empty env returns nil for non-legacy image",
-		},
-		{
-			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: "",
-			},
-		},
-		{
-			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: "void",
-			},
-		},
-		{
-			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: "none",
-			},
-			expectedDevices: &empty,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: gpuID,
-			},
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
-			env: map[string]string{
-				envNVVisibleDevices: gpuID,
-			},
-			legacyImage:     true,
-			expectedDevices: &gpuID,
-		},
-		{
-			description:     "empty env returns all for legacy image",
-			legacyImage:     true,
-			expectedDevices: &all,
-		},
-		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
-		// not enabled
-		{
-			description: "missing NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envDockerResourceGPUs: anotherGPUID,
-			},
-		},
-		{
-			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   "",
-				envDockerResourceGPUs: anotherGPUID,
-			},
-		},
-		{
-			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   "void",
-				envDockerResourceGPUs: anotherGPUID,
-			},
-		},
-		{
-			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   "none",
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &empty,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   gpuID,
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
-			env: map[string]string{
-				envNVVisibleDevices:   gpuID,
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			legacyImage:     true,
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "empty env returns all for legacy image",
-			env: map[string]string{
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			legacyImage:     true,
-			expectedDevices: &all,
-		},
-		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
-		// enabled
-		{
-			description: "empty env returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-		},
-		{
-			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: "",
-			},
-		},
-		{
-			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: "void",
-			},
-		},
-		{
-			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: "none",
-			},
-			expectedDevices: &empty,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: gpuID,
-			},
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: gpuID,
-			},
-			legacyImage:     true,
-			expectedDevices: &gpuID,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS is selected if present",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &anotherGPUID,
-		},
-		{
-			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
-			envSwarmGPU: &envDockerResourceGPUs,
-			env: map[string]string{
-				envNVVisibleDevices:   gpuID,
-				envDockerResourceGPUs: anotherGPUID,
-			},
-			expectedDevices: &anotherGPUID,
-		},
-	}
-
-	for i, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
-			envSwarmGPU = tc.envSwarmGPU
-			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
-			if tc.expectedDevices == nil {
-				require.Nil(t, devices, "%d: %v", i, tc)
-				return
-			}
-
-			require.NotNil(t, devices, "%d: %v", i, tc)
-			require.Equal(t, *tc.expectedDevices, *devices, "%d: %v", i, tc)
-		})
-	}
-}
-
-func TestGetDriverCapabilities(t *testing.T) {
-
-	supportedCapabilities := "compute,utility,display,video"
-
-	testCases := []struct {
-		description           string
-		env                   map[string]string
-		legacyImage           bool
-		supportedCapabilities string
-		expectedPanic         bool
-		expectedCapabilities  string
-	}{
-		{
-			description: "Env is set for legacy image",
-			env: map[string]string{
-				envNVDriverCapabilities: "display,video",
-			},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  "display,video",
-		},
-		{
-			description: "Env is all for legacy image",
-			env: map[string]string{
-				envNVDriverCapabilities: "all",
-			},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  supportedCapabilities,
-		},
-		{
-			description: "Env is empty for legacy image",
-			env: map[string]string{
-				envNVDriverCapabilities: "",
-			},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
-		},
-		{
-			description:           "Env unset for legacy image is 'all'",
-			env:                   map[string]string{},
-			legacyImage:           true,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  supportedCapabilities,
-		},
-		{
-			description: "Env is set for modern image",
-			env: map[string]string{
-				envNVDriverCapabilities: "display,video",
-			},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  "display,video",
-		},
-		{
-			description:           "Env unset for modern image is default",
-			env:                   map[string]string{},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
-		},
-		{
-			description: "Env is all for modern image",
-			env: map[string]string{
-				envNVDriverCapabilities: "all",
-			},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  supportedCapabilities,
-		},
-		{
-			description: "Env is empty for modern image",
-			env: map[string]string{
-				envNVDriverCapabilities: "",
-			},
-			legacyImage:           false,
-			supportedCapabilities: supportedCapabilities,
-			expectedCapabilities:  defaultDriverCapabilities.String(),
-		},
-		{
-			description: "Invalid capabilities panic",
-			env: map[string]string{
-				envNVDriverCapabilities: "compute,utility",
-			},
-			supportedCapabilities: "not-compute,not-utility",
-			expectedPanic:         true,
-		},
-		{
-			description:           "Default is restricted for modern image",
-			legacyImage:           false,
-			supportedCapabilities: "compute",
-			expectedCapabilities:  "compute",
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			var capabilites DriverCapabilities
-
-			getDriverCapabilities := func() {
-				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
-				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
-			}
-
-			if tc.expectedPanic {
-				require.Panics(t, getDriverCapabilities)
-				return
-			}
-
-			getDriverCapabilities()
-			require.EqualValues(t, tc.expectedCapabilities, capabilites)
-		})
-	}
-}

cmd/nvidia-container-toolkit/hook_config.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"log"
-	"os"
-	"path"
-	"reflect"
-
-	"github.com/BurntSushi/toml"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-)
-
-const (
-	configPath = "/etc/nvidia-container-runtime/config.toml"
-	driverPath = "/run/nvidia/driver"
-)
-
-var defaultPaths = [...]string{
-	path.Join(driverPath, configPath),
-	configPath,
-}
-
-// CLIConfig : options for nvidia-container-cli.
-type CLIConfig struct {
-	Root        *string  `toml:"root"`
-	Path        *string  `toml:"path"`
-	Environment []string `toml:"environment"`
-	Debug       *string  `toml:"debug"`
-	Ldcache     *string  `toml:"ldcache"`
-	LoadKmods   bool     `toml:"load-kmods"`
-	NoPivot     bool     `toml:"no-pivot"`
-	NoCgroups   bool     `toml:"no-cgroups"`
-	User        *string  `toml:"user"`
-	Ldconfig    *string  `toml:"ldconfig"`
-}
-
-// HookConfig : options for the nvidia-container-toolkit.
-type HookConfig struct {
-	DisableRequire                 bool               `toml:"disable-require"`
-	SwarmResource                  *string            `toml:"swarm-resource"`
-	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
-	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
-
-	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
-}
-
-func getDefaultHookConfig() HookConfig {
-	return HookConfig{
-		DisableRequire:                 false,
-		SwarmResource:                  nil,
-		AcceptEnvvarUnprivileged:       true,
-		AcceptDeviceListAsVolumeMounts: false,
-		SupportedDriverCapabilities:    allDriverCapabilities,
-		NvidiaContainerCLI: CLIConfig{
-			Root:        nil,
-			Path:        nil,
-			Environment: []string{},
-			Debug:       nil,
-			Ldcache:     nil,
-			LoadKmods:   true,
-			NoPivot:     false,
-			NoCgroups:   false,
-			User:        nil,
-			Ldconfig:    nil,
-		},
-		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
-	}
-}
-
-func getHookConfig() (config HookConfig) {
-	var err error
-
-	if len(*configflag) > 0 {
-		config = getDefaultHookConfig()
-		_, err = toml.DecodeFile(*configflag, &config)
-		if err != nil {
-			log.Panicln("couldn't open configuration file:", err)
-		}
-	} else {
-		for _, p := range defaultPaths {
-			config = getDefaultHookConfig()
-			_, err = toml.DecodeFile(p, &config)
-			if err == nil {
-				break
-			} else if !os.IsNotExist(err) {
-				log.Panicln("couldn't open default configuration file:", err)
-			}
-		}
-	}
-
-	if config.SupportedDriverCapabilities == all {
-		config.SupportedDriverCapabilities = allDriverCapabilities
-	}
-	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
-	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
-		configName := config.getConfigOption("SupportedDriverCapabilities")
-		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
-	}
-
-	return config
-}
-
-// getConfigOption returns the toml config option associated with the
-// specified struct field.
-func (c HookConfig) getConfigOption(fieldName string) string {
-	t := reflect.TypeOf(c)
-	f, ok := t.FieldByName(fieldName)
-	if !ok {
-		return fieldName
-	}
-	v, ok := f.Tag.Lookup("toml")
-	if !ok {
-		return fieldName
-	}
-	return v
-}

cmd/nvidia-ctk-installer/container/README.md

@@ -15,28 +15,23 @@ docker setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`.
 
 Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
 
-**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
 
 | Flags                                                       | Installed Runtimes              | Default Runtime       |
 |-------------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--runtime-name nvidia`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--runtime-name NAME`                                       | `NAME`, `nvidia-experimental`   | `NAME`                |
-| `--runtime-name nvidia-experimental`                        | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
-| `--set-as-default`                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-name nvidia`                    | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-name NAME`                      | `NAME`, `nvidia-experimental`   | `NAME`                |
-| `--set-as-default --runtime-name nvidia-experimental`       | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
-| `--set-as-default=false`                                    | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default=false --runtime-name NAME`                | `NAME`, `nvidia-experimental`   | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia`              | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| **NONE SPECIFIED**                                          | `nvidia`                        | `nvidia`              |
+| `--runtime-name nvidia`                                     | `nvidia`                        | `nvidia`              |
+| `--runtime-name NAME`                                       | `NAME`                          | `NAME`                |
+| `--set-as-default`                                          | `nvidia`                        | `nvidia`              |
+| `--set-as-default --runtime-name nvidia`                    | `nvidia`                        | `nvidia`              |
+| `--set-as-default --runtime-name NAME`                      | `NAME`                          | `NAME`                |
+| `--set-as-default=false`                                    | `nvidia`                        | **NOT SET**           |
+| `--set-as-default=false --runtime-name NAME`                | `NAME`                          | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia`              | `nvidia`                        | **NOT SET**           |
 
 These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
 
@@ -48,7 +43,7 @@ containerd setup \
         /run/nvidia/toolkit
 ```
 
-Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`.
 
 Adding the `--set-as-default` flag as follows:
 ```bash
@@ -59,19 +54,15 @@ containerd setup \
 ```
 will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
 
-**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
-
 The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
 
 | Flags                                                  | Installed Runtime Classes       | Default Runtime Class |
 |--------------------------------------------------------|:--------------------------------|:----------------------|
-| **NONE SPECIFIED**                                     | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--runtime-class NAME`                                 | `NAME`, `nvidia-experimental`   | **NOT SET**           |
-| `--runtime-class nvidia`                               | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--runtime-class nvidia-experimental`                  | `nvidia`, `nvidia-experimental` | **NOT SET**           |
-| `--set-as-default`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-class NAME`                | `NAME`, `nvidia-experimental`   | `NAME`                |
-| `--set-as-default --runtime-class nvidia`              | `nvidia`, `nvidia-experimental` | `nvidia`              |
-| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| **NONE SPECIFIED**                                     | `nvidia`                        | **NOT SET**           |
+| `--runtime-class NAME`                                 | `NAME`                          | **NOT SET**           |
+| `--runtime-class nvidia`                               | `nvidia`                        | **NOT SET**           |
+| `--set-as-default`                                     | `nvidia`                        | `nvidia`              |
+| `--set-as-default --runtime-class NAME`                | `NAME`                          | `NAME`                |
+| `--set-as-default --runtime-class nvidia`              | `nvidia`                        | `nvidia`              |
 
 These combinations also hold for the environment variables that map to the command line flags.

cmd/nvidia-ctk-installer/container/container.go

@@ -0,0 +1,182 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package container
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+)
+
+const (
+	restartModeNone    = "none"
+	restartModeSignal  = "signal"
+	restartModeSystemd = "systemd"
+)
+
+// Options defines the shared options for the CLIs to configure containers runtimes.
+type Options struct {
+	Config string
+	Socket string
+	// ExecutablePath specifies the path to the container runtime executable.
+	// This is used to extract the current config, for example.
+	// If a HostRootMount is specified, this path is relative to the host root
+	// mount.
+	ExecutablePath string
+	// EnabledCDI indicates whether CDI should be enabled.
+	EnableCDI     bool
+	RuntimeName   string
+	RuntimeDir    string
+	SetAsDefault  bool
+	RestartMode   string
+	HostRootMount string
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context, o *Options) error {
+	if o.RuntimeDir != "" {
+		logrus.Debug("Runtime directory already set; ignoring arguments")
+		return nil
+	}
+
+	args := c.Args()
+
+	logrus.Infof("Parsing arguments: %v", args.Slice())
+	if c.NArg() != 1 {
+		return fmt.Errorf("incorrect number of arguments")
+	}
+
+	o.RuntimeDir = args.Get(0)
+
+	logrus.Infof("Successfully parsed arguments")
+
+	return nil
+}
+
+// Configure applies the options to the specified config
+func (o Options) Configure(cfg engine.Interface) error {
+	err := o.UpdateConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+	return o.flush(cfg)
+}
+
+// Unconfigure removes the options from the specified config
+func (o Options) Unconfigure(cfg engine.Interface) error {
+	err := o.RevertConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+	return o.flush(cfg)
+}
+
+// flush flushes the specified config to disk
+func (o Options) flush(cfg engine.Interface) error {
+	logrus.Infof("Flushing config to %v", o.Config)
+	n, err := cfg.Save(o.Config)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+	if n == 0 {
+		logrus.Infof("Config file is empty, removed")
+	}
+	return nil
+}
+
+// UpdateConfig updates the specified config to include the nvidia runtimes
+func (o Options) UpdateConfig(cfg engine.Interface) error {
+	runtimes := operator.GetRuntimes(
+		operator.WithNvidiaRuntimeName(o.RuntimeName),
+		operator.WithSetAsDefault(o.SetAsDefault),
+		operator.WithRoot(o.RuntimeDir),
+	)
+	for name, runtime := range runtimes {
+		err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault)
+		if err != nil {
+			return fmt.Errorf("failed to update runtime %q: %v", name, err)
+		}
+	}
+
+	if o.EnableCDI {
+		cfg.EnableCDI()
+	}
+
+	return nil
+}
+
+// RevertConfig reverts the specified config to remove the nvidia runtimes
+func (o Options) RevertConfig(cfg engine.Interface) error {
+	runtimes := operator.GetRuntimes(
+		operator.WithNvidiaRuntimeName(o.RuntimeName),
+		operator.WithSetAsDefault(o.SetAsDefault),
+		operator.WithRoot(o.RuntimeDir),
+	)
+	for name := range runtimes {
+		err := cfg.RemoveRuntime(name)
+		if err != nil {
+			return fmt.Errorf("failed to remove runtime %q: %v", name, err)
+		}
+	}
+
+	return nil
+}
+
+// Restart restarts the specified service
+func (o Options) Restart(service string, withSignal func(string) error) error {
+	switch o.RestartMode {
+	case restartModeNone:
+		logrus.Warningf("Skipping restart of %v due to --restart-mode=%v", service, o.RestartMode)
+		return nil
+	case restartModeSignal:
+		return withSignal(o.Socket)
+	case restartModeSystemd:
+		return o.SystemdRestart(service)
+	}
+
+	return fmt.Errorf("invalid restart mode specified: %v", o.RestartMode)
+}
+
+// SystemdRestart restarts the specified service using systemd
+func (o Options) SystemdRestart(service string) error {
+	var args []string
+	var msg string
+	if o.HostRootMount != "" {
+		msg = " on host"
+		args = append(args, "chroot", o.HostRootMount)
+	}
+	args = append(args, "systemctl", "restart", service)
+
+	logrus.Infof("Restarting %v%v using systemd: %v", service, msg, args)
+
+	//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error restarting %v using systemd: %v", service, err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/operator/operator.go

@@ -0,0 +1,128 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package operator
+
+import "path/filepath"
+
+const (
+	defaultRuntimeName = "nvidia"
+)
+
+// Runtime defines a runtime to be configured.
+// The path and whether the runtime is the default runtime can be specified
+type Runtime struct {
+	name         string
+	Path         string
+	SetAsDefault bool
+}
+
+// Runtimes defines a set of runtimes to be configured for use in the GPU Operator
+type Runtimes map[string]Runtime
+
+type config struct {
+	root              string
+	nvidiaRuntimeName string
+	setAsDefault      bool
+}
+
+// GetRuntimes returns the set of runtimes to be configured for use with the GPU Operator.
+func GetRuntimes(opts ...Option) Runtimes {
+	c := &config{}
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	if c.nvidiaRuntimeName == "" {
+		c.nvidiaRuntimeName = defaultRuntimeName
+	}
+
+	runtimes := make(Runtimes)
+	runtimes.add(c.nvidiaRuntime())
+
+	modes := []string{"cdi", "legacy"}
+	for _, mode := range modes {
+		runtimes.add(c.modeRuntime(mode))
+	}
+	return runtimes
+}
+
+// DefaultRuntimeName returns the name of the default runtime.
+func (r Runtimes) DefaultRuntimeName() string {
+	for _, runtime := range r {
+		if runtime.SetAsDefault {
+			return runtime.name
+		}
+	}
+	return ""
+}
+
+// Add a runtime to the set of runtimes.
+func (r Runtimes) add(runtime Runtime) {
+	r[runtime.name] = runtime
+}
+
+// nvidiaRuntime creates a runtime that corresponds to the nvidia runtime.
+// If name is equal to one of the predefined runtimes, `nvidia` is used as the runtime name instead.
+func (c config) nvidiaRuntime() Runtime {
+	predefinedRuntimes := map[string]struct{}{
+		"nvidia-cdi":    {},
+		"nvidia-legacy": {},
+	}
+	name := c.nvidiaRuntimeName
+	if _, isPredefinedRuntime := predefinedRuntimes[name]; isPredefinedRuntime {
+		name = defaultRuntimeName
+	}
+	return c.newRuntime(name, "nvidia-container-runtime")
+}
+
+// modeRuntime creates a runtime for the specified mode.
+func (c config) modeRuntime(mode string) Runtime {
+	return c.newRuntime("nvidia-"+mode, "nvidia-container-runtime."+mode)
+}
+
+// newRuntime creates a runtime based on the configuration
+func (c config) newRuntime(name string, binary string) Runtime {
+	return Runtime{
+		name:         name,
+		Path:         filepath.Join(c.root, binary),
+		SetAsDefault: c.setAsDefault && name == c.nvidiaRuntimeName,
+	}
+}
+
+// Option is a functional option for configuring set of runtimes.
+type Option func(*config)
+
+// WithRoot sets the root directory for the runtime binaries.
+func WithRoot(root string) Option {
+	return func(c *config) {
+		c.root = root
+	}
+}
+
+// WithNvidiaRuntimeName sets the name of the nvidia runtime.
+func WithNvidiaRuntimeName(name string) Option {
+	return func(c *config) {
+		c.nvidiaRuntimeName = name
+	}
+}
+
+// WithSetAsDefault sets the default runtime to the nvidia runtime.
+func WithSetAsDefault(set bool) Option {
+	return func(c *config) {
+		c.setAsDefault = set
+	}
+}

cmd/nvidia-ctk-installer/container/operator/operator_test.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package operator
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptions(t *testing.T) {
+	testCases := []struct {
+		setAsDefault           bool
+		nvidiaRuntimeName      string
+		expectedDefaultRuntime string
+		expectedRuntimes       Runtimes
+	}{
+		{
+			expectedRuntimes: Runtimes{
+				"nvidia": Runtime{
+					name: "nvidia",
+					Path: "/usr/bin/nvidia-container-runtime",
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:           true,
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimes: Runtimes{
+				"nvidia": Runtime{
+					name:         "nvidia",
+					Path:         "/usr/bin/nvidia-container-runtime",
+					SetAsDefault: true,
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:           true,
+			nvidiaRuntimeName:      "nvidia",
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimes: Runtimes{
+				"nvidia": Runtime{
+					name:         "nvidia",
+					Path:         "/usr/bin/nvidia-container-runtime",
+					SetAsDefault: true,
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:           true,
+			nvidiaRuntimeName:      "NAME",
+			expectedDefaultRuntime: "NAME",
+			expectedRuntimes: Runtimes{
+				"NAME": Runtime{
+					name:         "NAME",
+					Path:         "/usr/bin/nvidia-container-runtime",
+					SetAsDefault: true,
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+		{
+			setAsDefault:      false,
+			nvidiaRuntimeName: "NAME",
+			expectedRuntimes: Runtimes{
+				"NAME": Runtime{
+					name: "NAME",
+					Path: "/usr/bin/nvidia-container-runtime",
+				},
+				"nvidia-cdi": Runtime{
+					name: "nvidia-cdi",
+					Path: "/usr/bin/nvidia-container-runtime.cdi",
+				},
+				"nvidia-legacy": Runtime{
+					name: "nvidia-legacy",
+					Path: "/usr/bin/nvidia-container-runtime.legacy",
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			runtimes := GetRuntimes(
+				WithNvidiaRuntimeName(tc.nvidiaRuntimeName),
+				WithSetAsDefault(tc.setAsDefault),
+				WithRoot("/usr/bin"),
+			)
+
+			require.EqualValues(t, tc.expectedRuntimes, runtimes)
+			require.Equal(t, tc.expectedDefaultRuntime, runtimes.DefaultRuntimeName())
+		})
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/config_v1_test.go

@@ -0,0 +1,584 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package containerd
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		legacyConfig                 bool
+		setAsDefault                 bool
+		runtimeName                  string
+		expectedDefaultRuntimeName   interface{}
+		expectedDefaultRuntimeBinary interface{}
+	}{
+		{},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			runtimeName:                  "NAME",
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   "nvidia",
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			runtimeName:                  "NAME",
+			expectedDefaultRuntimeName:   "NAME",
+			expectedDefaultRuntimeBinary: nil,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName:  tc.runtimeName,
+				RuntimeDir:   runtimeDir,
+				SetAsDefault: tc.setAsDefault,
+			}
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithUseLegacyConfig(tc.legacyConfig),
+				containerd.WithConfigVersion(1),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			cfg := v1.(*containerd.ConfigV1)
+			defaultRuntimeName := cfg.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+
+			defaultRuntime := cfg.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"})
+			if tc.expectedDefaultRuntimeBinary == nil {
+				require.Nil(t, defaultRuntime)
+			} else {
+				require.NotNil(t, defaultRuntime)
+
+				expected, err := defaultRuntimeTomlConfigV1(tc.expectedDefaultRuntimeBinary.(string))
+				require.NoError(t, err)
+
+				configContents, _ := toml.Marshal(defaultRuntime.(*toml.Tree))
+				expectedContents, _ := toml.Marshal(expected)
+
+				require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, tc)
+			}
+		})
+	}
+}
+
+func TestUpdateV1Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":    "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithConfigVersion(1),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v1.String())
+		})
+	}
+}
+
+func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+										"Runtime":     "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(runcConfigMapV1("/runc-binary"))),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithConfigVersion(1),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v1.String())
+		})
+	}
+}
+
+func TestUpdateV1EnableCDI(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		enableCDI              bool
+		expectedEnableCDIValue interface{}
+	}{
+		{},
+		{
+			enableCDI:              false,
+			expectedEnableCDIValue: nil,
+		},
+		{
+			enableCDI:              true,
+			expectedEnableCDIValue: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v", tc.enableCDI), func(t *testing.T) {
+			o := &container.Options{
+				EnableCDI:   tc.enableCDI,
+				RuntimeName: "nvidia",
+				RuntimeDir:  runtimeDir,
+			}
+
+			cfg, err := toml.Empty.Load()
+			require.NoError(t, err)
+
+			v1 := &containerd.ConfigV1{
+				Logger:      logger,
+				Tree:        cfg,
+				RuntimeType: runtimeType,
+			}
+
+			err = o.UpdateConfig(v1)
+			require.NoError(t, err)
+
+			enableCDIValue := v1.GetPath([]string{"plugins", "cri", "containerd", "enable_cdi"})
+			require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue)
+		})
+	}
+}
+
+func TestRevertV1Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":        runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-cdi":    runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"),
+								"nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"),
+							},
+							"default_runtime":      defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"),
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: "nvidia",
+			}
+
+			expected, err := toml.TreeFromMap(tc.expected)
+			require.NoError(t, err)
+
+			v1, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(tc.config)),
+				containerd.WithRuntimeType(runtimeType),
+
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.RevertConfig(v1)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v1.String())
+		})
+	}
+}
+
+func defaultRuntimeTomlConfigV1(binary string) (*toml.Tree, error) {
+	return toml.TreeFromMap(defaultRuntimeV1(binary))
+}
+
+func defaultRuntimeV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+			"Runtime":    binary,
+		},
+	}
+}
+
+func runtimeMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+			"Runtime":    binary,
+		},
+	}
+}
+
+func runcConfigMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"plugins": map[string]interface{}{
+			"cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": map[string]interface{}{
+							"runtime_type":                    "runc_runtime_type",
+							"runtime_root":                    "runc_runtime_root",
+							"runtime_engine":                  "runc_runtime_engine",
+							"privileged_without_host_devices": true,
+							"options": map[string]interface{}{
+								"runc-option": "value",
+								"BinaryName":  binary,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go

@@ -0,0 +1,520 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package containerd
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	runtimeType = "runtime_type"
+)
+
+func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		setAsDefault               bool
+		runtimeName                string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{},
+		{
+			setAsDefault:               false,
+			runtimeName:                "nvidia",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               false,
+			runtimeName:                "NAME",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "nvidia",
+			expectedDefaultRuntimeName: "nvidia",
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName:  tc.runtimeName,
+				RuntimeDir:   runtimeDir,
+				SetAsDefault: tc.setAsDefault,
+			}
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			cfg := v2.(*containerd.Config)
+
+			defaultRuntimeName := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+		})
+	}
+}
+
+func TestUpdateV2Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runtime_type",
+									"runtime_root":                    "",
+									"runtime_engine":                  "",
+									"privileged_without_host_devices": false,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.Empty),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v2.String())
+		})
+	}
+
+}
+
+func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			runtimeName: "nvidia",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"nvidia": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			runtimeName: "NAME",
+			expectedConfig: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"runc": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/runc-binary",
+									},
+								},
+								"NAME": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime",
+									},
+								},
+								"nvidia-cdi": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.cdi",
+									},
+								},
+								"nvidia-legacy": map[string]interface{}{
+									"runtime_type":                    "runc_runtime_type",
+									"runtime_root":                    "runc_runtime_root",
+									"runtime_engine":                  "runc_runtime_engine",
+									"privileged_without_host_devices": true,
+									"container_annotations":           []string{"cdi.k8s.io/*"},
+									"options": map[string]interface{}{
+										"runc-option": "value",
+										"BinaryName":  "/test/runtime/dir/nvidia-container-runtime.legacy",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: tc.runtimeName,
+				RuntimeDir:  runtimeDir,
+			}
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(runcConfigMapV2("/runc-binary"))),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			expected, err := toml.TreeFromMap(tc.expectedConfig)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v2.String())
+		})
+	}
+}
+
+func TestUpdateV2ConfigEnableCDI(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		enableCDI              bool
+		expectedEnableCDIValue interface{}
+	}{
+		{},
+		{
+			enableCDI:              false,
+			expectedEnableCDIValue: nil,
+		},
+		{
+			enableCDI:              true,
+			expectedEnableCDIValue: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v", tc.enableCDI), func(t *testing.T) {
+			o := &container.Options{
+				EnableCDI:    tc.enableCDI,
+				RuntimeName:  "nvidia",
+				RuntimeDir:   runtimeDir,
+				SetAsDefault: false,
+			}
+
+			cfg, err := toml.LoadMap(map[string]interface{}{})
+			require.NoError(t, err)
+
+			v2 := &containerd.Config{
+				Logger:               logger,
+				Tree:                 cfg,
+				RuntimeType:          runtimeType,
+				CRIRuntimePluginName: "io.containerd.grpc.v1.cri",
+			}
+
+			err = o.UpdateConfig(v2)
+			require.NoError(t, err)
+
+			enableCDIValue := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_cdi"})
+			require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue)
+		})
+	}
+}
+
+func TestRevertV2Config(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+							},
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			o := &container.Options{
+				RuntimeName: "nvidia",
+			}
+
+			expected, err := toml.TreeFromMap(tc.expected)
+			require.NoError(t, err)
+
+			v2, err := containerd.New(
+				containerd.WithLogger(logger),
+				containerd.WithConfigSource(toml.FromMap(tc.config)),
+				containerd.WithRuntimeType(runtimeType),
+				containerd.WithContainerAnnotations("cdi.k8s.io/*"),
+			)
+			require.NoError(t, err)
+
+			err = o.RevertConfig(v2)
+			require.NoError(t, err)
+
+			require.Equal(t, expected.String(), v2.String())
+		})
+	}
+}
+
+func runtimeMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+		},
+	}
+}
+
+func runcConfigMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"version": 2,
+		"plugins": map[string]interface{}{
+			"io.containerd.grpc.v1.cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": map[string]interface{}{
+							"runtime_type":                    "runc_runtime_type",
+							"runtime_root":                    "runc_runtime_root",
+							"runtime_engine":                  "runc_runtime_engine",
+							"privileged_without_host_devices": true,
+							"options": map[string]interface{}{
+								"runc-option": "value",
+								"BinaryName":  binary,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go

@@ -0,0 +1,184 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package containerd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	Name = "containerd"
+
+	DefaultConfig      = "/etc/containerd/config.toml"
+	DefaultSocket      = "/run/containerd/containerd.sock"
+	DefaultRestartMode = "signal"
+
+	defaultRuntmeType = "io.containerd.runc.v2"
+)
+
+// Options stores the containerd-specific options
+type Options struct {
+	useLegacyConfig bool
+	runtimeType     string
+
+	ContainerRuntimeModesCDIAnnotationPrefixes cli.StringSlice
+
+	runtimeConfigOverrideJSON string
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.BoolFlag{
+			Name: "use-legacy-config",
+			Usage: "Specify whether a legacy (pre v1.3) config should be used. " +
+				"This ensures that a version 1 container config is created by default and that the " +
+				"containerd.runtimes.default_runtime config section is used to define the default " +
+				"runtime instead of container.default_runtime_name.",
+			Destination: &opts.useLegacyConfig,
+			EnvVars:     []string{"CONTAINERD_USE_LEGACY_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-type",
+			Usage:       "The runtime_type to use for the configured runtime classes",
+			Value:       defaultRuntmeType,
+			Destination: &opts.runtimeType,
+			EnvVars:     []string{"CONTAINERD_RUNTIME_TYPE"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "nvidia-container-runtime-modes.cdi.annotation-prefixes",
+			Destination: &opts.ContainerRuntimeModesCDIAnnotationPrefixes,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-config-override",
+			Destination: &opts.runtimeConfigOverrideJSON,
+			Usage:       "specify additional runtime options as a JSON string. The paths are relative to the runtime config.",
+			Value:       "{}",
+			EnvVars:     []string{"RUNTIME_CONFIG_OVERRIDE", "CONTAINERD_RUNTIME_CONFIG_OVERRIDE"},
+		},
+	}
+
+	return flags
+}
+
+// Setup updates a containerd configuration to include the nvidia-containerd-runtime and reloads it
+func Setup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o, co)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Configure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to configure containerd: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts a containerd configuration to remove the nvidia-containerd-runtime and reloads it
+func Cleanup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o, co)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Unconfigure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to unconfigure containerd: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// RestartContainerd restarts containerd depending on the value of restartModeFlag
+func RestartContainerd(o *container.Options) error {
+	return o.Restart("containerd", SignalContainerd)
+}
+
+// containerAnnotationsFromCDIPrefixes returns the container annotations to set for the given CDI prefixes.
+func (o *Options) containerAnnotationsFromCDIPrefixes() []string {
+	var annotations []string
+	for _, prefix := range o.ContainerRuntimeModesCDIAnnotationPrefixes.Value() {
+		annotations = append(annotations, prefix+"*")
+	}
+
+	return annotations
+}
+
+func (o *Options) runtimeConfigOverride() (map[string]interface{}, error) {
+	if o.runtimeConfigOverrideJSON == "" {
+		return nil, nil
+	}
+
+	runtimeOptions := make(map[string]interface{})
+	if err := json.Unmarshal([]byte(o.runtimeConfigOverrideJSON), &runtimeOptions); err != nil {
+		return nil, fmt.Errorf("failed to read %v as JSON: %w", o.runtimeConfigOverrideJSON, err)
+	}
+
+	return runtimeOptions, nil
+}
+
+func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error) {
+	cfg, err := getRuntimeConfig(o, co)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load containerd config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}
+
+func getRuntimeConfig(o *container.Options, co *Options) (engine.Interface, error) {
+	return containerd.New(
+		containerd.WithPath(o.Config),
+		containerd.WithConfigSource(
+			toml.LoadFirst(
+				containerd.CommandLineSource(o.HostRootMount, o.ExecutablePath),
+				toml.FromFile(o.Config),
+			),
+		),
+		containerd.WithRuntimeType(co.runtimeType),
+		containerd.WithUseLegacyConfig(co.useLegacyConfig),
+		containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
+	)
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd_linux.go

@@ -0,0 +1,113 @@
+/**
+# Copyright 2020-2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	socketMessageToGetPID = ""
+)
+
+// SignalContainerd sends a SIGHUP signal to the containerd daemon
+func SignalContainerd(socket string) error {
+	log.Infof("Sending SIGHUP signal to containerd")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'containerd' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warningf("Error signaling containerd, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warningf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled containerd")
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd_other.go

@@ -0,0 +1,29 @@
+//go:build !linux
+// +build !linux
+
+/**
+# Copyright 2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"errors"
+)
+
+// SignalContainerd is unsupported on non-linux platforms.
+func SignalContainerd(socket string) error {
+	return errors.New("SignalContainerd is unsupported on non-linux platforms")
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd_test.go

@@ -0,0 +1,72 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package containerd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRuntimeOptions(t *testing.T) {
+	testCases := []struct {
+		description   string
+		options       Options
+		expected      map[string]interface{}
+		expectedError error
+	}{
+		{
+			description: "empty is nil",
+		},
+		{
+			description: "empty json",
+			options: Options{
+				runtimeConfigOverrideJSON: "{}",
+			},
+			expected:      map[string]interface{}{},
+			expectedError: nil,
+		},
+		{
+			description: "SystemdCgroup is true",
+			options: Options{
+				runtimeConfigOverrideJSON: "{\"SystemdCgroup\": true}",
+			},
+			expected: map[string]interface{}{
+				"SystemdCgroup": true,
+			},
+			expectedError: nil,
+		},
+		{
+			description: "SystemdCgroup is false",
+			options: Options{
+				runtimeConfigOverrideJSON: "{\"SystemdCgroup\": false}",
+			},
+			expected: map[string]interface{}{
+				"SystemdCgroup": false,
+			},
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			runtimeOptions, err := tc.options.runtimeConfigOverride()
+			require.ErrorIs(t, tc.expectedError, err)
+			require.EqualValues(t, tc.expected, runtimeOptions)
+		})
+	}
+}

cmd/nvidia-ctk-installer/container/runtime/crio/crio.go

@@ -0,0 +1,210 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/ocihook"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
+)
+
+const (
+	Name = "crio"
+
+	defaultConfigMode = "hook"
+
+	// Hook-based settings
+	defaultHooksDir     = "/usr/share/containers/oci/hooks.d"
+	defaultHookFilename = "oci-nvidia-hook.json"
+
+	// Config-based settings
+	DefaultConfig      = "/etc/crio/crio.conf"
+	DefaultSocket      = "/var/run/crio/crio.sock"
+	DefaultRestartMode = "systemd"
+)
+
+// Options defines the cri-o specific options.
+type Options struct {
+	configMode string
+
+	// hook-specific options
+	hooksDir     string
+	hookFilename string
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "hooks-dir",
+			Usage:       "path to the cri-o hooks directory",
+			Value:       defaultHooksDir,
+			Destination: &opts.hooksDir,
+			EnvVars:     []string{"CRIO_HOOKS_DIR"},
+			DefaultText: defaultHooksDir,
+		},
+		&cli.StringFlag{
+			Name:        "hook-filename",
+			Usage:       "filename of the cri-o hook that will be created / removed in the hooks directory",
+			Value:       defaultHookFilename,
+			Destination: &opts.hookFilename,
+			EnvVars:     []string{"CRIO_HOOK_FILENAME"},
+			DefaultText: defaultHookFilename,
+		},
+		&cli.StringFlag{
+			Name:        "config-mode",
+			Usage:       "the configuration mode to use. One of [hook | config]",
+			Value:       defaultConfigMode,
+			Destination: &opts.configMode,
+			EnvVars:     []string{"CRIO_CONFIG_MODE"},
+		},
+	}
+
+	return flags
+}
+
+// Setup installs the prestart hook required to launch GPU-enabled containers
+func Setup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	switch co.configMode {
+	case "hook":
+		return setupHook(o, co)
+	case "config":
+		return setupConfig(o)
+	default:
+		return fmt.Errorf("invalid config-mode '%v'", co.configMode)
+	}
+}
+
+// setupHook installs the prestart hook required to launch GPU-enabled containers
+func setupHook(o *container.Options, co *Options) error {
+	log.Infof("Installing prestart hook")
+
+	hookPath := filepath.Join(co.hooksDir, co.hookFilename)
+	err := ocihook.CreateHook(hookPath, filepath.Join(o.RuntimeDir, config.NVIDIAContainerRuntimeHookExecutable))
+	if err != nil {
+		return fmt.Errorf("error creating hook: %v", err)
+	}
+
+	return nil
+}
+
+// setupConfig updates the cri-o config for the NVIDIA container runtime
+func setupConfig(o *container.Options) error {
+	log.Infof("Updating config file")
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Configure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to configure cri-o: %v", err)
+	}
+
+	err = RestartCrio(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart crio: %v", err)
+	}
+
+	return nil
+}
+
+// Cleanup removes the specified prestart hook
+func Cleanup(c *cli.Context, o *container.Options, co *Options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	switch co.configMode {
+	case "hook":
+		return cleanupHook(co)
+	case "config":
+		return cleanupConfig(o)
+	default:
+		return fmt.Errorf("invalid config-mode '%v'", co.configMode)
+	}
+}
+
+// cleanupHook removes the prestart hook
+func cleanupHook(co *Options) error {
+	log.Infof("Removing prestart hook")
+
+	hookPath := filepath.Join(co.hooksDir, co.hookFilename)
+	err := os.Remove(hookPath)
+	if err != nil {
+		return fmt.Errorf("error removing hook '%v': %v", hookPath, err)
+	}
+
+	return nil
+}
+
+// cleanupConfig removes the NVIDIA container runtime from the cri-o config
+func cleanupConfig(o *container.Options) error {
+	log.Infof("Reverting config file modifications")
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Unconfigure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to unconfigure cri-o: %v", err)
+	}
+
+	err = RestartCrio(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart crio: %v", err)
+	}
+
+	return nil
+}
+
+// RestartCrio restarts crio depending on the value of restartModeFlag
+func RestartCrio(o *container.Options) error {
+	return o.Restart("crio", func(string) error { return fmt.Errorf("supporting crio via signal is unsupported") })
+}
+
+func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load crio config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}
+
+func getRuntimeConfig(o *container.Options) (engine.Interface, error) {
+	return crio.New(
+		crio.WithPath(o.Config),
+		crio.WithConfigSource(
+			toml.LoadFirst(
+				crio.CommandLineSource(o.HostRootMount, o.ExecutablePath),
+				toml.FromFile(o.Config),
+			),
+		),
+	)
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker.go

@@ -0,0 +1,111 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
+)
+
+const (
+	Name = "docker"
+
+	DefaultConfig      = "/etc/docker/daemon.json"
+	DefaultSocket      = "/var/run/docker.sock"
+	DefaultRestartMode = "signal"
+)
+
+type Options struct{}
+
+func Flags(opts *Options) []cli.Flag {
+	return nil
+}
+
+// Setup updates docker configuration to include the nvidia runtime and reloads it
+func Setup(c *cli.Context, o *container.Options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Configure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to configure docker: %v", err)
+	}
+
+	err = RestartDocker(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart docker: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts docker configuration to remove the nvidia runtime and reloads it
+func Cleanup(c *cli.Context, o *container.Options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	cfg, err := getRuntimeConfig(o)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = o.Unconfigure(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to unconfigure docker: %v", err)
+	}
+
+	err = RestartDocker(o)
+	if err != nil {
+		return fmt.Errorf("unable to signal docker: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// RestartDocker restarts docker depending on the value of restartModeFlag
+func RestartDocker(o *container.Options) error {
+	return o.Restart("docker", SignalDocker)
+}
+
+func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
+	cfg, err := docker.New(
+		docker.WithPath(o.Config),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load docker config: %w", err)
+	}
+	return engine.GetBinaryPathsForRuntimes(cfg), nil
+}
+
+func getRuntimeConfig(o *container.Options) (engine.Interface, error) {
+	return docker.New(
+		docker.WithPath(o.Config),
+	)
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker_linux.go

@@ -0,0 +1,113 @@
+/**
+# Copyright 2021-2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	socketMessageToGetPID = "GET /info HTTP/1.0\r\n\r\n"
+)
+
+// SignalDocker sends a SIGHUP signal to docker daemon
+func SignalDocker(socket string) error {
+	log.Infof("Sending SIGHUP signal to docker")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'docker' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warningf("Error signaling docker, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warningf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled docker")
+
+	return nil
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker_other.go

@@ -0,0 +1,29 @@
+//go:build !linux
+// +build !linux
+
+/**
+# Copyright 2023 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package docker
+
+import (
+	"errors"
+)
+
+// SignalDocker is unsupported on non-linux platforms.
+func SignalDocker(socket string) error {
+	return errors.New("SignalDocker is unsupported on non-linux platforms")
+}

cmd/nvidia-ctk-installer/container/runtime/docker/docker_test.go

@@ -14,13 +14,16 @@
 # limitations under the License.
 */
 
-package main
+package docker
 
 import (
 	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/docker"
 )
 
 func TestUpdateConfigDefaultRuntime(t *testing.T) {
@@ -41,11 +44,6 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 			runtimeName:                "NAME",
 			expectedDefaultRuntimeName: "NAME",
 		},
-		{
-			setAsDefault:               true,
-			runtimeName:                "nvidia-experimental",
-			expectedDefaultRuntimeName: "nvidia-experimental",
-		},
 		{
 			setAsDefault:               true,
 			runtimeName:                "nvidia",
@@ -54,15 +52,15 @@ func TestUpdateConfigDefaultRuntime(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		o := &options{
-			setAsDefault: tc.setAsDefault,
-			runtimeName:  tc.runtimeName,
-			runtimeDir:   runtimeDir,
+		o := &container.Options{
+			RuntimeName:  tc.runtimeName,
+			RuntimeDir:   runtimeDir,
+			SetAsDefault: tc.setAsDefault,
 		}
 
-		config := map[string]interface{}{}
+		config := docker.Config(map[string]interface{}{})
 
-		err := UpdateConfig(config, o)
+		err := o.UpdateConfig(&config)
 		require.NoError(t, err, "%d: %v", i, tc)
 
 		defaultRuntimeName := config["default-runtime"]
@@ -74,7 +72,7 @@ func TestUpdateConfig(t *testing.T) {
 	const runtimeDir = "/test/runtime/dir"
 
 	testCases := []struct {
-		config         map[string]interface{}
+		config         docker.Config
 		setAsDefault   bool
 		runtimeName    string
 		expectedConfig map[string]interface{}
@@ -88,8 +86,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -105,25 +107,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
 					},
-				},
-			},
-		},
-		{
-			config:       map[string]interface{}{},
-			setAsDefault: false,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -145,8 +134,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -171,8 +164,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -191,28 +188,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
 						"args": []string{},
 					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"default-runtime": "runc",
-			},
-			setAsDefault: true,
-			runtimeName:  "nvidia-experimental",
-			expectedConfig: map[string]interface{}{
-				"default-runtime": "nvidia-experimental",
-				"runtimes": map[string]interface{}{
-					"nvidia": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime",
-						"args": []string{},
-					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -239,8 +220,12 @@ func TestUpdateConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -249,12 +234,15 @@ func TestUpdateConfig(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		options := &options{
-			setAsDefault: tc.setAsDefault,
-			runtimeName:  tc.runtimeName,
-			runtimeDir:   runtimeDir,
+		tc := tc
+
+		o := &container.Options{
+			RuntimeName:  tc.runtimeName,
+			RuntimeDir:   runtimeDir,
+			SetAsDefault: tc.setAsDefault,
 		}
-		err := UpdateConfig(tc.config, options)
+
+		err := o.UpdateConfig(&tc.config)
 		require.NoError(t, err, "%d: %v", i, tc)
 
 		configContent, err := json.MarshalIndent(tc.config, "", "    ")
@@ -269,7 +257,7 @@ func TestUpdateConfig(t *testing.T) {
 
 func TestRevertConfig(t *testing.T) {
 	testCases := []struct {
-		config         map[string]interface{}
+		config         docker.Config
 		expectedConfig map[string]interface{}
 	}{
 		{
@@ -290,7 +278,7 @@ func TestRevertConfig(t *testing.T) {
 		{
 			config: map[string]interface{}{
 				"runtimes": map[string]interface{}{
-					"nvidia-experimental": map[string]interface{}{
+					"nvidia": map[string]interface{}{
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
@@ -305,8 +293,12 @@ func TestRevertConfig(t *testing.T) {
 						"path": "/test/runtime/dir/nvidia-container-runtime",
 						"args": []string{},
 					},
-					"nvidia-experimental": map[string]interface{}{
-						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+					"nvidia-cdi": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.cdi",
+						"args": []string{},
+					},
+					"nvidia-legacy": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime.legacy",
 						"args": []string{},
 					},
 				},
@@ -368,7 +360,9 @@ func TestRevertConfig(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		err := RevertConfig(tc.config)
+		tc := tc
+		o := &container.Options{}
+		err := o.RevertConfig(&tc.config)
 
 		require.NoError(t, err, "%d: %v", i, tc)
 
@@ -381,43 +375,3 @@ func TestRevertConfig(t *testing.T) {
 		require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
 	}
 }
-
-func TestFlagsDefaultRuntime(t *testing.T) {
-	testCases := []struct {
-		setAsDefault bool
-		runtimeName  string
-		expected     string
-	}{
-		{
-			expected: "",
-		},
-		{
-			runtimeName: "not-bool",
-			expected:    "",
-		},
-		{
-			setAsDefault: false,
-			runtimeName:  "nvidia",
-			expected:     "",
-		},
-		{
-			setAsDefault: true,
-			runtimeName:  "nvidia",
-			expected:     "nvidia",
-		},
-		{
-			setAsDefault: true,
-			runtimeName:  "nvidia-experimental",
-			expected:     "nvidia-experimental",
-		},
-	}
-
-	for i, tc := range testCases {
-		f := options{
-			setAsDefault: tc.setAsDefault,
-			runtimeName:  tc.runtimeName,
-		}
-
-		require.Equal(t, tc.expected, f.getDefaultRuntime(), "%d: %v", i, tc)
-	}
-}

cmd/nvidia-ctk-installer/container/runtime/runtime.go

@@ -0,0 +1,204 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime/containerd"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime/docker"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/toolkit"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+const (
+	defaultSetAsDefault = true
+	// defaultRuntimeName specifies the NVIDIA runtime to be use as the default runtime if setting the default runtime is enabled
+	defaultRuntimeName   = "nvidia"
+	defaultHostRootMount = "/host"
+
+	runtimeSpecificDefault = "RUNTIME_SPECIFIC_DEFAULT"
+)
+
+type Options struct {
+	container.Options
+
+	containerdOptions containerd.Options
+	crioOptions       crio.Options
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "Path to the runtime config file",
+			Value:       runtimeSpecificDefault,
+			Destination: &opts.Config,
+			EnvVars:     []string{"RUNTIME_CONFIG", "CONTAINERD_CONFIG", "DOCKER_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "executable-path",
+			Usage:       "The path to the runtime executable. This is used to extract the current config",
+			Destination: &opts.ExecutablePath,
+			EnvVars:     []string{"RUNTIME_EXECUTABLE_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "socket",
+			Usage:       "Path to the runtime socket file",
+			Value:       runtimeSpecificDefault,
+			Destination: &opts.Socket,
+			EnvVars:     []string{"RUNTIME_SOCKET", "CONTAINERD_SOCKET", "DOCKER_SOCKET"},
+		},
+		&cli.StringFlag{
+			Name:        "restart-mode",
+			Usage:       "Specify how the runtime should be restarted; If 'none' is selected it will not be restarted [signal | systemd | none ]",
+			Value:       runtimeSpecificDefault,
+			Destination: &opts.RestartMode,
+			EnvVars:     []string{"RUNTIME_RESTART_MODE"},
+		},
+		&cli.BoolFlag{
+			Name:        "enable-cdi-in-runtime",
+			Usage:       "Enable CDI in the configured runt	ime",
+			Destination: &opts.EnableCDI,
+			EnvVars:     []string{"RUNTIME_ENABLE_CDI"},
+		},
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "Specify the path to the host root to be used when restarting the runtime using systemd",
+			Value:       defaultHostRootMount,
+			Destination: &opts.HostRootMount,
+			EnvVars:     []string{"HOST_ROOT_MOUNT"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-name",
+			Aliases:     []string{"nvidia-runtime-name", "runtime-class"},
+			Usage:       "Specify the name of the `nvidia` runtime. If set-as-default is selected, the runtime is used as the default runtime.",
+			Value:       defaultRuntimeName,
+			Destination: &opts.RuntimeName,
+			EnvVars:     []string{"NVIDIA_RUNTIME_NAME", "CONTAINERD_RUNTIME_CLASS", "DOCKER_RUNTIME_NAME"},
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Usage:       "Set the `nvidia` runtime as the default runtime.",
+			Value:       defaultSetAsDefault,
+			Destination: &opts.SetAsDefault,
+			EnvVars:     []string{"NVIDIA_RUNTIME_SET_AS_DEFAULT", "CONTAINERD_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+	}
+
+	flags = append(flags, containerd.Flags(&opts.containerdOptions)...)
+	flags = append(flags, crio.Flags(&opts.crioOptions)...)
+
+	return flags
+}
+
+// Validate checks whether the specified options are valid
+func (opts *Options) Validate(logger logger.Interface, c *cli.Context, runtime string, toolkitRoot string, to *toolkit.Options) error {
+	// We set this option here to ensure that it is available in future calls.
+	opts.RuntimeDir = toolkitRoot
+
+	if !c.IsSet("enable-cdi-in-runtime") {
+		opts.EnableCDI = to.CDI.Enabled
+	}
+
+	if opts.ExecutablePath != "" && opts.RuntimeName == docker.Name {
+		logger.Warningf("Ignoring executable-path=%q flag for %v", opts.ExecutablePath, opts.RuntimeName)
+		opts.ExecutablePath = ""
+	}
+
+	// Apply the runtime-specific config changes.
+	switch runtime {
+	case containerd.Name:
+		if opts.Config == runtimeSpecificDefault {
+			opts.Config = containerd.DefaultConfig
+		}
+		if opts.Socket == runtimeSpecificDefault {
+			opts.Socket = containerd.DefaultSocket
+		}
+		if opts.RestartMode == runtimeSpecificDefault {
+			opts.RestartMode = containerd.DefaultRestartMode
+		}
+	case crio.Name:
+		if opts.Config == runtimeSpecificDefault {
+			opts.Config = crio.DefaultConfig
+		}
+		if opts.Socket == runtimeSpecificDefault {
+			opts.Socket = crio.DefaultSocket
+		}
+		if opts.RestartMode == runtimeSpecificDefault {
+			opts.RestartMode = crio.DefaultRestartMode
+		}
+	case docker.Name:
+		if opts.Config == runtimeSpecificDefault {
+			opts.Config = docker.DefaultConfig
+		}
+		if opts.Socket == runtimeSpecificDefault {
+			opts.Socket = docker.DefaultSocket
+		}
+		if opts.RestartMode == runtimeSpecificDefault {
+			opts.RestartMode = docker.DefaultRestartMode
+		}
+	default:
+		return fmt.Errorf("undefined runtime %v", runtime)
+	}
+
+	return nil
+}
+
+func Setup(c *cli.Context, opts *Options, runtime string) error {
+	switch runtime {
+	case containerd.Name:
+		return containerd.Setup(c, &opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.Setup(c, &opts.Options, &opts.crioOptions)
+	case docker.Name:
+		return docker.Setup(c, &opts.Options)
+	default:
+		return fmt.Errorf("undefined runtime %v", runtime)
+	}
+}
+
+func Cleanup(c *cli.Context, opts *Options, runtime string) error {
+	switch runtime {
+	case containerd.Name:
+		return containerd.Cleanup(c, &opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.Cleanup(c, &opts.Options, &opts.crioOptions)
+	case docker.Name:
+		return docker.Cleanup(c, &opts.Options)
+	default:
+		return fmt.Errorf("undefined runtime %v", runtime)
+	}
+}
+
+func GetLowlevelRuntimePaths(opts *Options, runtime string) ([]string, error) {
+	switch runtime {
+	case containerd.Name:
+		return containerd.GetLowlevelRuntimePaths(&opts.Options, &opts.containerdOptions)
+	case crio.Name:
+		return crio.GetLowlevelRuntimePaths(&opts.Options)
+	case docker.Name:
+		return docker.GetLowlevelRuntimePaths(&opts.Options)
+	default:
+		return nil, fmt.Errorf("undefined runtime %v", runtime)
+	}
+}

cmd/nvidia-ctk-installer/main.go

@@ -0,0 +1,328 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/sys/unix"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/toolkit"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+const (
+	toolkitPidFilename = "toolkit.pid"
+	defaultPidFile     = "/run/nvidia/toolkit/" + toolkitPidFilename
+
+	defaultToolkitInstallDir = "/usr/local/nvidia"
+	toolkitSubDir            = "toolkit"
+
+	defaultRuntime = "docker"
+)
+
+var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}}
+var defaultLowLevelRuntimes = []string{"runc", "crun"}
+
+var waitingForSignal = make(chan bool, 1)
+var signalReceived = make(chan bool, 1)
+
+// options stores the command line arguments
+type options struct {
+	toolkitInstallDir string
+
+	noDaemon    bool
+	runtime     string
+	pidFile     string
+	sourceRoot  string
+	packageType string
+
+	toolkitOptions toolkit.Options
+	runtimeOptions runtime.Options
+}
+
+func (o options) toolkitRoot() string {
+	return filepath.Join(o.toolkitInstallDir, toolkitSubDir)
+}
+
+func main() {
+	logger := logger.New()
+	c := NewApp(logger)
+
+	// Run the CLI
+	logger.Infof("Starting %v", c.Name)
+	if err := c.Run(os.Args); err != nil {
+		logger.Errorf("error running %v: %v", c.Name, err)
+		os.Exit(1)
+	}
+
+	logger.Infof("Completed %v", c.Name)
+}
+
+// An app represents the nvidia-ctk-installer.
+type app struct {
+	logger logger.Interface
+
+	toolkit *toolkit.Installer
+}
+
+// NewApp creates the CLI app fro the specified options.
+func NewApp(logger logger.Interface) *cli.App {
+	a := app{
+		logger: logger,
+	}
+	return a.build()
+}
+
+func (a app) build() *cli.App {
+	options := options{
+		toolkitOptions: toolkit.Options{},
+	}
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "nvidia-ctk-installer"
+	c.Usage = "Install the NVIDIA Container Toolkit and configure the specified runtime to use the `nvidia` runtime."
+	c.Version = info.GetVersionString()
+	c.Before = func(ctx *cli.Context) error {
+		return a.Before(ctx, &options)
+	}
+	c.Action = func(ctx *cli.Context) error {
+		return a.Run(ctx, &options)
+	}
+
+	// Setup flags for the CLI
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "no-daemon",
+			Aliases:     []string{"n"},
+			Usage:       "terminate immediately after setting up the runtime. Note that no cleanup will be performed",
+			Destination: &options.noDaemon,
+			EnvVars:     []string{"NO_DAEMON"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Aliases:     []string{"r"},
+			Usage:       "the runtime to setup on this node. One of {'docker', 'crio', 'containerd'}",
+			Value:       defaultRuntime,
+			Destination: &options.runtime,
+			EnvVars:     []string{"RUNTIME"},
+		},
+		&cli.StringFlag{
+			Name:    "toolkit-install-dir",
+			Aliases: []string{"root"},
+			Usage: "The directory where the NVIDIA Container Toolkit is to be installed. " +
+				"The components of the toolkit will be installed to `ROOT`/toolkit. " +
+				"Note that in the case of a containerized installer, this is the path in the container and it is " +
+				"recommended that this match the path on the host.",
+			Value:       defaultToolkitInstallDir,
+			Destination: &options.toolkitInstallDir,
+			EnvVars:     []string{"TOOLKIT_INSTALL_DIR", "ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-source-root",
+			Usage:       "The folder where the required toolkit artifacts can be found. If this is not specified, the path /artifacts/{{ .ToolkitPackageType }} is used where ToolkitPackageType is the resolved package type",
+			Destination: &options.sourceRoot,
+			EnvVars:     []string{"TOOLKIT_SOURCE_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-package-type",
+			Usage:       "specify the package type to use for the toolkit. One of ['deb', 'rpm', 'auto', '']. If 'auto' or '' are used, the type is inferred automatically.",
+			Value:       "auto",
+			Destination: &options.packageType,
+			EnvVars:     []string{"TOOLKIT_PACKAGE_TYPE"},
+		},
+		&cli.StringFlag{
+			Name:        "pid-file",
+			Value:       defaultPidFile,
+			Usage:       "the path to a toolkit.pid file to ensure that only a single configuration instance is running",
+			Destination: &options.pidFile,
+			EnvVars:     []string{"TOOLKIT_PID_FILE", "PID_FILE"},
+		},
+	}
+
+	c.Flags = append(c.Flags, toolkit.Flags(&options.toolkitOptions)...)
+	c.Flags = append(c.Flags, runtime.Flags(&options.runtimeOptions)...)
+
+	return c
+}
+
+func (a *app) Before(c *cli.Context, o *options) error {
+	if o.sourceRoot == "" {
+		sourceRoot, err := a.resolveSourceRoot(o.runtimeOptions.HostRootMount, o.packageType)
+		if err != nil {
+			return fmt.Errorf("failed to resolve source root: %v", err)
+		}
+		a.logger.Infof("Resolved source root to %v", sourceRoot)
+		o.sourceRoot = sourceRoot
+	}
+
+	a.toolkit = toolkit.NewInstaller(
+		toolkit.WithLogger(a.logger),
+		toolkit.WithSourceRoot(o.sourceRoot),
+		toolkit.WithToolkitRoot(o.toolkitRoot()),
+	)
+	return a.validateFlags(c, o)
+}
+
+func (a *app) validateFlags(c *cli.Context, o *options) error {
+	if o.toolkitInstallDir == "" {
+		return fmt.Errorf("the install root must be specified")
+	}
+	if _, exists := availableRuntimes[o.runtime]; !exists {
+		return fmt.Errorf("unknown runtime: %v", o.runtime)
+	}
+	if filepath.Base(o.pidFile) != toolkitPidFilename {
+		return fmt.Errorf("invalid toolkit.pid path %v", o.pidFile)
+	}
+
+	if err := a.toolkit.ValidateOptions(&o.toolkitOptions); err != nil {
+		return err
+	}
+	if err := o.runtimeOptions.Validate(a.logger, c, o.runtime, o.toolkitRoot(), &o.toolkitOptions); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Run installs the NVIDIA Container Toolkit and updates the requested runtime.
+// If the application is run as a daemon, the application waits and unconfigures
+// the runtime on termination.
+func (a *app) Run(c *cli.Context, o *options) error {
+	err := a.initialize(o.pidFile)
+	if err != nil {
+		return fmt.Errorf("unable to initialize: %v", err)
+	}
+	defer a.shutdown(o.pidFile)
+
+	if len(o.toolkitOptions.ContainerRuntimeRuntimes.Value()) == 0 {
+		lowlevelRuntimePaths, err := runtime.GetLowlevelRuntimePaths(&o.runtimeOptions, o.runtime)
+		if err != nil {
+			return fmt.Errorf("unable to determine runtime options: %w", err)
+		}
+		lowlevelRuntimePaths = append(lowlevelRuntimePaths, defaultLowLevelRuntimes...)
+
+		o.toolkitOptions.ContainerRuntimeRuntimes = *cli.NewStringSlice(lowlevelRuntimePaths...)
+	}
+
+	err = a.toolkit.Install(c, &o.toolkitOptions)
+	if err != nil {
+		return fmt.Errorf("unable to install toolkit: %v", err)
+	}
+
+	err = runtime.Setup(c, &o.runtimeOptions, o.runtime)
+	if err != nil {
+		return fmt.Errorf("unable to setup runtime: %v", err)
+	}
+
+	if !o.noDaemon {
+		err = a.waitForSignal()
+		if err != nil {
+			return fmt.Errorf("unable to wait for signal: %v", err)
+		}
+
+		err = runtime.Cleanup(c, &o.runtimeOptions, o.runtime)
+		if err != nil {
+			return fmt.Errorf("unable to cleanup runtime: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (a *app) initialize(pidFile string) error {
+	a.logger.Infof("Initializing")
+
+	if dir := filepath.Dir(pidFile); dir != "" {
+		err := os.MkdirAll(dir, 0755)
+		if err != nil {
+			return fmt.Errorf("unable to create folder for pidfile: %w", err)
+		}
+	}
+
+	f, err := os.Create(pidFile)
+	if err != nil {
+		return fmt.Errorf("unable to create pidfile: %v", err)
+	}
+
+	err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	if err != nil {
+		a.logger.Warningf("Unable to get exclusive lock on '%v'", pidFile)
+		a.logger.Warningf("This normally means an instance of the NVIDIA toolkit Container is already running, aborting")
+		return fmt.Errorf("unable to get flock on pidfile: %v", err)
+	}
+
+	_, err = fmt.Fprintf(f, "%v\n", os.Getpid())
+	if err != nil {
+		return fmt.Errorf("unable to write PID to pidfile: %v", err)
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGPIPE, syscall.SIGTERM)
+	go func() {
+		<-sigs
+		select {
+		case <-waitingForSignal:
+			signalReceived <- true
+		default:
+			a.logger.Infof("Signal received, exiting early")
+			a.shutdown(pidFile)
+			os.Exit(0)
+		}
+	}()
+
+	return nil
+}
+
+func (a *app) waitForSignal() error {
+	a.logger.Infof("Waiting for signal")
+	waitingForSignal <- true
+	<-signalReceived
+	return nil
+}
+
+func (a *app) shutdown(pidFile string) {
+	a.logger.Infof("Shutting Down")
+
+	err := os.Remove(pidFile)
+	if err != nil {
+		a.logger.Warningf("Unable to remove pidfile: %v", err)
+	}
+}
+
+func (a *app) resolveSourceRoot(hostRoot string, packageType string) (string, error) {
+	resolvedPackageType, err := a.resolvePackageType(hostRoot, packageType)
+	if err != nil {
+		return "", err
+	}
+	switch resolvedPackageType {
+	case "deb":
+		return "/artifacts/deb", nil
+	case "rpm":
+		return "/artifacts/rpm", nil
+	default:
+		return "", fmt.Errorf("invalid package type: %v", resolvedPackageType)
+	}
+}
+
+func (a *app) resolvePackageType(hostRoot string, packageType string) (rPackageTypes string, rerr error) {
+	if packageType != "" && packageType != "auto" {
+		return packageType, nil
+	}
+
+	locator := lookup.NewExecutableLocator(a.logger, hostRoot)
+	if candidates, err := locator.Locate("/usr/bin/rpm"); err == nil && len(candidates) > 0 {
+		return "rpm", nil
+	}
+
+	if candidates, err := locator.Locate("/usr/bin/dpkg"); err == nil && len(candidates) > 0 {
+		return "deb", nil
+	}
+
+	return "deb", nil
+}

cmd/nvidia-ctk-installer/main_test.go

@@ -0,0 +1,455 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestApp(t *testing.T) {
+	t.Setenv("__NVCT_TESTING_DEVICES_ARE_FILES", "true")
+	logger, _ := testlog.NewNullLogger()
+
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	artifactRoot := filepath.Join(moduleRoot, "testdata", "installer", "artifacts")
+	hostRoot := filepath.Join(moduleRoot, "testdata", "lookup", "rootfs-1")
+
+	testCases := []struct {
+		description           string
+		args                  []string
+		expectedToolkitConfig string
+		expectedRuntimeConfig string
+	}{
+		{
+			description: "no args",
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+    [nvidia-container-runtime.modes.legacy]
+      cuda-compat-mode = "ldconfig"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+        },
+        "nvidia-cdi": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+        },
+        "nvidia-legacy": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+        }
+    }
+}`,
+		},
+		{
+			description: "CDI enabled enables CDI in docker",
+			args:        []string{"--cdi-enabled", "--create-device-nodes=none"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+    [nvidia-container-runtime.modes.legacy]
+      cuda-compat-mode = "ldconfig"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `{
+    "default-runtime": "nvidia",
+    "features": {
+        "cdi": true
+    },
+    "runtimes": {
+        "nvidia": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+        },
+        "nvidia-cdi": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+        },
+        "nvidia-legacy": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+        }
+    }
+}`,
+		},
+		{
+			description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in Docker",
+			args:        []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+    [nvidia-container-runtime.modes.legacy]
+      cuda-compat-mode = "ldconfig"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+        },
+        "nvidia-cdi": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+        },
+        "nvidia-legacy": {
+            "args": [],
+            "path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+        }
+    }
+}`,
+		},
+		{
+			description: "CDI enabled enables CDI in containerd",
+			args:        []string{"--cdi-enabled", "--runtime=containerd"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+    [nvidia-container-runtime.modes.legacy]
+      cuda-compat-mode = "ldconfig"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `version = 2
+
+[plugins]
+
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_cdi = true
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+`,
+		},
+		{
+			description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in containerd",
+			args:        []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false", "--runtime=containerd"},
+			expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
+accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = false
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+swarm-resource = ""
+
+[nvidia-container-cli]
+  debug = ""
+  environment = []
+  ldcache = ""
+  ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
+  load-kmods = true
+  no-cgroups = false
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
+  root = "/run/nvidia/driver"
+  user = ""
+
+[nvidia-container-runtime]
+  debug = "/dev/null"
+  log-level = "info"
+  mode = "auto"
+  runtimes = ["runc", "crun"]
+
+  [nvidia-container-runtime.modes]
+
+    [nvidia-container-runtime.modes.cdi]
+      annotation-prefixes = ["cdi.k8s.io/"]
+      default-kind = "nvidia.com/gpu"
+      spec-dirs = ["/etc/cdi", "/var/run/cdi"]
+
+    [nvidia-container-runtime.modes.csv]
+      mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+    [nvidia-container-runtime.modes.legacy]
+      cuda-compat-mode = "ldconfig"
+
+[nvidia-container-runtime-hook]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
+  skip-mode-detection = true
+
+[nvidia-ctk]
+  path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
+`,
+			expectedRuntimeConfig: `version = 2
+
+[plugins]
+
+  [plugins."io.containerd.grpc.v1.cri"]
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
+            BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			testRoot := t.TempDir()
+
+			cdiOutputDir := filepath.Join(testRoot, "/var/run/cdi")
+			runtimeConfigFile := filepath.Join(testRoot, "config.file")
+
+			toolkitRoot := filepath.Join(testRoot, "toolkit-test")
+			toolkitConfigFile := filepath.Join(toolkitRoot, "toolkit/.config/nvidia-container-runtime/config.toml")
+
+			app := NewApp(logger)
+
+			testArgs := []string{
+				"nvidia-ctk-installer",
+				"--toolkit-install-dir=" + toolkitRoot,
+				"--no-daemon",
+				"--cdi-output-dir=" + cdiOutputDir,
+				"--config=" + runtimeConfigFile,
+				"--create-device-nodes=none",
+				"--driver-root-ctr-path=" + hostRoot,
+				"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
+				"--restart-mode=none",
+				"--toolkit-source-root=" + filepath.Join(artifactRoot, "deb"),
+			}
+
+			err := app.Run(append(testArgs, tc.args...))
+
+			require.NoError(t, err)
+
+			require.FileExists(t, toolkitConfigFile)
+			toolkitConfigFileContents, err := os.ReadFile(toolkitConfigFile)
+			require.NoError(t, err)
+			require.EqualValues(t, strings.ReplaceAll(tc.expectedToolkitConfig, "{{ .toolkitRoot }}", toolkitRoot), string(toolkitConfigFileContents))
+
+			require.FileExists(t, runtimeConfigFile)
+			runtimeConfigFileContents, err := os.ReadFile(runtimeConfigFile)
+			require.NoError(t, err)
+			require.EqualValues(t, strings.ReplaceAll(tc.expectedRuntimeConfig, "{{ .toolkitRoot }}", toolkitRoot), string(runtimeConfigFileContents))
+		})
+	}
+
+}

cmd/nvidia-ctk-installer/toolkit/installer/artifact-root.go

@@ -0,0 +1,85 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+// An artifactRoot is used as a source for installed artifacts.
+// It is refined by a directory path, a library locator, and an executable locator.
+type artifactRoot struct {
+	path        string
+	libraries   lookup.Locator
+	executables lookup.Locator
+}
+
+func newArtifactRoot(logger logger.Interface, rootDirectoryPath string) (*artifactRoot, error) {
+	relativeLibrarySearchPaths := []string{
+		"/usr/lib64",
+		"/usr/lib/x86_64-linux-gnu",
+		"/usr/lib/aarch64-linux-gnu",
+	}
+	var librarySearchPaths []string
+	for _, l := range relativeLibrarySearchPaths {
+		librarySearchPaths = append(librarySearchPaths, filepath.Join(rootDirectoryPath, l))
+	}
+
+	a := artifactRoot{
+		path: rootDirectoryPath,
+		libraries: lookup.NewLibraryLocator(
+			lookup.WithLogger(logger),
+			lookup.WithCount(1),
+			lookup.WithSearchPaths(librarySearchPaths...),
+		),
+		executables: lookup.NewExecutableLocator(
+			logger,
+			rootDirectoryPath,
+		),
+	}
+
+	return &a, nil
+}
+
+func (r *artifactRoot) findLibrary(name string) (string, error) {
+	candidates, err := r.libraries.Locate(name)
+	if err != nil {
+		return "", fmt.Errorf("error locating library: %w", err)
+	}
+	if len(candidates) == 0 {
+		return "", fmt.Errorf("library %v not found", name)
+	}
+
+	return candidates[0], nil
+}
+
+func (r *artifactRoot) findExecutable(name string) (string, error) {
+	candidates, err := r.executables.Locate(name)
+	if err != nil {
+		return "", fmt.Errorf("error locating executable: %w", err)
+	}
+	if len(candidates) == 0 {
+		return "", fmt.Errorf("executable %v not found", name)
+	}
+
+	return candidates[0], nil
+}

cmd/nvidia-ctk-installer/toolkit/installer/directory.go

@@ -0,0 +1,47 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type createDirectory struct {
+	logger logger.Interface
+}
+
+func (t *ToolkitInstaller) createDirectory() Installer {
+	return &createDirectory{
+		logger: t.logger,
+	}
+}
+
+func (d *createDirectory) Install(dir string) error {
+	if dir == "" {
+		return nil
+	}
+	d.logger.Infof("Creating directory '%v'", dir)
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return fmt.Errorf("error creating directory: %v", err)
+	}
+	return nil
+}

cmd/nvidia-ctk-installer/toolkit/installer/executables.go

@@ -0,0 +1,179 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"io"
+	"path/filepath"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+)
+
+type executable struct {
+	requiresKernelModule bool
+	path                 string
+	symlink              string
+	env                  map[string]string
+}
+
+func (t *ToolkitInstaller) collectExecutables(destDir string) ([]Installer, error) {
+	configFilePath := t.ConfigFilePath(destDir)
+
+	executables := []executable{
+		{
+			path: "nvidia-ctk",
+		},
+		{
+			path: "nvidia-cdi-hook",
+		},
+	}
+	for _, runtime := range operator.GetRuntimes() {
+		e := executable{
+			path:                 runtime.Path,
+			requiresKernelModule: true,
+			env: map[string]string{
+				config.FilePathOverrideEnvVar: configFilePath,
+			},
+		}
+		executables = append(executables, e)
+	}
+	executables = append(executables,
+		executable{
+			path: "nvidia-container-cli",
+			env:  map[string]string{"LD_LIBRARY_PATH": destDir + ":$LD_LIBRARY_PATH"},
+		},
+	)
+
+	executables = append(executables,
+		executable{
+			path:    "nvidia-container-runtime-hook",
+			symlink: "nvidia-container-toolkit",
+			env: map[string]string{
+				config.FilePathOverrideEnvVar: configFilePath,
+			},
+		},
+	)
+
+	var installers []Installer
+	for _, executable := range executables {
+		executablePath, err := t.artifactRoot.findExecutable(executable.path)
+		if err != nil {
+			if t.ignoreErrors {
+				log.Errorf("Ignoring error: %v", err)
+				continue
+			}
+			return nil, err
+		}
+
+		wrappedExecutableFilename := filepath.Base(executablePath)
+		dotRealFilename := wrappedExecutableFilename + ".real"
+
+		w := &wrapper{
+			Source:            executablePath,
+			WrappedExecutable: dotRealFilename,
+			CheckModules:      executable.requiresKernelModule,
+			Envvars: map[string]string{
+				"PATH": strings.Join([]string{destDir, "$PATH"}, ":"),
+			},
+		}
+		for k, v := range executable.env {
+			w.Envvars[k] = v
+		}
+
+		installers = append(installers, w)
+
+		if executable.symlink == "" {
+			continue
+		}
+		link := symlink{
+			linkname: executable.symlink,
+			target:   filepath.Base(executablePath),
+		}
+		installers = append(installers, link)
+	}
+
+	return installers, nil
+
+}
+
+type wrapper struct {
+	Source            string
+	Envvars           map[string]string
+	WrappedExecutable string
+	CheckModules      bool
+}
+
+type render struct {
+	*wrapper
+	DestDir string
+}
+
+func (w *wrapper) Install(destDir string) error {
+	// Copy the executable with a .real extension.
+	mode, err := installFile(w.Source, filepath.Join(destDir, w.WrappedExecutable))
+	if err != nil {
+		return err
+	}
+
+	// Create a wrapper file.
+	r := render{
+		wrapper: w,
+		DestDir: destDir,
+	}
+	content, err := r.render()
+	if err != nil {
+		return fmt.Errorf("failed to render wrapper: %w", err)
+	}
+	wrapperFile := filepath.Join(destDir, filepath.Base(w.Source))
+	return installContent(content, wrapperFile, mode|0111)
+}
+
+func (w *render) render() (io.Reader, error) {
+	wrapperTemplate := `#! /bin/sh
+{{- if (.CheckModules) }}
+cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
+if [ "${?}" != "0" ]; then
+	echo "nvidia driver modules are not yet loaded, invoking runc directly"
+	exec runc "$@"
+fi
+{{- end }}
+{{- range $key, $value := .Envvars }}
+{{$key}}={{$value}} \
+{{- end }}
+	{{ .DestDir }}/{{ .WrappedExecutable }} \
+		"$@"
+`
+
+	var content bytes.Buffer
+	tmpl, err := template.New("wrapper").Parse(wrapperTemplate)
+	if err != nil {
+		return nil, err
+	}
+	if err := tmpl.Execute(&content, w); err != nil {
+		return nil, err
+	}
+
+	return &content, nil
+}

cmd/nvidia-ctk-installer/toolkit/installer/executables_test.go

@@ -0,0 +1,91 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWrapperRender(t *testing.T) {
+	testCases := []struct {
+		description string
+		w           *wrapper
+		expected    string
+	}{
+		{
+			description: "executable is added",
+			w: &wrapper{
+				WrappedExecutable: "some-runtime",
+			},
+			expected: `#! /bin/sh
+	/dest-dir/some-runtime \
+		"$@"
+`,
+		},
+		{
+			description: "module check is added",
+			w: &wrapper{
+				WrappedExecutable: "some-runtime",
+				CheckModules:      true,
+			},
+			expected: `#! /bin/sh
+cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
+if [ "${?}" != "0" ]; then
+	echo "nvidia driver modules are not yet loaded, invoking runc directly"
+	exec runc "$@"
+fi
+	/dest-dir/some-runtime \
+		"$@"
+`,
+		},
+		{
+			description: "environment is added",
+			w: &wrapper{
+				WrappedExecutable: "some-runtime",
+				Envvars: map[string]string{
+					"PATH": "/foo/bar/baz",
+				},
+			},
+			expected: `#! /bin/sh
+PATH=/foo/bar/baz \
+	/dest-dir/some-runtime \
+		"$@"
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			r := render{
+				wrapper: tc.w,
+				DestDir: "/dest-dir",
+			}
+			reader, err := r.render()
+			require.NoError(t, err)
+
+			var content bytes.Buffer
+			_, err = content.ReadFrom(reader)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expected, content.String())
+		})
+	}
+}

cmd/nvidia-ctk-installer/toolkit/installer/file-installer_mock.go

@@ -0,0 +1,188 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package installer
+
+import (
+	"io"
+	"os"
+	"sync"
+)
+
+// Ensure, that fileInstallerMock does implement fileInstaller.
+// If this is not the case, regenerate this file with moq.
+var _ fileInstaller = &fileInstallerMock{}
+
+// fileInstallerMock is a mock implementation of fileInstaller.
+//
+//	func TestSomethingThatUsesfileInstaller(t *testing.T) {
+//
+//		// make and configure a mocked fileInstaller
+//		mockedfileInstaller := &fileInstallerMock{
+//			installContentFunc: func(reader io.Reader, s string, v os.FileMode) error {
+//				panic("mock out the installContent method")
+//			},
+//			installFileFunc: func(s1 string, s2 string) (os.FileMode, error) {
+//				panic("mock out the installFile method")
+//			},
+//			installSymlinkFunc: func(s1 string, s2 string) error {
+//				panic("mock out the installSymlink method")
+//			},
+//		}
+//
+//		// use mockedfileInstaller in code that requires fileInstaller
+//		// and then make assertions.
+//
+//	}
+type fileInstallerMock struct {
+	// installContentFunc mocks the installContent method.
+	installContentFunc func(reader io.Reader, s string, v os.FileMode) error
+
+	// installFileFunc mocks the installFile method.
+	installFileFunc func(s1 string, s2 string) (os.FileMode, error)
+
+	// installSymlinkFunc mocks the installSymlink method.
+	installSymlinkFunc func(s1 string, s2 string) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// installContent holds details about calls to the installContent method.
+		installContent []struct {
+			// Reader is the reader argument value.
+			Reader io.Reader
+			// S is the s argument value.
+			S string
+			// V is the v argument value.
+			V os.FileMode
+		}
+		// installFile holds details about calls to the installFile method.
+		installFile []struct {
+			// S1 is the s1 argument value.
+			S1 string
+			// S2 is the s2 argument value.
+			S2 string
+		}
+		// installSymlink holds details about calls to the installSymlink method.
+		installSymlink []struct {
+			// S1 is the s1 argument value.
+			S1 string
+			// S2 is the s2 argument value.
+			S2 string
+		}
+	}
+	lockinstallContent sync.RWMutex
+	lockinstallFile    sync.RWMutex
+	lockinstallSymlink sync.RWMutex
+}
+
+// installContent calls installContentFunc.
+func (mock *fileInstallerMock) installContent(reader io.Reader, s string, v os.FileMode) error {
+	if mock.installContentFunc == nil {
+		panic("fileInstallerMock.installContentFunc: method is nil but fileInstaller.installContent was just called")
+	}
+	callInfo := struct {
+		Reader io.Reader
+		S      string
+		V      os.FileMode
+	}{
+		Reader: reader,
+		S:      s,
+		V:      v,
+	}
+	mock.lockinstallContent.Lock()
+	mock.calls.installContent = append(mock.calls.installContent, callInfo)
+	mock.lockinstallContent.Unlock()
+	return mock.installContentFunc(reader, s, v)
+}
+
+// installContentCalls gets all the calls that were made to installContent.
+// Check the length with:
+//
+//	len(mockedfileInstaller.installContentCalls())
+func (mock *fileInstallerMock) installContentCalls() []struct {
+	Reader io.Reader
+	S      string
+	V      os.FileMode
+} {
+	var calls []struct {
+		Reader io.Reader
+		S      string
+		V      os.FileMode
+	}
+	mock.lockinstallContent.RLock()
+	calls = mock.calls.installContent
+	mock.lockinstallContent.RUnlock()
+	return calls
+}
+
+// installFile calls installFileFunc.
+func (mock *fileInstallerMock) installFile(s1 string, s2 string) (os.FileMode, error) {
+	if mock.installFileFunc == nil {
+		panic("fileInstallerMock.installFileFunc: method is nil but fileInstaller.installFile was just called")
+	}
+	callInfo := struct {
+		S1 string
+		S2 string
+	}{
+		S1: s1,
+		S2: s2,
+	}
+	mock.lockinstallFile.Lock()
+	mock.calls.installFile = append(mock.calls.installFile, callInfo)
+	mock.lockinstallFile.Unlock()
+	return mock.installFileFunc(s1, s2)
+}
+
+// installFileCalls gets all the calls that were made to installFile.
+// Check the length with:
+//
+//	len(mockedfileInstaller.installFileCalls())
+func (mock *fileInstallerMock) installFileCalls() []struct {
+	S1 string
+	S2 string
+} {
+	var calls []struct {
+		S1 string
+		S2 string
+	}
+	mock.lockinstallFile.RLock()
+	calls = mock.calls.installFile
+	mock.lockinstallFile.RUnlock()
+	return calls
+}
+
+// installSymlink calls installSymlinkFunc.
+func (mock *fileInstallerMock) installSymlink(s1 string, s2 string) error {
+	if mock.installSymlinkFunc == nil {
+		panic("fileInstallerMock.installSymlinkFunc: method is nil but fileInstaller.installSymlink was just called")
+	}
+	callInfo := struct {
+		S1 string
+		S2 string
+	}{
+		S1: s1,
+		S2: s2,
+	}
+	mock.lockinstallSymlink.Lock()
+	mock.calls.installSymlink = append(mock.calls.installSymlink, callInfo)
+	mock.lockinstallSymlink.Unlock()
+	return mock.installSymlinkFunc(s1, s2)
+}
+
+// installSymlinkCalls gets all the calls that were made to installSymlink.
+// Check the length with:
+//
+//	len(mockedfileInstaller.installSymlinkCalls())
+func (mock *fileInstallerMock) installSymlinkCalls() []struct {
+	S1 string
+	S2 string
+} {
+	var calls []struct {
+		S1 string
+		S2 string
+	}
+	mock.lockinstallSymlink.RLock()
+	calls = mock.calls.installSymlink
+	mock.lockinstallSymlink.RUnlock()
+	return calls
+}

cmd/nvidia-ctk-installer/toolkit/installer/installer.go

@@ -0,0 +1,172 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+//go:generate moq -rm -fmt=goimports -out installer_mock.go . Installer
+type Installer interface {
+	Install(string) error
+}
+
+type ToolkitInstaller struct {
+	logger       logger.Interface
+	ignoreErrors bool
+	sourceRoot   string
+
+	artifactRoot *artifactRoot
+
+	ensureTargetDirectory Installer
+}
+
+var _ Installer = (*ToolkitInstaller)(nil)
+
+// New creates a toolkit installer with the specified options.
+func New(opts ...Option) (*ToolkitInstaller, error) {
+	t := &ToolkitInstaller{
+		sourceRoot: "/",
+	}
+	for _, opt := range opts {
+		opt(t)
+	}
+
+	if t.logger == nil {
+		t.logger = logger.New()
+	}
+	if t.artifactRoot == nil {
+		artifactRoot, err := newArtifactRoot(t.logger, t.sourceRoot)
+		if err != nil {
+			return nil, err
+		}
+		t.artifactRoot = artifactRoot
+	}
+
+	if t.ensureTargetDirectory == nil {
+		t.ensureTargetDirectory = t.createDirectory()
+	}
+
+	return t, nil
+}
+
+// Install ensures that the required toolkit files are installed in the specified directory.
+func (t *ToolkitInstaller) Install(destDir string) error {
+	var installers []Installer
+
+	installers = append(installers, t.ensureTargetDirectory)
+
+	libraries, err := t.collectLibraries()
+	if err != nil {
+		return fmt.Errorf("failed to collect libraries: %w", err)
+	}
+	installers = append(installers, libraries...)
+
+	executables, err := t.collectExecutables(destDir)
+	if err != nil {
+		return fmt.Errorf("failed to collect executables: %w", err)
+	}
+	installers = append(installers, executables...)
+
+	var errs error
+	for _, i := range installers {
+		errs = errors.Join(errs, i.Install(destDir))
+	}
+
+	return errs
+}
+
+func (t *ToolkitInstaller) ConfigFilePath(destDir string) string {
+	toolkitConfigDir := filepath.Join(destDir, ".config", "nvidia-container-runtime")
+	return filepath.Join(toolkitConfigDir, "config.toml")
+}
+
+type symlink struct {
+	linkname string
+	target   string
+}
+
+func (s symlink) Install(destDir string) error {
+	symlinkPath := filepath.Join(destDir, s.linkname)
+	return installSymlink(s.target, symlinkPath)
+}
+
+//go:generate moq -rm -fmt=goimports -out file-installer_mock.go . fileInstaller
+type fileInstaller interface {
+	installContent(io.Reader, string, os.FileMode) error
+	installFile(string, string) (os.FileMode, error)
+	installSymlink(string, string) error
+}
+
+var installSymlink = installSymlinkStub
+
+func installSymlinkStub(target string, link string) error {
+	err := os.Symlink(target, link)
+	if err != nil {
+		return fmt.Errorf("error creating symlink '%v' => '%v': %v", link, target, err)
+	}
+	return nil
+}
+
+var installFile = installFileStub
+
+func installFileStub(src string, dest string) (os.FileMode, error) {
+	sourceInfo, err := os.Stat(src)
+	if err != nil {
+		return 0, fmt.Errorf("error getting file info for '%v': %v", src, err)
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return 0, fmt.Errorf("error opening source: %w", err)
+	}
+	defer source.Close()
+
+	mode := sourceInfo.Mode()
+	if err := installContent(source, dest, mode); err != nil {
+		return 0, err
+	}
+	return mode, nil
+}
+
+var installContent = installContentStub
+
+func installContentStub(content io.Reader, dest string, mode fs.FileMode) error {
+	destination, err := os.Create(dest)
+	if err != nil {
+		return fmt.Errorf("error creating destination: %w", err)
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, content)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	err = os.Chmod(dest, mode)
+	if err != nil {
+		return fmt.Errorf("error setting mode for '%v': %v", dest, err)
+	}
+	return nil
+}

cmd/nvidia-ctk-installer/toolkit/installer/installer_mock.go

@@ -0,0 +1,74 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package installer
+
+import (
+	"sync"
+)
+
+// Ensure, that InstallerMock does implement Installer.
+// If this is not the case, regenerate this file with moq.
+var _ Installer = &InstallerMock{}
+
+// InstallerMock is a mock implementation of Installer.
+//
+//	func TestSomethingThatUsesInstaller(t *testing.T) {
+//
+//		// make and configure a mocked Installer
+//		mockedInstaller := &InstallerMock{
+//			InstallFunc: func(s string) error {
+//				panic("mock out the Install method")
+//			},
+//		}
+//
+//		// use mockedInstaller in code that requires Installer
+//		// and then make assertions.
+//
+//	}
+type InstallerMock struct {
+	// InstallFunc mocks the Install method.
+	InstallFunc func(s string) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Install holds details about calls to the Install method.
+		Install []struct {
+			// S is the s argument value.
+			S string
+		}
+	}
+	lockInstall sync.RWMutex
+}
+
+// Install calls InstallFunc.
+func (mock *InstallerMock) Install(s string) error {
+	if mock.InstallFunc == nil {
+		panic("InstallerMock.InstallFunc: method is nil but Installer.Install was just called")
+	}
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockInstall.Lock()
+	mock.calls.Install = append(mock.calls.Install, callInfo)
+	mock.lockInstall.Unlock()
+	return mock.InstallFunc(s)
+}
+
+// InstallCalls gets all the calls that were made to Install.
+// Check the length with:
+//
+//	len(mockedInstaller.InstallCalls())
+func (mock *InstallerMock) InstallCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockInstall.RLock()
+	calls = mock.calls.Install
+	mock.lockInstall.RUnlock()
+	return calls
+}

cmd/nvidia-ctk-installer/toolkit/installer/installer_test.go

@@ -0,0 +1,251 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+)
+
+func TestToolkitInstaller(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	type contentCall struct {
+		wrapper string
+		path    string
+		mode    fs.FileMode
+	}
+	var contentCalls []contentCall
+
+	installer := &fileInstallerMock{
+		installFileFunc: func(s1, s2 string) (os.FileMode, error) {
+			return 0666, nil
+		},
+		installContentFunc: func(reader io.Reader, s string, fileMode fs.FileMode) error {
+			var b bytes.Buffer
+			if _, err := b.ReadFrom(reader); err != nil {
+				return err
+			}
+			contents := contentCall{
+				wrapper: b.String(),
+				path:    s,
+				mode:    fileMode,
+			}
+
+			contentCalls = append(contentCalls, contents)
+			return nil
+		},
+		installSymlinkFunc: func(s1, s2 string) error {
+			return nil
+		},
+	}
+	installFile = installer.installFile
+	installContent = installer.installContent
+	installSymlink = installer.installSymlink
+
+	root := "/artifacts/test"
+	libraries := &lookup.LocatorMock{
+		LocateFunc: func(s string) ([]string, error) {
+			switch s {
+			case "libnvidia-container.so.1":
+				return []string{filepath.Join(root, "libnvidia-container.so.987.65.43")}, nil
+			case "libnvidia-container-go.so.1":
+				return []string{filepath.Join(root, "libnvidia-container-go.so.1.23.4")}, nil
+			}
+			return nil, fmt.Errorf("%v not found", s)
+		},
+	}
+	executables := &lookup.LocatorMock{
+		LocateFunc: func(s string) ([]string, error) {
+			switch s {
+			case "nvidia-container-runtime.cdi":
+				fallthrough
+			case "nvidia-container-runtime.legacy":
+				fallthrough
+			case "nvidia-container-runtime":
+				fallthrough
+			case "nvidia-ctk":
+				fallthrough
+			case "nvidia-container-cli":
+				fallthrough
+			case "nvidia-container-runtime-hook":
+				fallthrough
+			case "nvidia-cdi-hook":
+				return []string{filepath.Join(root, "usr/bin", s)}, nil
+			}
+			return nil, fmt.Errorf("%v not found", s)
+		},
+	}
+
+	r := &artifactRoot{
+		libraries:   libraries,
+		executables: executables,
+	}
+
+	createDirectory := &InstallerMock{
+		InstallFunc: func(c string) error {
+			return nil
+		},
+	}
+	i := ToolkitInstaller{
+		logger:                logger,
+		artifactRoot:          r,
+		ensureTargetDirectory: createDirectory,
+	}
+
+	err := i.Install("/foo/bar/baz")
+	require.NoError(t, err)
+
+	require.ElementsMatch(t,
+		[]struct {
+			S string
+		}{
+			{"/foo/bar/baz"},
+		},
+		createDirectory.InstallCalls(),
+	)
+
+	require.ElementsMatch(t,
+		installer.installFileCalls(),
+		[]struct {
+			S1 string
+			S2 string
+		}{
+			{"/artifacts/test/libnvidia-container-go.so.1.23.4", "/foo/bar/baz/libnvidia-container-go.so.1.23.4"},
+			{"/artifacts/test/libnvidia-container.so.987.65.43", "/foo/bar/baz/libnvidia-container.so.987.65.43"},
+			{"/artifacts/test/usr/bin/nvidia-container-runtime.cdi", "/foo/bar/baz/nvidia-container-runtime.cdi.real"},
+			{"/artifacts/test/usr/bin/nvidia-container-runtime.legacy", "/foo/bar/baz/nvidia-container-runtime.legacy.real"},
+			{"/artifacts/test/usr/bin/nvidia-container-runtime", "/foo/bar/baz/nvidia-container-runtime.real"},
+			{"/artifacts/test/usr/bin/nvidia-ctk", "/foo/bar/baz/nvidia-ctk.real"},
+			{"/artifacts/test/usr/bin/nvidia-cdi-hook", "/foo/bar/baz/nvidia-cdi-hook.real"},
+			{"/artifacts/test/usr/bin/nvidia-container-cli", "/foo/bar/baz/nvidia-container-cli.real"},
+			{"/artifacts/test/usr/bin/nvidia-container-runtime-hook", "/foo/bar/baz/nvidia-container-runtime-hook.real"},
+		},
+	)
+
+	require.ElementsMatch(t,
+		installer.installSymlinkCalls(),
+		[]struct {
+			S1 string
+			S2 string
+		}{
+			{"libnvidia-container-go.so.1.23.4", "/foo/bar/baz/libnvidia-container-go.so.1"},
+			{"libnvidia-container.so.987.65.43", "/foo/bar/baz/libnvidia-container.so.1"},
+			{"nvidia-container-runtime-hook", "/foo/bar/baz/nvidia-container-toolkit"},
+		},
+	)
+
+	require.ElementsMatch(t,
+		contentCalls,
+		[]contentCall{
+			{
+				path: "/foo/bar/baz/nvidia-container-runtime",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
+if [ "${?}" != "0" ]; then
+	echo "nvidia driver modules are not yet loaded, invoking runc directly"
+	exec runc "$@"
+fi
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-container-runtime.real \
+		"$@"
+`,
+			},
+			{
+				path: "/foo/bar/baz/nvidia-container-runtime.cdi",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
+if [ "${?}" != "0" ]; then
+	echo "nvidia driver modules are not yet loaded, invoking runc directly"
+	exec runc "$@"
+fi
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-container-runtime.cdi.real \
+		"$@"
+`,
+			},
+			{
+				path: "/foo/bar/baz/nvidia-container-runtime.legacy",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1
+if [ "${?}" != "0" ]; then
+	echo "nvidia driver modules are not yet loaded, invoking runc directly"
+	exec runc "$@"
+fi
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-container-runtime.legacy.real \
+		"$@"
+`,
+			},
+			{
+				path: "/foo/bar/baz/nvidia-ctk",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-ctk.real \
+		"$@"
+`,
+			},
+			{
+				path: "/foo/bar/baz/nvidia-cdi-hook",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-cdi-hook.real \
+		"$@"
+`,
+			},
+			{
+				path: "/foo/bar/baz/nvidia-container-cli",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+LD_LIBRARY_PATH=/foo/bar/baz:$LD_LIBRARY_PATH \
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-container-cli.real \
+		"$@"
+`,
+			},
+			{
+				path: "/foo/bar/baz/nvidia-container-runtime-hook",
+				mode: 0777,
+				wrapper: `#! /bin/sh
+NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \
+PATH=/foo/bar/baz:$PATH \
+	/foo/bar/baz/nvidia-container-runtime-hook.real \
+		"$@"
+`,
+			},
+		},
+	)
+}

cmd/nvidia-ctk-installer/toolkit/installer/libraries.go

@@ -0,0 +1,73 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import (
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// collectLibraries locates and installs the libraries that are part of
+// the nvidia-container-toolkit.
+// A predefined set of library candidates are considered, with the first one
+// resulting in success being installed to the toolkit folder. The install process
+// resolves the symlink for the library and copies the versioned library itself.
+func (t *ToolkitInstaller) collectLibraries() ([]Installer, error) {
+	requiredLibraries := []string{
+		"libnvidia-container.so.1",
+		"libnvidia-container-go.so.1",
+	}
+
+	var installers []Installer
+	for _, l := range requiredLibraries {
+		libraryPath, err := t.artifactRoot.findLibrary(l)
+		if err != nil {
+			if t.ignoreErrors {
+				log.Errorf("Ignoring error: %v", err)
+				continue
+			}
+			return nil, err
+		}
+
+		installers = append(installers, library(libraryPath))
+
+		if filepath.Base(libraryPath) == l {
+			continue
+		}
+
+		link := symlink{
+			linkname: l,
+			target:   filepath.Base(libraryPath),
+		}
+		installers = append(installers, link)
+	}
+
+	return installers, nil
+}
+
+type library string
+
+// Install copies the library l to the destination folder.
+// The same basename is used in the destination folder.
+func (l library) Install(destinationDir string) error {
+	dest := filepath.Join(destinationDir, filepath.Base(string(l)))
+
+	_, err := installFile(string(l), dest)
+	return err
+}

cmd/nvidia-ctk-installer/toolkit/installer/options.go

@@ -0,0 +1,47 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package installer
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+type Option func(*ToolkitInstaller)
+
+func WithLogger(logger logger.Interface) Option {
+	return func(ti *ToolkitInstaller) {
+		ti.logger = logger
+	}
+}
+
+func WithArtifactRoot(artifactRoot *artifactRoot) Option {
+	return func(ti *ToolkitInstaller) {
+		ti.artifactRoot = artifactRoot
+	}
+}
+
+func WithIgnoreErrors(ignoreErrors bool) Option {
+	return func(ti *ToolkitInstaller) {
+		ti.ignoreErrors = ignoreErrors
+	}
+}
+
+// WithSourceRoot sets the root directory for locating artifacts to be installed.
+func WithSourceRoot(sourceRoot string) Option {
+	return func(ti *ToolkitInstaller) {
+		ti.sourceRoot = sourceRoot
+	}
+}

cmd/nvidia-ctk-installer/toolkit/options.go

@@ -0,0 +1,40 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package toolkit
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// An Option provides a mechanism to configure an Installer.
+type Option func(*Installer)
+
+func WithLogger(logger logger.Interface) Option {
+	return func(i *Installer) {
+		i.logger = logger
+	}
+}
+
+func WithToolkitRoot(toolkitRoot string) Option {
+	return func(i *Installer) {
+		i.toolkitRoot = toolkitRoot
+	}
+}
+
+func WithSourceRoot(sourceRoot string) Option {
+	return func(i *Installer) {
+		i.sourceRoot = sourceRoot
+	}
+}

cmd/nvidia-ctk-installer/toolkit/toolkit.go

@@ -0,0 +1,534 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package toolkit
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+	"tags.cncf.io/container-device-interface/pkg/parser"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/toolkit/installer"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	transformroot "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform/root"
+)
+
+const (
+	// DefaultNvidiaDriverRoot specifies the default NVIDIA driver run directory
+	DefaultNvidiaDriverRoot = "/run/nvidia/driver"
+)
+
+type cdiOptions struct {
+	Enabled   bool
+	outputDir string
+	kind      string
+	vendor    string
+	class     string
+}
+
+type Options struct {
+	DriverRoot        string
+	DevRoot           string
+	DriverRootCtrPath string
+	DevRootCtrPath    string
+
+	ContainerRuntimeMode     string
+	ContainerRuntimeDebug    string
+	ContainerRuntimeLogLevel string
+
+	ContainerRuntimeModesCdiDefaultKind        string
+	ContainerRuntimeModesCDIAnnotationPrefixes cli.StringSlice
+
+	ContainerRuntimeRuntimes cli.StringSlice
+
+	ContainerRuntimeHookSkipModeDetection bool
+
+	ContainerCLIDebug string
+
+	// CDI stores the CDI options for the toolkit.
+	CDI cdiOptions
+
+	createDeviceNodes cli.StringSlice
+
+	acceptNVIDIAVisibleDevicesWhenUnprivileged bool
+	acceptNVIDIAVisibleDevicesAsVolumeMounts   bool
+
+	ignoreErrors bool
+
+	optInFeatures cli.StringSlice
+}
+
+func Flags(opts *Options) []cli.Flag {
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Aliases:     []string{"nvidia-driver-root"},
+			Value:       DefaultNvidiaDriverRoot,
+			Destination: &opts.DriverRoot,
+			EnvVars:     []string{"NVIDIA_DRIVER_ROOT", "DRIVER_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root-ctr-path",
+			Value:       DefaultNvidiaDriverRoot,
+			Destination: &opts.DriverRootCtrPath,
+			EnvVars:     []string{"DRIVER_ROOT_CTR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "dev-root",
+			Usage:       "Specify the root where `/dev` is located. If this is not specified, the driver-root is assumed.",
+			Destination: &opts.DevRoot,
+			EnvVars:     []string{"NVIDIA_DEV_ROOT", "DEV_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "dev-root-ctr-path",
+			Usage:       "Specify the root where `/dev` is located in the container. If this is not specified, the driver-root-ctr-path is assumed.",
+			Destination: &opts.DevRootCtrPath,
+			EnvVars:     []string{"DEV_ROOT_CTR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.debug",
+			Aliases:     []string{"nvidia-container-runtime-debug"},
+			Usage:       "Specify the location of the debug log file for the NVIDIA Container Runtime",
+			Destination: &opts.ContainerRuntimeDebug,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_DEBUG"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.log-level",
+			Aliases:     []string{"nvidia-container-runtime-debug-log-level"},
+			Destination: &opts.ContainerRuntimeLogLevel,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_LOG_LEVEL"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.mode",
+			Aliases:     []string{"nvidia-container-runtime-mode"},
+			Destination: &opts.ContainerRuntimeMode,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime.modes.cdi.default-kind",
+			Destination: &opts.ContainerRuntimeModesCdiDefaultKind,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_DEFAULT_KIND"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "nvidia-container-runtime.modes.cdi.annotation-prefixes",
+			Destination: &opts.ContainerRuntimeModesCDIAnnotationPrefixes,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "nvidia-container-runtime.runtimes",
+			Destination: &opts.ContainerRuntimeRuntimes,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_RUNTIMES"},
+		},
+		&cli.BoolFlag{
+			Name:        "nvidia-container-runtime-hook.skip-mode-detection",
+			Value:       true,
+			Destination: &opts.ContainerRuntimeHookSkipModeDetection,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_HOOK_SKIP_MODE_DETECTION"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-cli.debug",
+			Aliases:     []string{"nvidia-container-cli-debug"},
+			Usage:       "Specify the location of the debug log file for the NVIDIA Container CLI",
+			Destination: &opts.ContainerCLIDebug,
+			EnvVars:     []string{"NVIDIA_CONTAINER_CLI_DEBUG"},
+		},
+		&cli.BoolFlag{
+			Name:        "accept-nvidia-visible-devices-envvar-when-unprivileged",
+			Usage:       "Set the accept-nvidia-visible-devices-envvar-when-unprivileged config option",
+			Value:       true,
+			Destination: &opts.acceptNVIDIAVisibleDevicesWhenUnprivileged,
+			EnvVars:     []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_ENVVAR_WHEN_UNPRIVILEGED"},
+		},
+		&cli.BoolFlag{
+			Name:        "accept-nvidia-visible-devices-as-volume-mounts",
+			Usage:       "Set the accept-nvidia-visible-devices-as-volume-mounts config option",
+			Destination: &opts.acceptNVIDIAVisibleDevicesAsVolumeMounts,
+			EnvVars:     []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_AS_VOLUME_MOUNTS"},
+		},
+		&cli.BoolFlag{
+			Name:        "cdi-enabled",
+			Aliases:     []string{"enable-cdi"},
+			Usage:       "enable the generation of a CDI specification",
+			Destination: &opts.CDI.Enabled,
+			EnvVars:     []string{"CDI_ENABLED", "ENABLE_CDI"},
+		},
+		&cli.StringFlag{
+			Name:        "cdi-output-dir",
+			Usage:       "the directory where the CDI output files are to be written. If this is set to '', no CDI specification is generated.",
+			Value:       "/var/run/cdi",
+			Destination: &opts.CDI.outputDir,
+			EnvVars:     []string{"CDI_OUTPUT_DIR"},
+		},
+		&cli.StringFlag{
+			Name:        "cdi-kind",
+			Usage:       "the vendor string to use for the generated CDI specification",
+			Value:       "management.nvidia.com/gpu",
+			Destination: &opts.CDI.kind,
+			EnvVars:     []string{"CDI_KIND"},
+		},
+		&cli.BoolFlag{
+			Name:        "ignore-errors",
+			Usage:       "ignore errors when installing the NVIDIA Container toolkit. This is used for testing purposes only.",
+			Hidden:      true,
+			Destination: &opts.ignoreErrors,
+		},
+		&cli.StringSliceFlag{
+			Name:        "create-device-nodes",
+			Usage:       "(Only applicable with --cdi-enabled) specifies which device nodes should be created. If any one of the options is set to '' or 'none', no device nodes will be created.",
+			Value:       cli.NewStringSlice("control"),
+			Destination: &opts.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "opt-in-features",
+			Hidden:      true,
+			Destination: &opts.optInFeatures,
+			EnvVars:     []string{"NVIDIA_CONTAINER_TOOLKIT_OPT_IN_FEATURES"},
+		},
+	}
+
+	return flags
+}
+
+// An Installer is used to install the NVIDIA Container Toolkit from the toolkit container.
+type Installer struct {
+	logger logger.Interface
+
+	sourceRoot string
+	// toolkitRoot specifies the destination path at which the toolkit is installed.
+	toolkitRoot string
+}
+
+// NewInstaller creates an installer for the NVIDIA Container Toolkit.
+func NewInstaller(opts ...Option) *Installer {
+	i := &Installer{}
+	for _, opt := range opts {
+		opt(i)
+	}
+
+	if i.logger == nil {
+		i.logger = logger.New()
+	}
+
+	return i
+}
+
+// ValidateOptions checks whether the specified options are valid
+func (t *Installer) ValidateOptions(opts *Options) error {
+	if t == nil {
+		return fmt.Errorf("toolkit installer is not initilized")
+	}
+	if t.toolkitRoot == "" {
+		return fmt.Errorf("invalid --toolkit-root option: %v", t.toolkitRoot)
+	}
+
+	vendor, class := parser.ParseQualifier(opts.CDI.kind)
+	if err := parser.ValidateVendorName(vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := parser.ValidateClassName(class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	opts.CDI.vendor = vendor
+	opts.CDI.class = class
+
+	if opts.CDI.Enabled && opts.CDI.outputDir == "" {
+		t.logger.Warning("Skipping CDI spec generation (no output directory specified)")
+		opts.CDI.Enabled = false
+	}
+
+	isDisabled := false
+	for _, mode := range opts.createDeviceNodes.Value() {
+		if mode != "" && mode != "none" && mode != "control" {
+			return fmt.Errorf("invalid --create-device-nodes value: %v", mode)
+		}
+		if mode == "" || mode == "none" {
+			isDisabled = true
+			break
+		}
+	}
+	if !opts.CDI.Enabled && !isDisabled {
+		t.logger.Info("disabling device node creation since --cdi-enabled=false")
+		isDisabled = true
+	}
+	if isDisabled {
+		opts.createDeviceNodes = *cli.NewStringSlice()
+	}
+
+	return nil
+}
+
+// Install installs the components of the NVIDIA container toolkit.
+// Any existing installation is removed.
+func (t *Installer) Install(cli *cli.Context, opts *Options) error {
+	if t == nil {
+		return fmt.Errorf("toolkit installer is not initilized")
+	}
+	t.logger.Infof("Installing NVIDIA container toolkit to '%v'", t.toolkitRoot)
+
+	t.logger.Infof("Removing existing NVIDIA container toolkit installation")
+	err := os.RemoveAll(t.toolkitRoot)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error removing toolkit directory: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error removing toolkit directory: %v", err))
+	}
+
+	// Create a toolkit installer to actually install the toolkit components.
+	toolkit, err := installer.New(
+		installer.WithLogger(t.logger),
+		installer.WithSourceRoot(t.sourceRoot),
+		installer.WithIgnoreErrors(opts.ignoreErrors),
+	)
+	if err != nil {
+		if !opts.ignoreErrors {
+			return fmt.Errorf("could not create toolkit installer: %w", err)
+		}
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not create toolkit installer: %w", err))
+	}
+	if err := toolkit.Install(t.toolkitRoot); err != nil {
+		if !opts.ignoreErrors {
+			return fmt.Errorf("could not install toolkit components: %w", err)
+		}
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not install toolkit components: %w", err))
+	}
+
+	err = t.installToolkitConfig(cli, opts, toolkit.ConfigFilePath(t.toolkitRoot))
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error installing NVIDIA container toolkit config: %v", err))
+	}
+
+	err = t.createDeviceNodes(opts)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error creating device nodes: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error creating device nodes: %v", err))
+	}
+
+	nvidiaCDIHookPath := filepath.Join(t.toolkitRoot, "nvidia-cdi-hook")
+	err = t.generateCDISpec(opts, nvidiaCDIHookPath)
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("error generating CDI specification: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("error generating CDI specification: %v", err))
+	}
+
+	return nil
+}
+
+// installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring
+// that the settings are updated to match the desired install and nvidia driver directories.
+func (t *Installer) installToolkitConfig(c *cli.Context, opts *Options, toolkitConfigPath string) error {
+
+	t.logger.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
+
+	err := t.createDirectories(filepath.Dir(toolkitConfigPath))
+	if err != nil && !opts.ignoreErrors {
+		return fmt.Errorf("could not create required directories: %v", err)
+	} else if err != nil {
+		t.logger.Errorf("Ignoring error: %v", fmt.Errorf("could not create required directories: %v", err))
+	}
+	nvidiaContainerCliExecutablePath := filepath.Join(t.toolkitRoot, "nvidia-container-cli")
+	nvidiaCTKPath := filepath.Join(t.toolkitRoot, "nvidia-ctk")
+	nvidiaContainerRuntimeHookPath := filepath.Join(t.toolkitRoot, "nvidia-container-runtime-hook")
+
+	cfg, err := config.New()
+	if err != nil {
+		return fmt.Errorf("could not open source config file: %v", err)
+	}
+
+	targetConfig, err := os.Create(toolkitConfigPath)
+	if err != nil {
+		return fmt.Errorf("could not create target config file: %v", err)
+	}
+	defer targetConfig.Close()
+
+	// Read the ldconfig path from the config as this may differ per platform
+	// On ubuntu-based systems this ends in `.real`
+	ldconfigPath := fmt.Sprintf("%s", cfg.GetDefault("nvidia-container-cli.ldconfig", "/sbin/ldconfig"))
+	// Use the driver run root as the root:
+	driverLdconfigPath := config.NormalizeLDConfigPath("@" + filepath.Join(opts.DriverRoot, strings.TrimPrefix(ldconfigPath, "@/")))
+
+	configValues := map[string]interface{}{
+		// Set the options in the root toml table
+		"accept-nvidia-visible-devices-envvar-when-unprivileged": opts.acceptNVIDIAVisibleDevicesWhenUnprivileged,
+		"accept-nvidia-visible-devices-as-volume-mounts":         opts.acceptNVIDIAVisibleDevicesAsVolumeMounts,
+		// Set the nvidia-container-cli options
+		"nvidia-container-cli.root":     opts.DriverRoot,
+		"nvidia-container-cli.path":     nvidiaContainerCliExecutablePath,
+		"nvidia-container-cli.ldconfig": driverLdconfigPath,
+		// Set nvidia-ctk options
+		"nvidia-ctk.path": nvidiaCTKPath,
+		// Set the nvidia-container-runtime-hook options
+		"nvidia-container-runtime-hook.path":                nvidiaContainerRuntimeHookPath,
+		"nvidia-container-runtime-hook.skip-mode-detection": opts.ContainerRuntimeHookSkipModeDetection,
+	}
+
+	toolkitRuntimeList := opts.ContainerRuntimeRuntimes.Value()
+	if len(toolkitRuntimeList) > 0 {
+		configValues["nvidia-container-runtime.runtimes"] = toolkitRuntimeList
+	}
+
+	for _, optInFeature := range opts.optInFeatures.Value() {
+		configValues["features."+optInFeature] = true
+	}
+
+	for key, value := range configValues {
+		cfg.Set(key, value)
+	}
+
+	// Set the optional config options
+	optionalConfigValues := map[string]interface{}{
+		"nvidia-container-runtime.debug":                         opts.ContainerRuntimeDebug,
+		"nvidia-container-runtime.log-level":                     opts.ContainerRuntimeLogLevel,
+		"nvidia-container-runtime.mode":                          opts.ContainerRuntimeMode,
+		"nvidia-container-runtime.modes.cdi.annotation-prefixes": opts.ContainerRuntimeModesCDIAnnotationPrefixes,
+		"nvidia-container-runtime.modes.cdi.default-kind":        opts.ContainerRuntimeModesCdiDefaultKind,
+		"nvidia-container-runtime.runtimes":                      opts.ContainerRuntimeRuntimes,
+		"nvidia-container-cli.debug":                             opts.ContainerCLIDebug,
+	}
+
+	for key, value := range optionalConfigValues {
+		if !c.IsSet(key) {
+			t.logger.Infof("Skipping unset option: %v", key)
+			continue
+		}
+		if value == nil {
+			t.logger.Infof("Skipping option with nil value: %v", key)
+			continue
+		}
+
+		switch v := value.(type) {
+		case string:
+			if v == "" {
+				continue
+			}
+		case cli.StringSlice:
+			if len(v.Value()) == 0 {
+				continue
+			}
+			value = v.Value()
+		default:
+			t.logger.Warningf("Unexpected type for option %v=%v: %T", key, value, v)
+		}
+
+		cfg.Set(key, value)
+	}
+
+	if _, err := cfg.WriteTo(targetConfig); err != nil {
+		return fmt.Errorf("error writing config: %v", err)
+	}
+
+	os.Stdout.WriteString("Using config:\n")
+	if _, err = cfg.WriteTo(os.Stdout); err != nil {
+		t.logger.Warningf("Failed to output config to STDOUT: %v", err)
+	}
+
+	return nil
+}
+
+func (t *Installer) createDirectories(dir ...string) error {
+	for _, d := range dir {
+		t.logger.Infof("Creating directory '%v'", d)
+		err := os.MkdirAll(d, 0755)
+		if err != nil {
+			return fmt.Errorf("error creating directory: %v", err)
+		}
+	}
+	return nil
+}
+
+func (t *Installer) createDeviceNodes(opts *Options) error {
+	modes := opts.createDeviceNodes.Value()
+	if len(modes) == 0 {
+		return nil
+	}
+
+	devices, err := nvdevices.New(
+		nvdevices.WithDevRoot(opts.DevRootCtrPath),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create library: %v", err)
+	}
+
+	for _, mode := range modes {
+		t.logger.Infof("Creating %v device nodes at %v", mode, opts.DevRootCtrPath)
+		if mode != "control" {
+			t.logger.Warningf("Unrecognised device mode: %v", mode)
+			continue
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create control device nodes: %v", err)
+		}
+	}
+	return nil
+}
+
+// generateCDISpec generates a CDI spec for use in management containers
+func (t *Installer) generateCDISpec(opts *Options, nvidiaCDIHookPath string) error {
+	if !opts.CDI.Enabled {
+		return nil
+	}
+	t.logger.Info("Generating CDI spec for management containers")
+	cdilib, err := nvcdi.New(
+		nvcdi.WithLogger(t.logger),
+		nvcdi.WithMode(nvcdi.ModeManagement),
+		nvcdi.WithDriverRoot(opts.DriverRootCtrPath),
+		nvcdi.WithDevRoot(opts.DevRootCtrPath),
+		nvcdi.WithNVIDIACDIHookPath(nvidiaCDIHookPath),
+		nvcdi.WithVendor(opts.CDI.vendor),
+		nvcdi.WithClass(opts.CDI.class),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI library for management containers: %v", err)
+	}
+
+	spec, err := cdilib.GetSpec()
+	if err != nil {
+		return fmt.Errorf("failed to genereate CDI spec for management containers: %v", err)
+	}
+
+	transformer := transformroot.NewDriverTransformer(
+		transformroot.WithDriverRoot(opts.DriverRootCtrPath),
+		transformroot.WithTargetDriverRoot(opts.DriverRoot),
+		transformroot.WithDevRoot(opts.DevRootCtrPath),
+		transformroot.WithTargetDevRoot(opts.DevRoot),
+	)
+	if err := transformer.Transform(spec.Raw()); err != nil {
+		return fmt.Errorf("failed to transform driver root in CDI spec: %v", err)
+	}
+
+	name, err := cdi.GenerateNameForSpec(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI name for management containers: %v", err)
+	}
+	err = spec.Save(filepath.Join(opts.CDI.outputDir, name))
+	if err != nil {
+		return fmt.Errorf("failed to save CDI spec for management containers: %v", err)
+	}
+
+	return nil
+}

cmd/nvidia-ctk-installer/toolkit/toolkit_test.go

@@ -0,0 +1,232 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package toolkit
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestInstall(t *testing.T) {
+	t.Setenv("__NVCT_TESTING_DEVICES_ARE_FILES", "true")
+	logger, _ := testlog.NewNullLogger()
+
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	artifactRoot := filepath.Join(moduleRoot, "testdata", "installer", "artifacts")
+
+	testCases := []struct {
+		description     string
+		hostRoot        string
+		packageType     string
+		cdiEnabled      bool
+		expectedError   error
+		expectedCdiSpec string
+	}{
+		{
+			hostRoot:    "rootfs-empty",
+			packageType: "deb",
+		},
+		{
+			hostRoot:    "rootfs-empty",
+			packageType: "rpm",
+		},
+		{
+			hostRoot:      "rootfs-empty",
+			packageType:   "deb",
+			cdiEnabled:    true,
+			expectedError: fmt.Errorf("no NVIDIA device nodes found"),
+		},
+		{
+			hostRoot:    "rootfs-1",
+			packageType: "deb",
+			cdiEnabled:  true,
+			expectedCdiSpec: `---
+cdiVersion: 0.5.0
+kind: example.com/class
+devices:
+    - name: all
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: /host/driver/root/dev/nvidia0
+            - path: /dev/nvidiactl
+              hostPath: /host/driver/root/dev/nvidiactl
+            - path: /dev/nvidia-caps-imex-channels/channel0
+              hostPath: /host/driver/root/dev/nvidia-caps-imex-channels/channel0
+            - path: /dev/nvidia-caps-imex-channels/channel1
+              hostPath: /host/driver/root/dev/nvidia-caps-imex-channels/channel1
+            - path: /dev/nvidia-caps-imex-channels/channel2047
+              hostPath: /host/driver/root/dev/nvidia-caps-imex-channels/channel2047
+containerEdits:
+    env:
+        - NVIDIA_CTK_LIBCUDA_DIR=/lib/x86_64-linux-gnu
+        - NVIDIA_VISIBLE_DEVICES=void
+    hooks:
+        - hookName: createContainer
+          path: {{ .toolkitRoot }}/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-symlinks
+            - --link
+            - libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: {{ .toolkitRoot }}/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-soname-symlinks
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: {{ .toolkitRoot }}/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - update-ldcache
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+    mounts:
+        - hostPath: /host/driver/root/lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - rbind
+            - rprivate
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		// hostRoot := filepath.Join(moduleRoot, "testdata", "lookup", tc.hostRoot)
+		t.Run(tc.description, func(t *testing.T) {
+			testRoot := t.TempDir()
+			toolkitRoot := filepath.Join(testRoot, "toolkit-test")
+			cdiOutputDir := filepath.Join(moduleRoot, "toolkit-test", "/var/cdi")
+			sourceRoot := filepath.Join(artifactRoot, tc.packageType)
+			options := Options{
+				DriverRoot:        "/host/driver/root",
+				DriverRootCtrPath: filepath.Join(moduleRoot, "testdata", "lookup", tc.hostRoot),
+				CDI: cdiOptions{
+					Enabled:   tc.cdiEnabled,
+					outputDir: cdiOutputDir,
+					kind:      "example.com/class",
+				},
+			}
+
+			ti := NewInstaller(
+				WithLogger(logger),
+				WithToolkitRoot(toolkitRoot),
+				WithSourceRoot(sourceRoot),
+			)
+			require.NoError(t, ti.ValidateOptions(&options))
+
+			err := ti.Install(&cli.Context{}, &options)
+			if tc.expectedError == nil {
+				require.NoError(t, err)
+			} else {
+				require.Contains(t, err.Error(), tc.expectedError.Error())
+			}
+
+			require.DirExists(t, toolkitRoot)
+			requireSymlink(t, toolkitRoot, "libnvidia-container.so.1", "libnvidia-container.so.99.88.77")
+			requireSymlink(t, toolkitRoot, "libnvidia-container-go.so.1", "libnvidia-container-go.so.99.88.77")
+
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-cdi-hook")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-cli")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime-hook")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime.cdi")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-container-runtime.legacy")
+			requireWrappedExecutable(t, toolkitRoot, "nvidia-ctk")
+
+			requireSymlink(t, toolkitRoot, "nvidia-container-toolkit", "nvidia-container-runtime-hook")
+
+			// TODO: Add checks for wrapper contents
+			// grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+			// grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+
+			require.DirExists(t, filepath.Join(toolkitRoot, ".config"))
+			require.DirExists(t, filepath.Join(toolkitRoot, ".config", "nvidia-container-runtime"))
+			require.FileExists(t, filepath.Join(toolkitRoot, ".config", "nvidia-container-runtime", "config.toml"))
+
+			cfgToml, err := config.New(config.WithConfigFile(filepath.Join(toolkitRoot, ".config", "nvidia-container-runtime", "config.toml")))
+			require.NoError(t, err)
+
+			cfg, err := cfgToml.Config()
+			require.NoError(t, err)
+
+			// Ensure that the config file has the required contents.
+			// TODO: Add checks for additional config options.
+			require.Equal(t, "/host/driver/root", cfg.NVIDIAContainerCLIConfig.Root)
+			require.Equal(t, "@/host/driver/root/sbin/ldconfig", string(cfg.NVIDIAContainerCLIConfig.Ldconfig))
+			require.EqualValues(t, filepath.Join(toolkitRoot, "nvidia-container-cli"), cfg.NVIDIAContainerCLIConfig.Path)
+			require.EqualValues(t, filepath.Join(toolkitRoot, "nvidia-ctk"), cfg.NVIDIACTKConfig.Path)
+
+			if len(tc.expectedCdiSpec) > 0 {
+				cdiSpecFile := filepath.Join(cdiOutputDir, "example.com-class.yaml")
+				require.FileExists(t, cdiSpecFile)
+				info, err := os.Stat(cdiSpecFile)
+				require.NoError(t, err)
+				require.NotZero(t, info.Mode()&0004)
+				contents, err := os.ReadFile(cdiSpecFile)
+				require.NoError(t, err)
+				require.Equal(t, strings.ReplaceAll(tc.expectedCdiSpec, "{{ .toolkitRoot }}", toolkitRoot), string(contents))
+			}
+		})
+	}
+}
+
+func requireWrappedExecutable(t *testing.T, toolkitRoot string, expectedExecutable string) {
+	requireExecutable(t, toolkitRoot, expectedExecutable)
+	requireExecutable(t, toolkitRoot, expectedExecutable+".real")
+}
+
+func requireExecutable(t *testing.T, toolkitRoot string, expectedExecutable string) {
+	executable := filepath.Join(toolkitRoot, expectedExecutable)
+	require.FileExists(t, executable)
+	info, err := os.Lstat(executable)
+	require.NoError(t, err)
+	require.Zero(t, info.Mode()&os.ModeSymlink)
+	require.NotZero(t, info.Mode()&0111)
+}
+
+func requireSymlink(t *testing.T, toolkitRoot string, expectedLink string, expectedTarget string) {
+	link := filepath.Join(toolkitRoot, expectedLink)
+	require.FileExists(t, link)
+	target, err := symlinks.Resolve(link)
+	require.NoError(t, err)
+	require.Equal(t, expectedTarget, target)
+}

cmd/nvidia-ctk/README.md

@@ -1,3 +1,73 @@
 # NVIDIA Container Toolkit CLI
 
 The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.
+
+## Configure the NVIDIA Container Toolkit
+
+The `config` command of the `nvidia-ctk` CLI allows a user to display and manipulate the NVIDIA Container Toolkit
+configuration.
+
+For example, running the following command:
+```bash
+nvidia-ctk config default
+```
+will display the default config for the detected platform.
+
+Whereas
+```bash
+nvidia-ctk config
+```
+will display the effective NVIDIA Container Toolkit config using the configured config file, and running:
+
+Individual config options can be set by specifying these are key-value pairs to the `--set` argument:
+
+```bash
+nvidia-ctk config --set nvidia-container-cli.no-cgroups=true
+```
+
+By default, all commands output to `STDOUT`, but specifying the `--output` flag writes the config to the specified file.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://tags.cncf.io/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,55 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/list"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+		transform.NewCommand(m.logger),
+		list.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,337 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	cdi "tags.cncf.io/container-device-interface/pkg/parser"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
+)
+
+const (
+	allDeviceName = "all"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	output               string
+	format               string
+	deviceNameStrategies cli.StringSlice
+	driverRoot           string
+	devRoot              string
+	nvidiaCDIHookPath    string
+	ldconfigPath         string
+	mode                 string
+	vendor               string
+	class                string
+
+	configSearchPaths  cli.StringSlice
+	librarySearchPaths cli.StringSlice
+	disabledHooks      cli.StringSlice
+
+	csv struct {
+		files          cli.StringSlice
+		ignorePatterns cli.StringSlice
+	}
+
+	// the following are used for dependency injection during spec generation.
+	nvmllib nvml.Interface
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "config-search-path",
+			Usage:       "Specify the path to search for config files when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.configSearchPaths,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_CONFIG_SEARCH_PATHS"},
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_OUTPUT_FILE_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       spec.FormatYAML,
+			Destination: &opts.format,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_OUTPUT_FORMAT"},
+		},
+		&cli.StringFlag{
+			Name:    "mode",
+			Aliases: []string{"discovery-mode"},
+			Usage: "The mode to use when discovering the available entities. " +
+				"One of [" + strings.Join(nvcdi.AllModes[string](), " | ") + "]. " +
+				"If mode is set to 'auto' the mode will be determined based on the system configuration.",
+			Value:       string(nvcdi.ModeAuto),
+			Destination: &opts.mode,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "dev-root",
+			Usage:       "Specify the root where `/dev` is located. If this is not specified, the driver-root is assumed.",
+			Destination: &opts.devRoot,
+			EnvVars:     []string{"NVIDIA_CTK_DEV_ROOT"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. If this is specified multiple times, the devices will be duplicated for each strategy. One of [index | uuid | type-index]",
+			Value:       cli.NewStringSlice(nvcdi.DeviceNameStrategyIndex, nvcdi.DeviceNameStrategyUUID),
+			Destination: &opts.deviceNameStrategies,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_DEVICE_NAME_STRATEGIES"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &opts.driverRoot,
+			EnvVars:     []string{"NVIDIA_CTK_DRIVER_ROOT"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "library-search-path",
+			Usage:       "Specify the path to search for libraries when discovering the entities that should be included in the CDI specification.\n\tNote: This option only applies to CSV mode.",
+			Destination: &opts.librarySearchPaths,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_LIBRARY_SEARCH_PATHS"},
+		},
+		&cli.StringFlag{
+			Name:    "nvidia-cdi-hook-path",
+			Aliases: []string{"nvidia-ctk-path"},
+			Usage: "Specify the path to use for the nvidia-cdi-hook in the generated CDI specification. " +
+				"If not specified, the PATH will be searched for `nvidia-cdi-hook`. " +
+				"NOTE: That if this is specified as `nvidia-ctk`, the PATH will be searched for `nvidia-ctk` instead.",
+			Destination: &opts.nvidiaCDIHookPath,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_HOOK_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "ldconfig-path",
+			Usage:       "Specify the path to use for ldconfig in the generated CDI specification",
+			Destination: &opts.ldconfigPath,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_LDCONFIG_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "vendor",
+			Aliases:     []string{"cdi-vendor"},
+			Usage:       "the vendor string to use for the generated CDI specification.",
+			Value:       "nvidia.com",
+			Destination: &opts.vendor,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_VENDOR"},
+		},
+		&cli.StringFlag{
+			Name:        "class",
+			Aliases:     []string{"cdi-class"},
+			Usage:       "the class string to use for the generated CDI specification.",
+			Value:       "gpu",
+			Destination: &opts.class,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_CLASS"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.file",
+			Usage:       "The path to the list of CSV files to use when generating the CDI specification in CSV mode.",
+			Value:       cli.NewStringSlice(csv.DefaultFileList()...),
+			Destination: &opts.csv.files,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_CSV_FILES"},
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv.ignore-pattern",
+			Usage:       "specify a pattern the CSV mount specifications.",
+			Destination: &opts.csv.ignorePatterns,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_CSV_IGNORE_PATTERNS"},
+		},
+		&cli.StringSliceFlag{
+			Name:    "disable-hook",
+			Aliases: []string{"disable-hooks"},
+			Usage: "specify a specific hook to skip when generating CDI " +
+				"specifications. This can be specified multiple times and the " +
+				"special hook name 'all' can be used ensure that the generated " +
+				"CDI specification does not include any hooks.",
+			Destination: &opts.disabledHooks,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_GENERATE_DISABLED_HOOKS"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	opts.format = strings.ToLower(opts.format)
+	switch opts.format {
+	case spec.FormatJSON:
+	case spec.FormatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", opts.format)
+	}
+
+	opts.mode = strings.ToLower(opts.mode)
+	if !nvcdi.IsValidMode(opts.mode) {
+		return fmt.Errorf("invalid discovery mode: %v", opts.mode)
+	}
+
+	for _, strategy := range opts.deviceNameStrategies.Value() {
+		_, err := nvcdi.NewDeviceNamer(strategy)
+		if err != nil {
+			return err
+		}
+	}
+
+	opts.nvidiaCDIHookPath = config.ResolveNVIDIACDIHookPath(m.logger, opts.nvidiaCDIHookPath)
+
+	if outputFileFormat := formatFromFilename(opts.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			opts.format = outputFileFormat
+		} else if outputFileFormat != opts.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", opts.format, outputFileFormat)
+		}
+	}
+
+	if err := cdi.ValidateVendorName(opts.vendor); err != nil {
+		return fmt.Errorf("invalid CDI vendor name: %v", err)
+	}
+	if err := cdi.ValidateClassName(opts.class); err != nil {
+		return fmt.Errorf("invalid CDI class name: %v", err)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := m.generateSpec(opts)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
+
+	if opts.output == "" {
+		_, err := spec.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return spec.Save(opts.output)
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return spec.FormatJSON
+	case ".yaml", ".yml":
+		return spec.FormatYAML
+	}
+
+	return ""
+}
+
+func (m command) generateSpec(opts *options) (spec.Interface, error) {
+	var deviceNamers []nvcdi.DeviceNamer
+	for _, strategy := range opts.deviceNameStrategies.Value() {
+		deviceNamer, err := nvcdi.NewDeviceNamer(strategy)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create device namer: %v", err)
+		}
+		deviceNamers = append(deviceNamers, deviceNamer)
+	}
+
+	cdiOptions := []nvcdi.Option{
+		nvcdi.WithLogger(m.logger),
+		nvcdi.WithDriverRoot(opts.driverRoot),
+		nvcdi.WithDevRoot(opts.devRoot),
+		nvcdi.WithNVIDIACDIHookPath(opts.nvidiaCDIHookPath),
+		nvcdi.WithLdconfigPath(opts.ldconfigPath),
+		nvcdi.WithDeviceNamers(deviceNamers...),
+		nvcdi.WithMode(opts.mode),
+		nvcdi.WithConfigSearchPaths(opts.configSearchPaths.Value()),
+		nvcdi.WithLibrarySearchPaths(opts.librarySearchPaths.Value()),
+		nvcdi.WithCSVFiles(opts.csv.files.Value()),
+		nvcdi.WithCSVIgnorePatterns(opts.csv.ignorePatterns.Value()),
+		// We set the following to allow for dependency injection:
+		nvcdi.WithNvmlLib(opts.nvmllib),
+	}
+
+	for _, hook := range opts.disabledHooks.Value() {
+		cdiOptions = append(cdiOptions, nvcdi.WithDisabledHook(hook))
+	}
+
+	cdilib, err := nvcdi.New(cdiOptions...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CDI library: %v", err)
+	}
+
+	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+
+	commonEdits, err := cdilib.GetCommonEdits()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
+	}
+
+	return spec.New(
+		spec.WithVendor(opts.vendor),
+		spec.WithClass(opts.class),
+		spec.WithDeviceSpecs(deviceSpecs),
+		spec.WithEdits(*commonEdits.ContainerEdits),
+		spec.WithFormat(opts.format),
+		spec.WithMergedDeviceOptions(
+			transform.WithName(allDeviceName),
+			transform.WithSkipIfExists(true),
+		),
+		spec.WithPermissions(0644),
+	)
+}

cmd/nvidia-ctk/cdi/generate/generate_test.go

@@ -0,0 +1,402 @@
+/**
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	"github.com/NVIDIA/go-nvml/pkg/nvml/mock/dgxa100"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+)
+
+func TestGenerateSpec(t *testing.T) {
+	t.Setenv("__NVCT_TESTING_DEVICES_ARE_FILES", "true")
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	driverRoot := filepath.Join(moduleRoot, "testdata", "lookup", "rootfs-1")
+
+	logger, _ := testlog.NewNullLogger()
+	testCases := []struct {
+		description           string
+		options               options
+		expectedValidateError error
+		expectedOptions       options
+		expectedError         error
+		expectedSpec          string
+	}{
+		{
+			description: "default",
+			options: options{
+				format:     "yaml",
+				mode:       "nvml",
+				vendor:     "example.com",
+				class:      "device",
+				driverRoot: driverRoot,
+			},
+			expectedOptions: options{
+				format:            "yaml",
+				mode:              "nvml",
+				vendor:            "example.com",
+				class:             "device",
+				nvidiaCDIHookPath: "/usr/bin/nvidia-cdi-hook",
+				driverRoot:        driverRoot,
+			},
+			expectedSpec: `---
+cdiVersion: 0.5.0
+kind: example.com/device
+devices:
+    - name: "0"
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+    - name: all
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+containerEdits:
+    env:
+        - NVIDIA_CTK_LIBCUDA_DIR=/lib/x86_64-linux-gnu
+        - NVIDIA_VISIBLE_DEVICES=void
+    deviceNodes:
+        - path: /dev/nvidiactl
+          hostPath: {{ .driverRoot }}/dev/nvidiactl
+    hooks:
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-symlinks
+            - --link
+            - libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - enable-cuda-compat
+            - --host-driver-version=999.88.77
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-soname-symlinks
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - update-ldcache
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - disable-device-node-modification
+          env:
+            - NVIDIA_CTK_DEBUG=false
+    mounts:
+        - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - rbind
+            - rprivate
+`,
+		},
+		{
+			description: "disableHooks1",
+			options: options{
+				format:        "yaml",
+				mode:          "nvml",
+				vendor:        "example.com",
+				class:         "device",
+				driverRoot:    driverRoot,
+				disabledHooks: valueOf(cli.NewStringSlice("enable-cuda-compat")),
+			},
+			expectedOptions: options{
+				format:            "yaml",
+				mode:              "nvml",
+				vendor:            "example.com",
+				class:             "device",
+				nvidiaCDIHookPath: "/usr/bin/nvidia-cdi-hook",
+				driverRoot:        driverRoot,
+				disabledHooks:     valueOf(cli.NewStringSlice("enable-cuda-compat")),
+			},
+			expectedSpec: `---
+cdiVersion: 0.5.0
+kind: example.com/device
+devices:
+    - name: "0"
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+    - name: all
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+containerEdits:
+    env:
+        - NVIDIA_CTK_LIBCUDA_DIR=/lib/x86_64-linux-gnu
+        - NVIDIA_VISIBLE_DEVICES=void
+    deviceNodes:
+        - path: /dev/nvidiactl
+          hostPath: {{ .driverRoot }}/dev/nvidiactl
+    hooks:
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-symlinks
+            - --link
+            - libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-soname-symlinks
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - update-ldcache
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - disable-device-node-modification
+          env:
+            - NVIDIA_CTK_DEBUG=false
+    mounts:
+        - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - rbind
+            - rprivate
+`,
+		},
+		{
+			description: "disableHooks2",
+			options: options{
+				format:        "yaml",
+				mode:          "nvml",
+				vendor:        "example.com",
+				class:         "device",
+				driverRoot:    driverRoot,
+				disabledHooks: valueOf(cli.NewStringSlice("enable-cuda-compat", "update-ldcache")),
+			},
+			expectedOptions: options{
+				format:            "yaml",
+				mode:              "nvml",
+				vendor:            "example.com",
+				class:             "device",
+				nvidiaCDIHookPath: "/usr/bin/nvidia-cdi-hook",
+				driverRoot:        driverRoot,
+				disabledHooks:     valueOf(cli.NewStringSlice("enable-cuda-compat", "update-ldcache")),
+			},
+			expectedSpec: `---
+cdiVersion: 0.5.0
+kind: example.com/device
+devices:
+    - name: "0"
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+    - name: all
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+containerEdits:
+    env:
+        - NVIDIA_CTK_LIBCUDA_DIR=/lib/x86_64-linux-gnu
+        - NVIDIA_VISIBLE_DEVICES=void
+    deviceNodes:
+        - path: /dev/nvidiactl
+          hostPath: {{ .driverRoot }}/dev/nvidiactl
+    hooks:
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-symlinks
+            - --link
+            - libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - create-soname-symlinks
+            - --folder
+            - /lib/x86_64-linux-gnu
+          env:
+            - NVIDIA_CTK_DEBUG=false
+        - hookName: createContainer
+          path: /usr/bin/nvidia-cdi-hook
+          args:
+            - nvidia-cdi-hook
+            - disable-device-node-modification
+          env:
+            - NVIDIA_CTK_DEBUG=false
+    mounts:
+        - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - rbind
+            - rprivate
+`,
+		},
+		{
+			description: "disableHooksAll",
+			options: options{
+				format:        "yaml",
+				mode:          "nvml",
+				vendor:        "example.com",
+				class:         "device",
+				driverRoot:    driverRoot,
+				disabledHooks: valueOf(cli.NewStringSlice("all")),
+			},
+			expectedOptions: options{
+				format:            "yaml",
+				mode:              "nvml",
+				vendor:            "example.com",
+				class:             "device",
+				nvidiaCDIHookPath: "/usr/bin/nvidia-cdi-hook",
+				driverRoot:        driverRoot,
+				disabledHooks:     valueOf(cli.NewStringSlice("all")),
+			},
+			expectedSpec: `---
+cdiVersion: 0.5.0
+kind: example.com/device
+devices:
+    - name: "0"
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+    - name: all
+      containerEdits:
+        deviceNodes:
+            - path: /dev/nvidia0
+              hostPath: {{ .driverRoot }}/dev/nvidia0
+containerEdits:
+    env:
+        - NVIDIA_CTK_LIBCUDA_DIR=/lib/x86_64-linux-gnu
+        - NVIDIA_VISIBLE_DEVICES=void
+    deviceNodes:
+        - path: /dev/nvidiactl
+          hostPath: {{ .driverRoot }}/dev/nvidiactl
+    mounts:
+        - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77
+          options:
+            - ro
+            - nosuid
+            - nodev
+            - rbind
+            - rprivate
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			c := command{
+				logger: logger,
+			}
+
+			err := c.validateFlags(nil, &tc.options)
+			require.ErrorIs(t, err, tc.expectedValidateError)
+			require.EqualValues(t, tc.expectedOptions, tc.options)
+
+			// Set up a mock server, reusing the DGX A100 mock.
+			server := dgxa100.New()
+			// Override the driver version to match the version in our mock filesystem.
+			server.SystemGetDriverVersionFunc = func() (string, nvml.Return) {
+				return "999.88.77", nvml.SUCCESS
+			}
+			// Set the device count to 1 explicitly since we only have a single device node.
+			server.DeviceGetCountFunc = func() (int, nvml.Return) {
+				return 1, nvml.SUCCESS
+			}
+			for _, d := range server.Devices {
+				// TODO: This is not implemented in the mock.
+				(d.(*dgxa100.Device)).GetMaxMigDeviceCountFunc = func() (int, nvml.Return) {
+					return 0, nvml.SUCCESS
+				}
+			}
+			tc.options.nvmllib = server
+
+			spec, err := c.generateSpec(&tc.options)
+			require.ErrorIs(t, err, tc.expectedError)
+
+			var buf bytes.Buffer
+			_, err = spec.WriteTo(&buf)
+			require.NoError(t, err)
+
+			require.Equal(t, strings.ReplaceAll(tc.expectedSpec, "{{ .driverRoot }}", driverRoot), buf.String())
+		})
+	}
+}
+
+// valueOf returns the value of a pointer.
+// Note that this does not check for a nil pointer and is only used for testing.
+func valueOf[T any](v *T) T {
+	return *v
+}

cmd/nvidia-ctk/cdi/list/list.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package list
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type config struct {
+	cdiSpecDirs cli.StringSlice
+}
+
+// NewCommand constructs a cdi list command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the command
+	c := cli.Command{
+		Name:  "list",
+		Usage: "List the available CDI devices",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "spec-dir",
+			Usage:       "specify the directories to scan for CDI specifications",
+			Value:       cli.NewStringSlice(cdi.DefaultSpecDirs...),
+			Destination: &cfg.cdiSpecDirs,
+			EnvVars:     []string{"NVIDIA_CTK_CDI_SPEC_DIRS"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *config) error {
+	if len(cfg.cdiSpecDirs.Value()) == 0 {
+		return errors.New("at least one CDI specification directory must be specified")
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	registry, err := cdi.NewCache(
+		cdi.WithAutoRefresh(false),
+		cdi.WithSpecDirs(cfg.cdiSpecDirs.Value()...),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create CDI cache: %v", err)
+	}
+
+	_ = registry.Refresh()
+	if errors := registry.GetErrors(); len(errors) > 0 {
+		m.logger.Warningf("The following registry errors were reported:")
+		for k, err := range errors {
+			m.logger.Warningf("%v: %v", k, err)
+		}
+	}
+
+	devices := registry.ListDevices()
+	m.logger.Infof("Found %d CDI devices", len(devices))
+	for _, device := range devices {
+		fmt.Printf("%s\n", device)
+	}
+
+	return nil
+}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -0,0 +1,169 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package root
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
+	transformroot "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform/root"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type transformOptions struct {
+	input  string
+	output string
+}
+
+type options struct {
+	transformOptions
+	from       string
+	to         string
+	relativeTo string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	c := cli.Command{
+		Name:  "root",
+		Usage: "Apply a root transform to a CDI specification",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &opts)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "from",
+			Usage:       "specify the root to be transformed",
+			Destination: &opts.from,
+		},
+		&cli.StringFlag{
+			Name:        "input",
+			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
+			Value:       "-",
+			Destination: &opts.input,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &opts.output,
+		},
+		&cli.StringFlag{
+			Name:        "relative-to",
+			Usage:       "specify whether the transform is relative to the host or to the container. One of [ host | container ]",
+			Value:       "host",
+			Destination: &opts.relativeTo,
+		},
+		&cli.StringFlag{
+			Name:        "to",
+			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
+			Value:       "",
+			Destination: &opts.to,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, opts *options) error {
+	switch opts.relativeTo {
+	case "host":
+	case "container":
+	default:
+		return fmt.Errorf("invalid --relative-to value: %v", opts.relativeTo)
+	}
+	return nil
+}
+
+func (m command) run(c *cli.Context, opts *options) error {
+	spec, err := opts.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load CDI specification: %w", err)
+	}
+
+	err = transformroot.New(
+		transformroot.WithRoot(opts.from),
+		transformroot.WithTargetRoot(opts.to),
+		transformroot.WithRelativeTo(opts.relativeTo),
+	).Transform(spec.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to transform CDI specification: %w", err)
+	}
+
+	return opts.Save(spec)
+}
+
+// Load lodas the input CDI specification
+func (o transformOptions) Load() (spec.Interface, error) {
+	contents, err := o.getContents()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read spec contents: %v", err)
+	}
+
+	raw, err := cdi.ParseSpec(contents)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
+	}
+
+	return spec.New(
+		spec.WithRawSpec(raw),
+	)
+}
+
+func (o transformOptions) getContents() ([]byte, error) {
+	if o.input == "-" {
+		return io.ReadAll(os.Stdin)
+	}
+
+	return os.ReadFile(o.input)
+}
+
+// Save saves the CDI specification to the output file
+func (o transformOptions) Save(s spec.Interface) error {
+	if o.output == "" {
+		_, err := s.WriteTo(os.Stdout)
+		if err != nil {
+			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
+		}
+		return nil
+	}
+
+	return s.Save(o.output)
+}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -0,0 +1,52 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package transform
+
+import (
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// NewCommand constructs a command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	c := cli.Command{
+		Name:  "transform",
+		Usage: "Apply a transform to a CDI specification",
+	}
+
+	c.Flags = []cli.Flag{}
+
+	c.Subcommands = []*cli.Command{
+		root.NewCommand(m.logger),
+	}
+
+	return &c
+}

cmd/nvidia-ctk/config/config.go

@@ -0,0 +1,251 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	createdefault "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/create-default"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/config/flags"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+// options stores the subcommand options
+type options struct {
+	flags.Options
+	setListSeparator string
+	sets             cli.StringSlice
+}
+
+// NewCommand constructs an config command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	opts := options{}
+
+	// Create the 'config' command
+	c := cli.Command{
+		Name:  "config",
+		Usage: "Interact with the NVIDIA Container Toolkit configuration",
+		Before: func(ctx *cli.Context) error {
+			return validateFlags(ctx, &opts)
+		},
+		Action: func(ctx *cli.Context) error {
+			return run(ctx, &opts)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config-file",
+			Aliases:     []string{"config", "c"},
+			Usage:       "Specify the config file to modify.",
+			Value:       config.GetConfigFilePath(),
+			Destination: &opts.Config,
+		},
+		&cli.StringSliceFlag{
+			Name: "set",
+			Usage: "Set a config value using the pattern 'key[=value]'. " +
+				"Specifying only 'key' is equivalent to 'key=true' for boolean settings. " +
+				"This flag can be specified multiple times, but only the last value for a specific " +
+				"config option is applied. " +
+				"If the setting represents a list, the elements are colon-separated.",
+			Destination: &opts.sets,
+		},
+		&cli.StringFlag{
+			Name:        "set-list-separator",
+			Usage:       "Specify a separator for lists applied using the set command.",
+			Hidden:      true,
+			Value:       ":",
+			Destination: &opts.setListSeparator,
+		},
+		&cli.BoolFlag{
+			Name:        "in-place",
+			Aliases:     []string{"i"},
+			Usage:       "Modify the config file in-place",
+			Destination: &opts.InPlace,
+		},
+		&cli.StringFlag{
+			Name:        "output",
+			Aliases:     []string{"o"},
+			Usage:       "Specify the output file to write to; If not specified, the output is written to stdout",
+			Destination: &opts.Output,
+		},
+	}
+
+	c.Subcommands = []*cli.Command{
+		createdefault.NewCommand(m.logger),
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, opts *options) error {
+	if opts.setListSeparator == "" {
+		return fmt.Errorf("set-list-separator must be set")
+	}
+	return nil
+}
+
+func run(c *cli.Context, opts *options) error {
+	cfgToml, err := config.New(
+		config.WithConfigFile(opts.Config),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create config: %v", err)
+	}
+
+	for _, set := range opts.sets.Value() {
+		key, value, err := setFlagToKeyValue(set, opts.setListSeparator)
+		if err != nil {
+			return fmt.Errorf("invalid --set option %v: %w", set, err)
+		}
+		if value == nil {
+			_ = cfgToml.Delete(key)
+		} else {
+			cfgToml.Set(key, value)
+		}
+	}
+
+	if err := opts.EnsureOutputFolder(); err != nil {
+		return fmt.Errorf("failed to create output directory: %v", err)
+	}
+	output, err := opts.CreateOutput()
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %v", err)
+	}
+	defer output.Close()
+
+	if _, err := cfgToml.Save(output); err != nil {
+		return fmt.Errorf("failed to save config: %v", err)
+	}
+
+	return nil
+}
+
+var errInvalidConfigOption = errors.New("invalid config option")
+var errUndefinedField = errors.New("undefined field")
+var errInvalidFormat = errors.New("invalid format")
+
+// setFlagToKeyValue converts a --set flag to a key-value pair.
+// The set flag is of the form key[=value], with the value being optional if key refers to a
+// boolean config option.
+func setFlagToKeyValue(setFlag string, setListSeparator string) (string, interface{}, error) {
+	setParts := strings.SplitN(setFlag, "=", 2)
+	key := setParts[0]
+
+	field, err := getField(key)
+	if err != nil {
+		return key, nil, fmt.Errorf("%w: %w", errInvalidConfigOption, err)
+	}
+
+	kind := field.Kind()
+	if len(setParts) != 2 {
+		if kind == reflect.Bool || (kind == reflect.Pointer && field.Elem().Kind() == reflect.Bool) {
+			return key, true, nil
+		}
+		return key, nil, fmt.Errorf("%w: expected key=value; got %v", errInvalidFormat, setFlag)
+	}
+
+	value := setParts[1]
+	if kind == reflect.Pointer && value != "nil" {
+		kind = field.Elem().Kind()
+	}
+	switch kind {
+	case reflect.Pointer:
+		return key, nil, nil
+	case reflect.Bool:
+		b, err := strconv.ParseBool(value)
+		if err != nil {
+			return key, value, fmt.Errorf("%w: %w", errInvalidFormat, err)
+		}
+		return key, b, nil
+	case reflect.String:
+		return key, value, nil
+	case reflect.Slice:
+		valueParts := []string{value}
+		for _, sep := range []string{setListSeparator, ","} {
+			if !strings.Contains(value, sep) {
+				continue
+			}
+			valueParts = strings.Split(value, sep)
+			break
+		}
+		switch field.Elem().Kind() {
+		case reflect.String:
+			return key, valueParts, nil
+		case reflect.Int:
+			var output []int64
+			for _, v := range valueParts {
+				vi, err := strconv.ParseInt(v, 10, 0)
+				if err != nil {
+					return key, nil, fmt.Errorf("%w: %w", errInvalidFormat, err)
+				}
+				output = append(output, vi)
+			}
+			return key, output, nil
+		}
+	}
+	return key, nil, fmt.Errorf("unsupported type for %v (%v)", setParts, kind)
+}
+
+func getField(key string) (reflect.Type, error) {
+	s, err := getStruct(reflect.TypeOf(config.Config{}), strings.Split(key, ".")...)
+	if err != nil {
+		return nil, err
+	}
+	return s.Type, err
+}
+
+func getStruct(current reflect.Type, paths ...string) (reflect.StructField, error) {
+	if len(paths) < 1 {
+		return reflect.StructField{}, fmt.Errorf("%w: no fields selected", errUndefinedField)
+	}
+	tomlField := paths[0]
+	for i := 0; i < current.NumField(); i++ {
+		f := current.Field(i)
+		v, ok := f.Tag.Lookup("toml")
+		if !ok {
+			continue
+		}
+		if strings.SplitN(v, ",", 2)[0] != tomlField {
+			continue
+		}
+		if len(paths) == 1 {
+			return f, nil
+		}
+		return getStruct(f.Type, paths[1:]...)
+	}
+	return reflect.StructField{}, fmt.Errorf("%w: %q", errUndefinedField, tomlField)
+}