Merge branch 'fix-git-branch' into 'main'

Fix git branch shell command

See merge request nvidia/container-toolkit/container-toolkit!277

.common-ci.yml

@@ -0,0 +1,284 @@
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+default:
+  image: docker
+  services:
+    - name: docker:dind
+      command: ["--experimental"]
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
+  BUILD_MULTI_ARCH_IMAGES: "true"
+
+stages:
+  - image
+  - lint
+  - go-checks
+  - go-build
+  - unit-tests
+  - package-build
+  - image-build
+  - test
+  - scan
+  - release
+
+.main-or-manual:
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: manual
+
+# Define the distribution targets
+.dist-amazonlinux2:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: amazonlinux2
+    PACKAGE_REPO_TYPE: rpm
+
+.dist-centos7:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: centos7
+    CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
+
+.dist-centos8:
+  variables:
+    DIST: centos8
+    CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
+
+.dist-debian10:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: debian10
+    PACKAGE_REPO_TYPE: debian
+
+.dist-debian9:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: debian9
+    PACKAGE_REPO_TYPE: debian
+
+.dist-fedora35:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: fedora35
+    PACKAGE_REPO_TYPE: rpm
+
+.dist-opensuse-leap15.1:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: opensuse-leap15.1
+    PACKAGE_REPO_TYPE: rpm
+
+.dist-ubi8:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: ubi8
+    CVE_UPDATES: "cyrus-sasl-lib"
+    PACKAGE_REPO_TYPE: rpm
+
+.dist-ubuntu16.04:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: ubuntu16.04
+    PACKAGE_REPO_TYPE: debian
+
+.dist-ubuntu18.04:
+  variables:
+    DIST: ubuntu18.04
+    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+    PACKAGE_REPO_TYPE: debian
+
+.dist-ubuntu20.04:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    DIST: ubuntu20.04
+    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+    PACKAGE_REPO_TYPE: debian
+
+.dist-packaging:
+  variables:
+    DIST: packaging
+
+# Define architecture targets
+.arch-aarch64:
+  variables:
+    ARCH: aarch64
+
+.arch-amd64:
+  variables:
+    ARCH: amd64
+
+.arch-arm64:
+  variables:
+    ARCH: arm64
+
+.arch-ppc64le:
+  rules:
+    - !reference [.main-or-manual, rules]
+  variables:
+    ARCH: ppc64le
+
+.arch-x86_64:
+  variables:
+    ARCH: x86_64
+
+# Define the platform targets
+.platform-amd64:
+  variables:
+    PLATFORM: linux/amd64
+
+.platform-arm64:
+  variables:
+    PLATFORM: linux/arm64
+
+# Define test helpers
+.integration:
+  stage: test
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache make bash jq
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - make -f build/container/Makefile test-${DIST}
+
+# Define the test targets
+test-packaging:
+  extends:
+    - .integration
+    - .dist-packaging
+  needs:
+    - image-packaging
+
+# Download the regctl binary for use in the release steps
+.regctl-setup:
+  before_script:
+    - export REGCTL_VERSION=v0.4.5
+    - apk add --no-cache curl
+    - mkdir -p bin
+    - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
+    - chmod a+x bin/regctl
+    - export PATH=$(pwd)/bin:${PATH}
+
+# .release forms the base of the deployment jobs which push images to the CI registry.
+# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
+# target os.
+.release:
+  stage: release
+  variables:
+    # Define the source image for the release
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    # OUT_IMAGE_VERSION is overridden for external releases
+    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - !reference [.regctl-setup, before_script]
+
+    # We ensure that the OUT_IMAGE_VERSION is set
+    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
+
+    # In the case where we are deploying a different version to the CI_COMMIT_SHA, we
+    # need to tag the image.
+    # Note: a leading 'v' is stripped from the version if present
+    - apk add --no-cache make bash
+  script:
+    # Log in to the "output" registry, tag the image and push the image
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
+    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"'
+    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"'
+
+    # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
+    # Target
+    - make -f build/container/Makefile push-${DIST}
+
+# Define a staging release step that pushes an image to an internal "staging" repository
+# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
+# outside of the release process.
+.release:staging:
+  extends:
+    - .release
+  variables:
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
+
+# Define an external release step that pushes an image to an external repository.
+# This includes a devlopment image off main.
+.release:external:
+  extends:
+    - .release
+  rules:
+    - if: $CI_COMMIT_TAG
+      variables:
+        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
+    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
+      variables:
+        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
+
+# Define the release jobs
+release:staging-centos7:
+  extends:
+    - .release:staging
+    - .dist-centos7
+  needs:
+    - image-centos7
+
+release:staging-ubi8:
+  extends:
+    - .release:staging
+    - .dist-ubi8
+  needs:
+    - image-ubi8
+
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - test-toolkit-ubuntu18.04
+    - test-containerd-ubuntu18.04
+    - test-crio-ubuntu18.04
+    - test-docker-ubuntu18.04
+
+release:staging-ubuntu20.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu20.04
+  needs:
+    - image-ubuntu20.04
+
+release:staging-packaging:
+  extends:
+    - .release:staging
+    - .dist-packaging
+  needs:
+    - test-packaging

.dockerignore

@@ -1,2 +1,2 @@
 .git
-dist
+/shared-*

.gitignore

@@ -1,3 +1,11 @@
 dist
 *.swp
 *.swo
+/coverage.out*
+/test/output/
+/nvidia-container-runtime
+/nvidia-container-runtime-hook
+/nvidia-container-toolkit
+/nvidia-ctk
+/shared-*
+/release-*

.gitlab-ci.yml

@@ -1,104 +1,373 @@
-# Build packages for all supported OS / ARCH combinations
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-stages:
-  - build-one
-  - build-all
+include:
+  - .common-ci.yml
 
-.build-setup: &build-setup
-  image: docker:19.03.8
+build-dev-image:
+  stage: image
+  script:
+    - apk --no-cache add make bash
+    - make .build-image
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - make .push-build-image
 
-  services:
-    - name: docker:19.03.8-dind
-      command: ["--experimental"]
+.requires-build-image:
+  image: "${BUILDIMAGE}"
 
+.go-check:
+  extends:
+    - .requires-build-image
+  stage: go-checks
+
+fmt:
+  extends:
+    - .go-check
+  script:
+    - make assert-fmt
+
+vet:
+  extends:
+    - .go-check
+  script:
+    - make vet
+
+lint:
+  extends:
+    - .go-check
+  script:
+    - make lint
+  allow_failure: true
+
+ineffassign:
+  extends:
+    - .go-check
+  script:
+    - make ineffassign
+  allow_failure: true
+
+misspell:
+  extends:
+    - .go-check
+  script:
+    - make misspell
+
+go-build:
+  extends:
+    - .requires-build-image
+  stage: go-build
+  script:
+    - make build
+
+unit-tests:
+  extends:
+    - .requires-build-image
+  stage: unit-tests
+  script:
+    - make coverage
+
+# Define the package build helpers
+.multi-arch-build:
   before_script:
-  - apk update
-  - apk upgrade
-  - apk add coreutils build-base sed git bash make
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
-
-# build-one jobs build packages for a single OS / ARCH combination.
-#
-# They are run during the first stage of the pipeline as a smoke test to ensure
-# that we can successfully build packages on all of our architectures for a
-# single OS. They are triggered on any change to an MR. No artifacts are
-# produced as part of build-one jobs.
-.build-one-setup: &build-one-setup
-  <<: *build-setup
-  stage: build-one
-  only:
-    - merge_requests
-
-# build-all jobs build packages for every OS / ARCH combination we support.
-#
-# They are run under two conditions:
-# 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0)
-# 2) Manually by a reviewer just before merging a MR.
-#
-# Unlike build-one jobs, it takes a long time to build the full suite
-# OS / ARCH combinations, so this is optimized to only run once per MR
-# (assuming it all passes). A full set of artifacts including the packages
-# built for each OS / ARCH are produced as a result of these jobs.
-.build-all-setup: &build-all-setup
-  <<: *build-setup
-  stage: build-all
-  timeout: 2h 30m
-  rules:
-    - if: $CI_COMMIT_TAG
-      when: always
-    - if: $CI_MERGE_REQUEST_ID
-      when: manual
+    - apk add --no-cache coreutils build-base sed git bash make
+    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
 
+.package-artifacts:
   variables:
-    ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}"
-    ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}"
-    DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_DIR}"
+    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
+    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
+    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
+
+.package-build:
+  extends:
+    - .multi-arch-build
+    - .package-artifacts
+  stage: package-build
+  timeout: 3h
+  script:
+    - ./scripts/build-packages.sh ${DIST}-${ARCH}
 
   artifacts:
     name: ${ARTIFACTS_NAME}
     paths:
-      - ${ARTIFACTS_DIR}
+      - ${ARTIFACTS_ROOT}
 
-# The full set of build-one jobs organizes to build
-# ubuntu18.04 in parallel on each of our supported ARCHs.
-build-one-amd64:
-  <<: *build-one-setup
-  script:
-    - make ubuntu18.04-amd64
+# Define the package build targets
+package-amazonlinux2-aarch64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-aarch64
 
-build-one-ppc64le:
-  <<: *build-one-setup
-  script:
-    - make ubuntu18.04-ppc64le
+package-amazonlinux2-x86_64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-x86_64
 
-build-one-arm64:
-  <<: *build-one-setup
-  script:
-    - make ubuntu18.04-arm64
+package-centos7-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-ppc64le
 
-# The full set of build-all jobs organized to
-# have builds for each ARCH run in parallel.
-build-all-amd64:
-  <<: *build-all-setup
-  script:
-    - make docker-amd64
+package-centos7-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
 
-build-all-x86_64:
-  <<: *build-all-setup
-  script:
-    - make docker-x86_64
+package-centos8-aarch64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-aarch64
 
-build-all-ppc64le:
-  <<: *build-all-setup
-  script:
-    - make docker-ppc64le
+package-centos8-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-ppc64le
 
-build-all-arm64:
-  <<: *build-all-setup
-  script:
-    - make docker-arm64
+package-centos8-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
 
-build-all-aarch64:
-  <<: *build-all-setup
+package-debian10-amd64:
+  extends:
+    - .package-build
+    - .dist-debian10
+    - .arch-amd64
+
+package-debian9-amd64:
+  extends:
+    - .package-build
+    - .dist-debian9
+    - .arch-amd64
+
+package-fedora35-aarch64:
+  extends:
+    - .package-build
+    - .dist-fedora35
+    - .arch-aarch64
+
+package-fedora35-x86_64:
+  extends:
+    - .package-build
+    - .dist-fedora35
+    - .arch-x86_64
+
+package-opensuse-leap15.1-x86_64:
+  extends:
+    - .package-build
+    - .dist-opensuse-leap15.1
+    - .arch-x86_64
+
+package-ubuntu16.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-amd64
+
+package-ubuntu16.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-ppc64le
+
+package-ubuntu18.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+package-ubuntu18.04-arm64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+package-ubuntu18.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
+.buildx-setup:
+  before_script:
+    -  export BUILDX_VERSION=v0.6.3
+    -  apk add --no-cache curl
+    -  mkdir -p ~/.docker/cli-plugins
+    -  curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
+    -  chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+    -  docker buildx create --use --platform=linux/amd64,linux/arm64
+
+    -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
+
+# Define the image build targets
+.image-build:
+  stage: image-build
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PUSH_ON_BUILD: "true"
+  before_script:
+    - !reference [.buildx-setup, before_script]
+
+    - apk add --no-cache bash make git
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
-    - make docker-aarch64
+    - make -f build/container/Makefile build-${DIST}
+
+image-centos7:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos7
+  needs:
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+
+image-ubi8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubi8
+  needs:
+    # Note: The ubi8 image uses the centos8 packages
+    - package-centos8-aarch64
+    - package-centos8-x86_64
+    - package-centos8-ppc64le
+
+image-ubuntu18.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu18.04
+  needs:
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
+
+image-ubuntu20.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu20.04
+  needs:
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-packaging
+  needs:
+    - job: package-centos8-aarch64
+    - job: package-centos8-x86_64
+    - job: package-ubuntu18.04-amd64
+    - job: package-ubuntu18.04-arm64
+    - job: package-amazonlinux2-aarch64
+      optional: true
+    - job: package-amazonlinux2-x86_64
+      optional: true
+    - job: package-centos7-ppc64le
+      optional: true
+    - job: package-centos7-x86_64
+      optional: true
+    - job: package-centos8-ppc64le
+      optional: true
+    - job: package-debian10-amd64
+      optional: true
+    - job: package-debian9-amd64
+      optional: true
+    - job: package-fedora35-aarch64
+      optional: true
+    - job: package-fedora35-x86_64
+      optional: true
+    - job: package-opensuse-leap15.1-x86_64
+      optional: true
+    - job: package-ubuntu16.04-amd64
+      optional: true
+    - job: package-ubuntu16.04-ppc64le
+      optional: true
+    - job: package-ubuntu18.04-ppc64le
+      optional: true
+
+# Define publish test helpers
+.test:toolkit:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "toolkit"
+
+.test:docker:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "docker"
+
+.test:containerd:
+  # TODO: The containerd tests fail due to issues with SIGHUP.
+  # Until this is resolved with retry up to twice and allow failure here.
+  retry: 2
+  allow_failure: true
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "containerd"
+
+.test:crio:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "crio"
+
+# Define the test targets
+test-toolkit-ubuntu18.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-containerd-ubuntu18.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-crio-ubuntu18.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-docker-ubuntu18.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "third_party/libnvidia-container"]
+	path = third_party/libnvidia-container
+	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
+	branch = main
+[submodule "third_party/nvidia-container-runtime"]
+	path = third_party/nvidia-container-runtime
+	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
+	branch = main
+[submodule "third_party/nvidia-docker"]
+	path = third_party/nvidia-docker
+	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
+	branch = main

.nvidia-ci.yml

@@ -0,0 +1,260 @@
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include:
+  - local: '.common-ci.yml'
+
+default:
+  tags:
+    - cnt
+    - container-dev
+    - docker/multi-arch
+    - docker/privileged
+    - os/linux
+    - type/docker
+
+variables:
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: "/certs"
+  # Release "devel"-tagged images off the main branch
+  RELEASE_DEVEL_BRANCH: "main"
+  DEVEL_RELEASE_IMAGE_VERSION: "devel"
+  # On the multi-arch builder we don't need the qemu setup.
+  SKIP_QEMU_SETUP: "1"
+  # Define the public staging registry
+  STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
+  STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
+  # TODO: We should set the kitmaker release folder here once we have the end-to-end workflow set up
+  KITMAKER_RELEASE_FOLDER: "testing"
+
+.image-pull:
+  stage: image-build
+  variables:
+    IN_REGISTRY: "${STAGING_REGISTRY}"
+    IN_IMAGE_NAME: container-toolkit
+    IN_VERSION: "${STAGING_VERSION}"
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PUSH_MULTIPLE_TAGS: "false"
+  # We delay the job start to allow the public pipeline to generate the required images.
+  rules:
+    - when: delayed
+      start_in: 30 minutes
+  timeout: 30 minutes
+  retry:
+    max: 2
+    when:
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+  before_script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache make bash
+    - >
+      regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
+  script:
+    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
+    - make -f build/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
+
+image-centos7:
+  extends:
+    - .dist-centos7
+    - .image-pull
+
+image-ubi8:
+  extends:
+    - .dist-ubi8
+    - .image-pull
+
+image-ubuntu18.04:
+  extends:
+    - .dist-ubuntu18.04
+    - .image-pull
+
+image-ubuntu20.04:
+  extends:
+    - .dist-ubuntu20.04
+    - .image-pull
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .dist-packaging
+    - .image-pull
+
+# We skip the integration tests for the internal CI:
+.integration:
+  stage: test
+  before_script:
+    - echo "Skipped in internal CI"
+  script:
+    - echo "Skipped in internal CI"
+
+# The .scan step forms the base of the image scan operation performed before releasing
+# images.
+.scan:
+  stage: scan
+  image: "${PULSE_IMAGE}"
+  variables:
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
+    IMAGE_ARCHIVE: "container-toolkit.tar"
+  rules:
+    - if: $SKIP_SCANS != "yes"
+    - when: manual
+  before_script:
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    # TODO: We should specify the architecture here and scan all architectures
+    - docker pull --platform="${PLATFORM}" "${IMAGE}"
+    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
+    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+    - >
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+  script:
+    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+  artifacts:
+    when: always
+    expire_in: 1 week
+    paths:
+      - pulse-cli.log
+      - licenses.json
+      - sbom.json
+      - vulns.json
+      - policy_evaluation.json
+
+# Define the scan targets
+scan-centos7-amd64:
+  extends:
+    - .dist-centos7
+    - .platform-amd64
+    - .scan
+  needs:
+    - image-centos7
+
+scan-centos7-arm64:
+  extends:
+    - .dist-centos7
+    - .platform-arm64
+    - .scan
+  needs:
+    - image-centos7
+    - scan-centos7-amd64
+
+scan-ubuntu18.04-amd64:
+  extends:
+    - .dist-ubuntu18.04
+    - .platform-amd64
+    - .scan
+  needs:
+    - image-ubuntu18.04
+
+scan-ubuntu20.04-amd64:
+  extends:
+    - .dist-ubuntu20.04
+    - .platform-amd64
+    - .scan
+  needs:
+    - image-ubuntu20.04
+
+scan-ubuntu20.04-arm64:
+  extends:
+    - .dist-ubuntu20.04
+    - .platform-arm64
+    - .scan
+  needs:
+    - image-ubuntu20.04
+    - scan-ubuntu20.04-amd64
+
+scan-ubi8-amd64:
+  extends:
+    - .dist-ubi8
+    - .platform-amd64
+    - .scan
+  needs:
+    - image-ubi8
+
+scan-ubi8-arm64:
+  extends:
+    - .dist-ubi8
+    - .platform-arm64
+    - .scan
+  needs:
+    - image-ubi8
+    - scan-ubi8-amd64
+
+# Define external release helpers
+.release:ngc:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+
+.release:packages:
+  stage: release
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
+  script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache bash git
+    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
+    # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
+
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+# Define the external release targets
+# Release to NGC
+release:ngc-centos7:
+  extends:
+    - .dist-centos7
+    - .release:ngc
+
+release:ngc-ubuntu18.04:
+  extends:
+    - .dist-ubuntu18.04
+    - .release:ngc
+
+release:ngc-ubuntu20.04:
+  extends:
+    - .dist-ubuntu20.04
+    - .release:ngc
+
+release:ngc-ubi8:
+  extends:
+    - .dist-ubi8
+    - .release:ngc

CHANGELOG.md

@@ -0,0 +1,226 @@
+# NVIDIA Container Toolkit Changelog
+
+## v1.12.0-rc.4
+
+* Generate a minimum CDI spec version for improved compatibility.
+* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
+* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
+## v1.12.0-rc.3
+
+* Don't fail if by-path symlinks for DRM devices do not exist
+* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
+* Ensure that the CDI output folder is created if required
+* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
+* Add injection of Wayland JSON files
+* Add GSP firmware paths to generated CDI specification
+* Add --root flag to nvidia-ctk cdi generate command
+
+## v1.12.0-rc.2
+
+* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
+* Improve logging of errors from the NVIDIA Container Runtime
+* Improve CDI specification generation to support rootless podman
+* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
+* [libnvidia-container] Skip creation of existing files when these are already mounted
+
+## v1.12.0-rc.1
+
+* Add support for multiple Docker Swarm resources
+* Improve injection of Vulkan configurations and libraries
+* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
+* [libnvidia-container] Include NVVM compiler library in compute libs
+
+## v1.11.0
+
+* Promote v1.11.0-rc.3 to v1.11.0
+
+## v1.11.0-rc.3
+
+* Build fedora35 packages
+* Introduce an `nvidia-container-toolkit-base` package for better dependency management
+* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
+* Inject platform files into container on Tegra-based systems
+* [toolkit container] Update CUDA base images to 11.7.1
+* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
+
+## v1.11.0-rc.2
+
+* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
+* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
+
+## v1.11.0-rc.1
+
+* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
+* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
+* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
+* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
+* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
+
+## v1.10.0
+
+* Promote v1.10.0-rc.3 to v1.10.0
+
+## v1.10.0-rc.3
+
+* Use default config instead of raising an error if config file cannot be found
+* Ignore NVIDIA_REQUIRE_JETPACK* environment variables for requirement checks
+* Fix bug in detection of Tegra systems where `/sys/devices/soc0/family` is ignored
+* Fix bug where links to devices were detected as devices
+* [libnvida-container] Fix bug introduced when adding libcudadebugger.so to list of libraries
+
+## v1.10.0-rc.2
+
+* Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
+* Switch to debug logging to reduce log verbosity
+* Support logging to logs requested in command line
+* Fix bug when launching containers with relative root path (e.g. using containerd)
+* Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
+* Fix failure to locate low-level runtime if PATH envvar is unset
+* Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
+* Use csv as default mode on Tegra systems without NVML
+* Add --version flag to all CLIs
+* [libnvidia-container] Bump libtirpc to 1.3.2
+* [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
+* [libnvidia-container] Add libcudadebugger.so to list of compute libraries
+
+## v1.10.0-rc.1
+
+* Include nvidia-ctk CLI in installed binaries
+* Add experimental option to NVIDIA Container Runtime
+
+## v1.9.0
+
+* [libnvidia-container] Add additional check for Tegra in /sys/.../family file in CLI
+* [libnvidia-container] Update jetpack-specific CLI option to only load Base CSV files by default
+* [libnvidia-container] Fix bug (from 1.8.0) when mounting GSP firmware into containers without /lib to /usr/lib symlinks
+* [libnvidia-container] Update nvml.h to CUDA 11.6.1 nvML_DEV 11.6.55
+* [libnvidia-container] Update switch statement to include new brands from latest nvml.h
+* [libnvidia-container] Process all --require flags on Jetson platforms
+* [libnvidia-container] Fix long-standing issue with running ldconfig on Debian systems
+
+## v1.8.1
+
+* [libnvidia-container] Fix bug in determining cgroup root when running in nested containers
+* [libnvidia-container] Fix permission issue when determining cgroup version
+
+## v1.8.0
+
+* Promote 1.8.0-rc.2-1 to 1.8.0
+
+## v1.8.0-rc.2
+
+* Remove support for building amazonlinux1 packages
+
+## v1.8.0-rc.1
+
+* [libnvidia-container] Add support for cgroupv2
+* Release toolkit-container images from nvidia-container-toolkit repository
+
+## v1.7.0
+
+* Promote 1.7.0-rc.1-1 to 1.7.0
+ * Bump Golang version to 1.16.4
+
+## v1.7.0-rc.1
+
+* Specify containerd runtime type as string in config tools to remove dependency on containerd package
+* Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
+
+## v1.6.0
+
+* Promote 1.6.0-rc.3-1 to 1.6.0
+ * Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
+
+## v1.6.0-rc.3
+
+* Add supported-driver-capabilities config option to the nvidia-container-toolkit
+* Move OCI and command line checks for runtime to internal oci package
+
+## v1.6.0-rc.2
+
+* Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+## v1.6.0-rc.1
+
+* Add AARCH64 package for Amazon Linux 2
+* Include nvidia-container-runtime into nvidia-container-toolkit package
+
+## v1.5.1
+
+* Fix bug where Docker Swarm device selection is ignored if NVIDIA_VISIBLE_DEVICES is also set
+* Improve unit testing by using require package and adding coverage reports
+* Remove unneeded go dependencies by running go mod tidy
+* Move contents of pkg directory to cmd for CLI tools
+* Ensure make binary target explicitly sets GOOS
+
+## v1.5.0
+
+* Add dependence on libnvidia-container-tools >= 1.4.0
+* Add golang check targets to Makefile
+* Add Jenkinsfile definition for build targets
+* Move docker.mk to docker folder
+
+## v1.4.2
+
+* Add dependence on libnvidia-container-tools >= 1.3.3
+
+## v1.4.1
+
+* Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
+* Add dependence on libnvidia-container-tools >= 1.3.2
+
+## v1.4.0
+
+* Add 'compute' capability to list of defaults
+* Add dependence on libnvidia-container-tools >= 1.3.1
+
+## v1.3.0
+
+* Promote 1.3.0-rc.2-1 to 1.3.0
+* Add dependence on libnvidia-container-tools >= 1.3.0
+
+## v1.3.0-rc.2
+
+* 2c180947 Add more tests for new semantics with device list from volume mounts
+* 7c003857 Refactor accepting device lists from volume mounts as a boolean
+
+## v1.3.0-rc.1
+
+* b50d86c1 Update build system to accept a TAG variable for things like rc.x
+* fe65573b Add common CI tests for things like golint, gofmt, unit tests, etc.
+* da6fbb34 Revert "Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*"
+* a7fb3330 Flip build-all targets to run automatically on merge requests
+* 8b248b66 Rename github.com/NVIDIA/container-toolkit to nvidia-container-toolkit
+* da36874e Add new config options to pull device list from mounted files instead of ENVVAR
+
+## v1.2.1
+
+* 4e6e0ed4 Add 'ngx' to list of*all* driver capabilities
+* 2f4af743 List config.toml as a config file in the RPM SPEC
+
+## v1.2.0
+
+*  8e0aab46 Fix repo listed in changelog for debian distributions
+*  320bb6e4 Update dependence on libnvidia-container to 1.2.0
+*  6cfc8097 Update package license to match source license
+*  e7dc3cbb Fix debian copyright file
+*  d3aee3e0 Add the 'ngx' driver capability
+
+## v1.1.2
+
+* c32237f3 Add support for parsing Linux Capabilities for older OCI specs
+
+## v1.1.1
+
+* d202aded Update dependence to libnvidia-container 1.1.1
+
+## v1.1.0
+
+* 4e4de762 Update build system to support multi-arch builds
+* fcc1d116 Add support for MIG (Multi-Instance GPUs)
+* d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
+* 60f165ad Add no-pivot option to toolkit
+
+## v1.0.5
+
+* Initial release. Replaces older package nvidia-container-runtime-hook. (Closes: #XXXXXX)

DEVELOPMENT.md

@@ -0,0 +1,61 @@
+# NVIDIA Container Toolkit Release Tooling
+
+This repository allows for the components of the NVIDIA container stack to be
+built and released as the NVIDIA Container Toolkit from a single repository. The components:
+* `libnvidia-container`
+* `nvidia-container-runtime`
+* `nvidia-docker`
+are included as submodules in the `third_party` folder.
+
+The `nvidia-container-toolkit` resides in this repo directly.
+
+## Building
+
+In oder to build the packages, the following command is executed
+```sh
+./scripts/build-packages.sh TARGET
+```
+where `TARGET` is a make target that is valid for each of the sub-components.
+
+These include:
+* `ubuntu18.04-amd64`
+* `centos8-x86_64`
+
+If no `TARGET` is specified, all valid release targets are built.
+
+The packages are generated in the `dist` folder.
+
+## Testing local changes
+
+In oder to use the same build logic to be used to generate packages with local changes,
+the location of the individual components can be overridded using the: `LIBNVIDIA_CONTAINER_ROOT`,
+`NVIDIA_CONTAINER_TOOLKIT_ROOT`, `NVIDIA_CONTAINER_RUNTIME_ROOT`, and `NVIDIA_DOCKER_ROOT`
+environment variables.
+
+## Testing packages locally
+
+The [test/release](./test/release/) folder contains documentation on how the installation of local or staged packages can be tested.
+
+
+## Releasing
+
+In order to release packages required for a release, a utility script
+[`scripts/release-packages.sh`](./scripts/release-packages.sh) is provided.
+This script can be executed as follows:
+
+```bash
+GPG_LOCAL_USER="GPG_USER" \
+MASTER_KEY_PATH=/path/to/gpg-master.key \
+SUB_KEY_PATH=/path/to/gpg-subkey.key \
+    ./scripts/release-packages.sh REPO PACKAGE_REPO_ROOT [REFERENCE]
+```
+
+Where `REPO` is one of `stable` or `experimental`, `PACKAGE_REPO_ROOT` is the local path to the `libnvidia-container` repository checked out to the `gh-pages` branch, and `REFERENCE` is the git SHA that is to be released. If reference is not specified `HEAD` is assumed.
+
+This scripts performs the following basic functions:
+* Pulls the package image defined by the `REFERENCE` git SHA from the staging registry,
+* Copies the required packages to the package repository at `PACKAGE_REPO_ROOT/REPO`,
+* Signs the packages using the specified GPG keys
+
+While the last two are performed, commits are added to the package repository. These can be pushed to the relevant repository.
+

Jenkinsfile

@@ -0,0 +1,142 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+podTemplate (cloud:'sw-gpu-cloudnative',
+    containers: [
+    containerTemplate(name: 'docker', image: 'docker:dind', ttyEnabled: true, privileged: true),
+    containerTemplate(name: 'golang', image: 'golang:1.16.3', ttyEnabled: true)
+  ]) {
+    node(POD_LABEL) {
+        def scmInfo
+
+        stage('checkout') {
+            scmInfo = checkout(scm)
+        }
+
+        stage('dependencies') {
+            container('golang') {
+                sh 'GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell'
+                sh 'GO111MODULE=off go get -u github.com/gordonklaus/ineffassign'
+                sh 'GO111MODULE=off go get -u golang.org/x/lint/golint'
+            }
+            container('docker') {
+                sh 'apk add --no-cache make bash git'
+            }
+        }
+        stage('check') {
+            parallel (
+                getGolangStages(["assert-fmt", "lint", "vet", "ineffassign", "misspell"])
+            )
+        }
+        stage('test') {
+            parallel (
+                getGolangStages(["test"])
+            )
+        }
+
+        def versionInfo
+        stage('version') {
+            container('docker') {
+                versionInfo = getVersionInfo(scmInfo)
+                println "versionInfo=${versionInfo}"
+            }
+        }
+
+        def dist = 'ubuntu20.04'
+        def arch = 'amd64'
+        def stageLabel = "${dist}-${arch}"
+
+        stage('build-one') {
+            container('docker') {
+                stage (stageLabel) {
+                    sh "make ${dist}-${arch}"
+                }
+            }
+        }
+
+        stage('release') {
+            container('docker') {
+                stage (stageLabel) {
+
+                    def component = 'main'
+                    def repository = 'sw-gpu-cloudnative-debian-local/pool/main/'
+
+                    def uploadSpec = """{
+                                        "files":
+                                        [  {
+                                                "pattern": "./dist/${dist}/${arch}/*.deb",
+                                                "target": "${repository}",
+                                                "props": "deb.distribution=${dist};deb.component=${component};deb.architecture=${arch}"
+                                            }
+                                        ]
+                                    }"""
+
+                    sh "echo starting release with versionInfo=${versionInfo}"
+                    if (versionInfo.isTag) {
+                        // upload to artifactory repository
+                        def server = Artifactory.server 'sw-gpu-artifactory'
+                        server.upload spec: uploadSpec
+                    } else {
+                        sh "echo skipping release for non-tagged build"
+                    }
+                }
+            }
+        }
+    }
+}
+
+def getGolangStages(def targets) {
+    stages = [:]
+
+    for (t in targets) {
+        stages[t] = getLintClosure(t)
+    }
+
+    return stages
+}
+
+def getLintClosure(def target) {
+    return {
+        container('golang') {
+            stage(target) {
+                sh "make ${target}"
+            }
+        }
+    }
+}
+
+// getVersionInfo returns a hash of version info
+def getVersionInfo(def scmInfo) {
+    def versionInfo = [
+        isTag: isTag(scmInfo.GIT_BRANCH)
+    ]
+
+    scmInfo.each { k, v -> versionInfo[k] = v }
+    return versionInfo
+}
+
+def isTag(def branch) {
+    if (!branch.startsWith('v')) {
+        return false
+    }
+
+    def version = shOutput('git describe --all --exact-match --always')
+    return version == "tags/${branch}"
+}
+
+def shOuptut(def script) {
+    return sh(script: script, returnStdout: true).trim()
+}

Makefile

@@ -1,18 +1,160 @@
-# Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 DOCKER   ?= docker
 MKDIR    ?= mkdir
 DIST_DIR ?= $(CURDIR)/dist
 
-LIB_NAME := nvidia-container-toolkit
-LIB_VERSION := 1.2.0
+include $(CURDIR)/versions.mk
 
-GOLANG_VERSION := 1.14.2
-GOLANG_PKG_PATH := github.com/NVIDIA/container-toolkit/pkg
+MODULE := github.com/NVIDIA/nvidia-container-toolkit
 
 # By default run all native docker-based targets
 docker-native:
-include $(CURDIR)/docker.mk
+include $(CURDIR)/docker/docker.mk
 
-binary:
-	go build -ldflags "-s -w" -o "$(LIB_NAME)" $(GOLANG_PKG_PATH)
+ifeq ($(IMAGE_NAME),)
+REGISTRY ?= nvidia
+IMAGE_NAME = $(REGISTRY)/container-toolkit
+endif
+
+BUILDIMAGE_TAG ?= golang$(GOLANG_VERSION)
+BUILDIMAGE ?= $(IMAGE_NAME)-build:$(BUILDIMAGE_TAG)
+
+EXAMPLES := $(patsubst ./examples/%/,%,$(sort $(dir $(wildcard ./examples/*/))))
+EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES))
+
+CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
+CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
+
+CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate licenses $(CHECK_TARGETS)
+
+TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
+
+DOCKER_TARGETS := $(patsubst %,docker-%, $(TARGETS))
+.PHONY: $(TARGETS) $(DOCKER_TARGETS)
+
+ifeq ($(VERSION),)
+CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+else
+CLI_VERSION = $(VERSION)
+endif
+CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
+
+GOOS ?= linux
+
+binaries: cmds
+ifneq ($(PREFIX),)
+cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
+endif
+cmds: $(CMD_TARGETS)
+$(CMD_TARGETS): cmd-%:
+	GOOS=$(GOOS) go build -ldflags "-s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+
+build:
+	GOOS=$(GOOS) go build ./...
+
+examples: $(EXAMPLE_TARGETS)
+$(EXAMPLE_TARGETS): example-%:
+	GOOS=$(GOOS) go build ./examples/$(*)
+
+all: check test build binary
+check: $(CHECK_TARGETS)
+
+# Apply go fmt to the codebase
+fmt:
+	go list -f '{{.Dir}}' $(MODULE)/... \
+		| xargs gofmt -s -l -w
+
+assert-fmt:
+	go list -f '{{.Dir}}' $(MODULE)/... \
+		| xargs gofmt -s -l > fmt.out
+	@if [ -s fmt.out ]; then \
+		echo "\nERROR: The following files are not formatted:\n"; \
+		cat fmt.out; \
+		rm fmt.out; \
+		exit 1; \
+	else \
+		rm fmt.out; \
+	fi
+
+ineffassign:
+	ineffassign $(MODULE)/...
+
+lint:
+# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
+	go list -f '{{.Dir}}' $(MODULE)/... | xargs golint -set_exit_status
+
+misspell:
+	misspell $(MODULE)/...
+
+vet:
+	go vet $(MODULE)/...
+
+licenses:
+	go-licenses csv $(MODULE)/...
+
+COVERAGE_FILE := coverage.out
+test: build cmds
+	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...
+
+coverage: test
+	cat $(COVERAGE_FILE) | grep -v "_mock.go" > $(COVERAGE_FILE).no-mocks
+	go tool cover -func=$(COVERAGE_FILE).no-mocks
+
+generate:
+	go generate $(MODULE)/...
+
+# Generate an image for containerized builds
+# Note: This image is local only
+.PHONY: .build-image .pull-build-image .push-build-image
+.build-image: docker/Dockerfile.devel
+	if [ x"$(SKIP_IMAGE_BUILD)" = x"" ]; then \
+		$(DOCKER) build \
+			--progress=plain \
+			--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+			--tag $(BUILDIMAGE) \
+			-f $(^) \
+			docker; \
+	fi
+
+.pull-build-image:
+	$(DOCKER) pull $(BUILDIMAGE)
+
+.push-build-image:
+	$(DOCKER) push $(BUILDIMAGE)
+
+$(DOCKER_TARGETS): docker-%: .build-image
+	@echo "Running 'make $(*)' in docker container $(BUILDIMAGE)"
+	$(DOCKER) run \
+		--rm \
+		-e GOCACHE=/tmp/.cache \
+		-v $(PWD):$(PWD) \
+		-w $(PWD) \
+		--user $$(id -u):$$(id -g) \
+		$(BUILDIMAGE) \
+			make $(*)
+
+# Start an interactive shell using the development image.
+PHONY: .shell
+.shell:
+	$(DOCKER) run \
+		--rm \
+		-ti \
+		-e GOCACHE=/tmp/.cache \
+		-v $(PWD):$(PWD) \
+		-w $(PWD) \
+		--user $$(id -u):$$(id -g) \
+		$(BUILDIMAGE)

README.md

@@ -0,0 +1,31 @@
+# NVIDIA Container Toolkit
+
+[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/main/LICENSE)
+[![Documentation](https://img.shields.io/badge/documentation-wiki-blue.svg?style=flat-square)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html)
+[![Package repository](https://img.shields.io/badge/packages-repository-b956e8.svg?style=flat-square)](https://nvidia.github.io/libnvidia-container)
+
+![nvidia-container-stack](https://cloud.githubusercontent.com/assets/3028125/12213714/5b208976-b632-11e5-8406-38d379ec46aa.png)
+
+## Introduction
+
+The NVIDIA Container Toolkit allows users to build and run GPU accelerated containers. The toolkit includes a container runtime [library](https://github.com/NVIDIA/libnvidia-container) and utilities to automatically configure containers to leverage NVIDIA GPUs.
+
+Product documentation including an architecture overview, platform support, and installation and usage guides can be found in the [documentation repository](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html).
+
+## Getting Started
+
+**Make sure you have installed the [NVIDIA driver](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#nvidia-drivers) for your Linux Distribution**
+**Note that you do not need to install the CUDA Toolkit on the host system, but the NVIDIA driver needs to be installed**
+
+For instructions on getting started with the NVIDIA Container Toolkit, refer to the [installation guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#installation-guide).
+
+## Usage
+
+The [user guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/user-guide.html) provides information on the configuration and command line options available when running GPU containers with Docker.
+
+## Issues and Contributing
+
+[Checkout the Contributing document!](CONTRIBUTING.md)
+
+* Please let us know by [filing a new issue](https://github.com/NVIDIA/nvidia-container-toolkit/issues/new)
+* You can contribute by creating a [merge request](https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/merge_requests/new) to our public GitLab repository

build/container/Dockerfile.centos

@@ -0,0 +1,97 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ARG BASE_DIST
+# See https://www.centos.org/centos-linux-eol/
+# and https://stackoverflow.com/a/70930049 for move to vault.centos.org
+# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud
+RUN [[ "${BASE_DIST}" != "centos8" ]] || \
+    ( \
+      sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
+      sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \
+    )
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+ARG ARTIFACTS_ROOT
+ARG PACKAGE_DIST
+COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
+
+WORKDIR /artifacts/packages
+
+ARG PACKAGE_VERSION
+ARG TARGETARCH
+ENV PACKAGE_ARCH ${TARGETARCH}
+RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
+    yum localinstall -y \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+# Install / upgrade packages here that are required to resolve CVEs
+ARG CVE_UPDATES
+RUN if [ -n "${CVE_UPDATES}" ]; then \
+        yum update -y ${CVE_UPDATES} && \
+        rm -rf /var/cache/yum/*; \
+    fi
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.packaging

@@ -0,0 +1,37 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ARG VERSION="N/A"
+
+ARG ARTIFACTS_ROOT
+COPY ${ARTIFACTS_ROOT} /artifacts/packages/
+
+WORKDIR /artifacts/packages
+
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG SOURCE_DATE_EPOCH
+
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN echo "IMAGE_EPOCH=$(date '+%s')" | sed 's/^/#/g' > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

build/container/Dockerfile.ubuntu

@@ -0,0 +1,105 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+# Remove the CUDA repository configurations to avoid issues with rotated GPG keys
+RUN rm -f /etc/apt/sources.list.d/cuda.list
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libcap2 \
+    curl \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+ARG ARTIFACTS_ROOT
+ARG PACKAGE_DIST
+COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
+
+WORKDIR /artifacts/packages
+
+ARG PACKAGE_VERSION
+ARG TARGETARCH
+ENV PACKAGE_ARCH ${TARGETARCH}
+
+ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
+ARG LIBNVIDIA_CONTAINER0_VERSION
+RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
+        curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
+            --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \
+        dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \
+    fi
+
+RUN dpkg -i \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work/
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+# Install / upgrade packages here that are required to resolve CVEs
+ARG CVE_UPDATES
+RUN if [ -n "${CVE_UPDATES}" ]; then \
+        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -0,0 +1,155 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BUILD_MULTI_ARCH_IMAGES ?= false
+DOCKER ?= docker
+
+BUILDX =
+ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
+BUILDX = buildx
+endif
+
+MKDIR    ?= mkdir
+DIST_DIR ?= $(CURDIR)/dist
+
+##### Global variables #####
+include $(CURDIR)/versions.mk
+
+ifeq ($(IMAGE_NAME),)
+REGISTRY ?= nvidia
+IMAGE_NAME := $(REGISTRY)/container-toolkit
+endif
+
+VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+IMAGE_VERSION := $(VERSION)
+
+IMAGE_TAG ?= $(VERSION)-$(DIST)
+IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
+
+OUT_IMAGE_NAME ?= $(IMAGE_NAME)
+OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
+OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
+OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
+
+##### Public rules #####
+DEFAULT_PUSH_TARGET := ubuntu20.04
+DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7
+
+META_TARGETS := packaging
+
+BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
+PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
+TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+
+.PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
+
+ifneq ($(BUILD_MULTI_ARCH_IMAGES),true)
+include $(CURDIR)/build/container/native-only.mk
+else
+include $(CURDIR)/build/container/multi-arch.mk
+endif
+
+# For the default push target we also push a short tag equal to the version.
+# We skip this for the development release
+DEVEL_RELEASE_IMAGE_VERSION ?= devel
+PUSH_MULTIPLE_TAGS ?= true
+ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
+PUSH_MULTIPLE_TAGS = false
+endif
+ifeq ($(PUSH_MULTIPLE_TAGS),true)
+push-$(DEFAULT_PUSH_TARGET): push-short
+endif
+
+push-%: DIST = $(*)
+push-short: DIST = $(DEFAULT_PUSH_TARGET)
+
+build-%: DIST = $(*)
+build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
+
+ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+
+# Use a generic build target to build the relevant images
+$(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
+	DOCKER_BUILDKIT=1 \
+		$(DOCKER) $(BUILDX) build --pull \
+		$(DOCKER_BUILD_OPTIONS) \
+		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
+		--tag $(IMAGE) \
+		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
+		--build-arg BASE_DIST="$(BASE_DIST)" \
+		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
+		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+		--build-arg LIBNVIDIA_CONTAINER0_VERSION="$(LIBNVIDIA_CONTAINER0_DEPENDENCY)" \
+		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
+		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
+		--build-arg VERSION="$(VERSION)" \
+		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
+		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
+		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
+		-f $(DOCKERFILE) \
+		$(CURDIR)
+
+
+build-ubuntu%: BASE_DIST = $(*)
+build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
+build-ubuntu%: PACKAGE_DIST = ubuntu18.04
+build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
+
+build-ubi8: BASE_DIST := ubi8
+build-ubi8: DOCKERFILE_SUFFIX := centos
+build-ubi8: PACKAGE_DIST = centos8
+build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+build-centos7: BASE_DIST = $(*)
+build-centos7: DOCKERFILE_SUFFIX := centos
+build-centos7: PACKAGE_DIST = $(BASE_DIST)
+build-centos7: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+build-packaging: BASE_DIST := ubuntu20.04
+build-packaging: DOCKERFILE_SUFFIX := packaging
+build-packaging: PACKAGE_ARCH := amd64
+build-packaging: PACKAGE_DIST = all
+build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+
+# Test targets
+test-%: DIST = $(*)
+
+TEST_CASES ?= toolkit docker crio containerd
+$(TEST_TARGETS): test-%:
+	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \
+		$(CURDIR)/shared-$(*) \
+		$(IMAGE) \
+			--no-cleanup-on-error
+
+.PHONY: test-packaging
+test-packaging: DIST = packaging
+test-packaging:
+	@echo "Testing package image contents"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/README.md

@@ -0,0 +1,4 @@
+# NVIDIA Container Toolkit Container
+
+This folder contains make and docker files for building the NVIDIA Container Toolkit Container.
+

build/container/multi-arch.mk

@@ -0,0 +1,37 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PUSH_ON_BUILD ?= false
+DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD)
+DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64
+
+REGCTL ?= regctl
+$(PUSH_TARGETS): push-%:
+	$(REGCTL) \
+	        image copy \
+	        $(IMAGE) $(OUT_IMAGE)
+
+push-short:
+	$(REGCTL) \
+	        image copy \
+	        $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)
+
+# We only have x86_64 packages for centos7
+build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
+# We only generate amd64 image for ubuntu18.04
+build-ubuntu18.04: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
+# We only generate a single image for packaging targets
+build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64

build/container/native-only.mk

@@ -0,0 +1,23 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
+$(PUSH_TARGETS): push-%:
+	$(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)"
+	$(DOCKER) push "$(OUT_IMAGE)"
+
+push-short:
+	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"
+	$(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"

cmd/nvidia-container-runtime-hook/capabilities.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"log"
+	"strings"
+)
+
+const (
+	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
+	defaultDriverCapabilities = DriverCapabilities("utility,compute")
+
+	none = DriverCapabilities("")
+	all  = DriverCapabilities("all")
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}
+
+// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
+// variable. Operations include default values, filtering, and handling meta values such as "all"
+type DriverCapabilities string
+
+// Intersection returns intersection between two sets of capabilities.
+func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
+	if capabilities == all {
+		return d
+	}
+	if d == all {
+		return capabilities
+	}
+
+	lookup := make(map[string]bool)
+	for _, c := range d.list() {
+		lookup[c] = true
+	}
+	var found []string
+	for _, c := range capabilities.list() {
+		if lookup[c] {
+			found = append(found, c)
+		}
+	}
+
+	intersection := DriverCapabilities(strings.Join(found, ","))
+	return intersection
+}
+
+// String returns the string representation of the driver capabilities
+func (d DriverCapabilities) String() string {
+	return string(d)
+}
+
+// list returns the driver capabilities as a list
+func (d DriverCapabilities) list() []string {
+	var caps []string
+	for _, c := range strings.Split(string(d), ",") {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) == 0 {
+			continue
+		}
+		caps = append(caps, trimmed)
+	}
+
+	return caps
+}

cmd/nvidia-container-runtime-hook/capabilities_test.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDriverCapabilitiesIntersection(t *testing.T) {
+	testCases := []struct {
+		capabilities          DriverCapabilities
+		supportedCapabilities DriverCapabilities
+		expectedIntersection  DriverCapabilities
+	}{
+		{
+			capabilities:          none,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          allDriverCapabilities,
+			supportedCapabilities: all,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: all,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: DriverCapabilities("cap1"),
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("cap0,cap1"),
+			supportedCapabilities: DriverCapabilities("cap1,cap0"),
+			expectedIntersection:  DriverCapabilities("cap0,cap1"),
+		},
+		{
+			capabilities:          defaultDriverCapabilities,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  defaultDriverCapabilities,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+		{
+			capabilities:          DriverCapabilities("cap1"),
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
+			require.EqualValues(t, tc.expectedIntersection, intersection)
+		})
+	}
+}
+
+func TestDriverCapabilitiesList(t *testing.T) {
+	testCases := []struct {
+		capabilities DriverCapabilities
+		expected     []string
+	}{
+		{
+			capabilities: DriverCapabilities(""),
+		},
+		{
+			capabilities: DriverCapabilities("  "),
+		},
+		{
+			capabilities: DriverCapabilities(","),
+		},
+		{
+			capabilities: DriverCapabilities(",cap"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap,"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap0,,cap1"),
+			expected:     []string{"cap0", "cap1"},
+		},
+		{
+			capabilities: DriverCapabilities("cap1,cap0,cap3"),
+			expected:     []string{"cap1", "cap0", "cap3"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			require.EqualValues(t, tc.expected, tc.capabilities.list())
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/container_config.go

@@ -6,14 +6,13 @@ import (
 	"log"
 	"os"
 	"path"
-	"strconv"
+	"path/filepath"
 	"strings"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"golang.org/x/mod/semver"
 )
 
-var envSwarmGPU *string
-
 const (
 	envCUDAVersion          = "CUDA_VERSION"
 	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
@@ -26,12 +25,11 @@ const (
 )
 
 const (
-	allDriverCapabilities     = "compute,compat32,graphics,utility,video,display"
-	defaultDriverCapabilities = "utility"
+	capSysAdmin = "CAP_SYS_ADMIN"
 )
 
 const (
-	capSysAdmin = "CAP_SYS_ADMIN"
+	deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
 )
 
 type nvidiaConfig struct {
@@ -50,17 +48,20 @@ type containerConfig struct {
 	Nvidia *nvidiaConfig
 }
 
+// Root from OCI runtime spec
 // github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L94-L100
 type Root struct {
 	Path string `json:"path"`
 }
 
+// Process from OCI runtime spec
 // github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L57
 type Process struct {
 	Env          []string         `json:"env,omitempty"`
 	Capabilities *json.RawMessage `json:"capabilities,omitempty" platform:"linux"`
 }
 
+// LinuxCapabilities from OCI runtime spec
 // https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L61
 type LinuxCapabilities struct {
 	Bounding    []string `json:"bounding,omitempty" platform:"linux"`
@@ -70,14 +71,26 @@ type LinuxCapabilities struct {
 	Ambient     []string `json:"ambient,omitempty" platform:"linux"`
 }
 
+// Mount from OCI runtime spec
+// https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L103
+type Mount struct {
+	Destination string   `json:"destination"`
+	Type        string   `json:"type,omitempty" platform:"linux,solaris"`
+	Source      string   `json:"source,omitempty"`
+	Options     []string `json:"options,omitempty"`
+}
+
+// Spec from OCI runtime spec
 // We use pointers to structs, similarly to the latest version of runtime-spec:
 // https://github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L5-L28
 type Spec struct {
 	Version *string  `json:"ociVersion"`
 	Process *Process `json:"process,omitempty"`
 	Root    *Root    `json:"root,omitempty"`
+	Mounts  []Mount  `json:"mounts,omitempty"`
 }
 
+// HookState holds state information about the hook
 type HookState struct {
 	Pid int `json:"pid,omitempty"`
 	// After 17.06, runc is using the runtime spec:
@@ -89,43 +102,6 @@ type HookState struct {
 	BundlePath string `json:"bundlePath"`
 }
 
-func parseCudaVersion(cudaVersion string) (vmaj, vmin, vpatch uint32) {
-	if _, err := fmt.Sscanf(cudaVersion, "%d.%d.%d\n", &vmaj, &vmin, &vpatch); err != nil {
-		vpatch = 0
-		if _, err := fmt.Sscanf(cudaVersion, "%d.%d\n", &vmaj, &vmin); err != nil {
-			vmin = 0
-			if _, err := fmt.Sscanf(cudaVersion, "%d\n", &vmaj); err != nil {
-				log.Panicln("invalid CUDA version:", cudaVersion)
-			}
-		}
-	}
-
-	return
-}
-
-func getEnvMap(e []string, config CLIConfig) (m map[string]string) {
-	m = make(map[string]string)
-	for _, s := range e {
-		p := strings.SplitN(s, "=", 2)
-		if len(p) != 2 {
-			log.Panicln("environment error")
-		}
-		m[p[0]] = p[1]
-	}
-	if config.AlphaMergeVisibleDevicesEnvvars {
-		var mergable []string
-		for k, v := range m {
-			if strings.HasPrefix(k, envNVVisibleDevices+"_") {
-				mergable = append(mergable, v)
-			}
-		}
-		if len(mergable) > 0 {
-			m[envNVVisibleDevices] = strings.Join(mergable, ",")
-		}
-	}
-	return
-}
-
 func loadSpec(path string) (spec *Spec) {
 	f, err := os.Open(path)
 	if err != nil {
@@ -174,7 +150,7 @@ func isPrivileged(s *Spec) bool {
 		// We only make sure that the bounding capabibility set has
 		// CAP_SYS_ADMIN. This allows us to make sure that the container was
 		// actually started as '--privileged', but also allow non-root users to
-		// access the priviliged NVIDIA capabilities.
+		// access the privileged NVIDIA capabilities.
 		caps = lc.Bounding
 	}
 
@@ -187,78 +163,145 @@ func isPrivileged(s *Spec) bool {
 	return false
 }
 
-func getDevices(env map[string]string) *string {
-	gpuVars := []string{envNVVisibleDevices}
-	if envSwarmGPU != nil {
-		// The Swarm resource has higher precedence.
-		gpuVars = append([]string{*envSwarmGPU}, gpuVars...)
-	}
-
-	for _, gpuVar := range gpuVars {
-		if devices, ok := env[gpuVar]; ok {
-			return &devices
+func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
+	// We check if the image has at least one of the Swarm resource envvars defined and use this
+	// if specified.
+	var hasSwarmEnvvar bool
+	for _, envvar := range swarmResourceEnvvars {
+		if _, exists := image[envvar]; exists {
+			hasSwarmEnvvar = true
+			break
 		}
 	}
+
+	var devices []string
+	if hasSwarmEnvvar {
+		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
+	} else {
+		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
+	}
+
+	if len(devices) == 0 {
+		return nil
+	}
+
+	devicesString := strings.Join(devices, ",")
+
+	return &devicesString
+}
+
+func getDevicesFromMounts(mounts []Mount) *string {
+	var devices []string
+	for _, m := range mounts {
+		root := filepath.Clean(deviceListAsVolumeMountsRoot)
+		source := filepath.Clean(m.Source)
+		destination := filepath.Clean(m.Destination)
+
+		// Only consider mounts who's host volume is /dev/null
+		if source != "/dev/null" {
+			continue
+		}
+		// Only consider container mount points that begin with 'root'
+		if len(destination) < len(root) {
+			continue
+		}
+		if destination[:len(root)] != root {
+			continue
+		}
+		// Grab the full path beyond 'root' and add it to the list of devices
+		device := destination[len(root):]
+		if len(device) > 0 && device[0] == '/' {
+			device = device[1:]
+		}
+		if len(device) == 0 {
+			continue
+		}
+		devices = append(devices, device)
+	}
+
+	if devices == nil {
+		return nil
+	}
+
+	ret := strings.Join(devices, ",")
+	return &ret
+}
+
+func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
+	// If enabled, try and get the device list from volume mounts first
+	if hookConfig.AcceptDeviceListAsVolumeMounts {
+		devices := getDevicesFromMounts(mounts)
+		if devices != nil {
+			return devices
+		}
+	}
+
+	// Fallback to reading from the environment variable if privileges are correct
+	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
+	if devices == nil {
+		return nil
+	}
+	if privileged || hookConfig.AcceptEnvvarUnprivileged {
+		return devices
+	}
+
+	configName := hookConfig.getConfigOption("AcceptEnvvarUnprivileged")
+	log.Printf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES (privileged=%v, %v=%v) ", privileged, configName, hookConfig.AcceptEnvvarUnprivileged)
+
 	return nil
 }
 
 func getMigConfigDevices(env map[string]string) *string {
-	gpuVars := []string{envNVMigConfigDevices}
-	for _, gpuVar := range gpuVars {
-		if devices, ok := env[gpuVar]; ok {
-			return &devices
-		}
+	if devices, ok := env[envNVMigConfigDevices]; ok {
+		return &devices
 	}
 	return nil
 }
 
 func getMigMonitorDevices(env map[string]string) *string {
-	gpuVars := []string{envNVMigMonitorDevices}
-	for _, gpuVar := range gpuVars {
-		if devices, ok := env[gpuVar]; ok {
-			return &devices
-		}
+	if devices, ok := env[envNVMigMonitorDevices]; ok {
+		return &devices
 	}
 	return nil
 }
 
-func getDriverCapabilities(env map[string]string) *string {
-	if capabilities, ok := env[envNVDriverCapabilities]; ok {
-		return &capabilities
-	}
-	return nil
-}
+func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
+	// We use the default driver capabilities by default. This is filtered to only include the
+	// supported capabilities
+	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
 
-func getRequirements(env map[string]string) []string {
-	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
-	var requirements []string
-	for name, value := range env {
-		if strings.HasPrefix(name, envNVRequirePrefix) {
-			requirements = append(requirements, value)
+	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
+
+	if !capsEnvSpecified && legacyImage {
+		// Environment variable unset with legacy image: set all capabilities.
+		return supportedDriverCapabilities
+	}
+
+	if capsEnvSpecified && len(capsEnv) > 0 {
+		// If the envvironment variable is specified and is non-empty, use the capabilities value
+		envCapabilities := DriverCapabilities(capsEnv)
+		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
+		if envCapabilities != all && capabilities != envCapabilities {
+			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
 		}
 	}
-	return requirements
+
+	return capabilities
 }
 
-// Mimic the new CUDA images if no capabilities or devices are specified.
-func getNvidiaConfigLegacy(env map[string]string, privileged bool) *nvidiaConfig {
+func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
+	legacyImage := image.IsLegacy()
+
 	var devices string
-	if d := getDevices(env); d == nil {
-		// Environment variable unset: default to "all".
-		devices = "all"
-	} else if len(*d) == 0 || *d == "void" {
-		// Environment variable empty or "void": not a GPU container.
-		return nil
-	} else {
-		// Environment variable non-empty and not "void".
+	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
 		devices = *d
-	}
-	if devices == "none" {
-		devices = ""
+	} else {
+		// 'nil' devices means this is not a GPU container.
+		return nil
 	}
 
 	var migConfigDevices string
-	if d := getMigConfigDevices(env); d != nil {
+	if d := getMigConfigDevices(image); d != nil {
 		migConfigDevices = *d
 	}
 	if !privileged && migConfigDevices != "" {
@@ -266,99 +309,21 @@ func getNvidiaConfigLegacy(env map[string]string, privileged bool) *nvidiaConfig
 	}
 
 	var migMonitorDevices string
-	if d := getMigMonitorDevices(env); d != nil {
+	if d := getMigMonitorDevices(image); d != nil {
 		migMonitorDevices = *d
 	}
 	if !privileged && migMonitorDevices != "" {
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	var driverCapabilities string
-	if c := getDriverCapabilities(env); c == nil {
-		// Environment variable unset: default to "all".
-		driverCapabilities = allDriverCapabilities
-	} else if len(*c) == 0 {
-		// Environment variable empty: use default capability.
-		driverCapabilities = defaultDriverCapabilities
-	} else {
-		// Environment variable non-empty.
-		driverCapabilities = *c
-	}
-	if driverCapabilities == "all" {
-		driverCapabilities = allDriverCapabilities
+	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+
+	requirements, err := image.GetRequirements()
+	if err != nil {
+		log.Panicln("failed to get requirements", err)
 	}
 
-	requirements := getRequirements(env)
-
-	vmaj, vmin, _ := parseCudaVersion(env[envCUDAVersion])
-	cudaRequire := fmt.Sprintf("cuda>=%d.%d", vmaj, vmin)
-	requirements = append(requirements, cudaRequire)
-
-	// Don't fail on invalid values.
-	disableRequire, _ := strconv.ParseBool(env[envNVDisableRequire])
-
-	return &nvidiaConfig{
-		Devices:            devices,
-		MigConfigDevices:   migConfigDevices,
-		MigMonitorDevices:  migMonitorDevices,
-		DriverCapabilities: driverCapabilities,
-		Requirements:       requirements,
-		DisableRequire:     disableRequire,
-	}
-}
-
-func getNvidiaConfig(env map[string]string, privileged bool) *nvidiaConfig {
-	legacyCudaVersion := env[envCUDAVersion]
-	cudaRequire := env[envNVRequireCUDA]
-	if len(legacyCudaVersion) > 0 && len(cudaRequire) == 0 {
-		// Legacy CUDA image detected.
-		return getNvidiaConfigLegacy(env, privileged)
-	}
-
-	var devices string
-	if d := getDevices(env); d == nil || len(*d) == 0 || *d == "void" {
-		// Environment variable unset or empty or "void": not a GPU container.
-		return nil
-	} else {
-		// Environment variable non-empty and not "void".
-		devices = *d
-	}
-	if devices == "none" {
-		devices = ""
-	}
-
-	var migConfigDevices string
-	if d := getMigConfigDevices(env); d != nil {
-		migConfigDevices = *d
-	}
-	if !privileged && migConfigDevices != "" {
-		log.Panicln("cannot set MIG_CONFIG_DEVICES in non privileged container")
-	}
-
-	var migMonitorDevices string
-	if d := getMigMonitorDevices(env); d != nil {
-		migMonitorDevices = *d
-	}
-	if !privileged && migMonitorDevices != "" {
-		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
-	}
-
-	var driverCapabilities string
-	if c := getDriverCapabilities(env); c == nil || len(*c) == 0 {
-		// Environment variable unset or set but empty: use default capability.
-		driverCapabilities = defaultDriverCapabilities
-	} else {
-		// Environment variable set and non-empty.
-		driverCapabilities = *c
-	}
-	if driverCapabilities == "all" {
-		driverCapabilities = allDriverCapabilities
-	}
-
-	requirements := getRequirements(env)
-
-	// Don't fail on invalid values.
-	disableRequire, _ := strconv.ParseBool(env[envNVDisableRequire])
+	disableRequire := image.HasDisableRequire()
 
 	return &nvidiaConfig{
 		Devices:            devices,
@@ -384,13 +349,16 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	env := getEnvMap(s.Process.Env, hook.NvidiaContainerCLI)
+	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
+	if err != nil {
+		log.Panicln(err)
+	}
+
 	privileged := isPrivileged(s)
-	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    env,
-		Nvidia: getNvidiaConfig(env, privileged),
+		Env:    image,
+		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/hook_config.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+)
+
+const (
+	configPath = "/etc/nvidia-container-runtime/config.toml"
+	driverPath = "/run/nvidia/driver"
+)
+
+var defaultPaths = [...]string{
+	path.Join(driverPath, configPath),
+	configPath,
+}
+
+// CLIConfig : options for nvidia-container-cli.
+type CLIConfig struct {
+	Root        *string  `toml:"root"`
+	Path        *string  `toml:"path"`
+	Environment []string `toml:"environment"`
+	Debug       *string  `toml:"debug"`
+	Ldcache     *string  `toml:"ldcache"`
+	LoadKmods   bool     `toml:"load-kmods"`
+	NoPivot     bool     `toml:"no-pivot"`
+	NoCgroups   bool     `toml:"no-cgroups"`
+	User        *string  `toml:"user"`
+	Ldconfig    *string  `toml:"ldconfig"`
+}
+
+// HookConfig : options for the nvidia-container-runtime-hook.
+type HookConfig struct {
+	DisableRequire                 bool               `toml:"disable-require"`
+	SwarmResource                  *string            `toml:"swarm-resource"`
+	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
+
+	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
+}
+
+func getDefaultHookConfig() HookConfig {
+	return HookConfig{
+		DisableRequire:                 false,
+		SwarmResource:                  nil,
+		AcceptEnvvarUnprivileged:       true,
+		AcceptDeviceListAsVolumeMounts: false,
+		SupportedDriverCapabilities:    allDriverCapabilities,
+		NvidiaContainerCLI: CLIConfig{
+			Root:        nil,
+			Path:        nil,
+			Environment: []string{},
+			Debug:       nil,
+			Ldcache:     nil,
+			LoadKmods:   true,
+			NoPivot:     false,
+			NoCgroups:   false,
+			User:        nil,
+			Ldconfig:    nil,
+		},
+		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
+	}
+}
+
+func getHookConfig() (config HookConfig) {
+	var err error
+
+	if len(*configflag) > 0 {
+		config = getDefaultHookConfig()
+		_, err = toml.DecodeFile(*configflag, &config)
+		if err != nil {
+			log.Panicln("couldn't open configuration file:", err)
+		}
+	} else {
+		for _, p := range defaultPaths {
+			config = getDefaultHookConfig()
+			_, err = toml.DecodeFile(p, &config)
+			if err == nil {
+				break
+			} else if !os.IsNotExist(err) {
+				log.Panicln("couldn't open default configuration file:", err)
+			}
+		}
+	}
+
+	if config.SupportedDriverCapabilities == all {
+		config.SupportedDriverCapabilities = allDriverCapabilities
+	}
+	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
+	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
+	}
+
+	return config
+}
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c HookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}
+
+// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
+func (c *HookConfig) getSwarmResourceEnvvars() []string {
+	if c.SwarmResource == nil {
+		return nil
+	}
+
+	candidates := strings.Split(*c.SwarmResource, ",")
+
+	var envvars []string
+	for _, c := range candidates {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) > 0 {
+			envvars = append(envvars, trimmed)
+		}
+	}
+
+	return envvars
+}

cmd/nvidia-container-runtime-hook/hook_config_test.go

@@ -0,0 +1,161 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHookConfig(t *testing.T) {
+	testCases := []struct {
+		lines                      []string
+		expectedPanic              bool
+		expectedDriverCapabilities DriverCapabilities
+	}{
+		{
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"all\"",
+			},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"compute,utility,not-compute\"",
+			},
+			expectedPanic: true,
+		},
+		{
+			lines:                      []string{},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"\"",
+			},
+			expectedDriverCapabilities: none,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"utility,compute\"",
+			},
+			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			var filename string
+			defer func() {
+				if len(filename) > 0 {
+					os.Remove(filename)
+				}
+				configflag = nil
+			}()
+
+			if tc.lines != nil {
+				configFile, err := os.CreateTemp("", "*.toml")
+				require.NoError(t, err)
+				defer configFile.Close()
+
+				filename = configFile.Name()
+				configflag = &filename
+
+				for _, line := range tc.lines {
+					_, err := configFile.WriteString(fmt.Sprintf("%s\n", line))
+					require.NoError(t, err)
+				}
+			}
+
+			var config HookConfig
+			getHookConfig := func() {
+				config = getHookConfig()
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getHookConfig)
+				return
+			}
+
+			getHookConfig()
+
+			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+		})
+	}
+}
+
+func TestGetSwarmResourceEnvvars(t *testing.T) {
+	testCases := []struct {
+		value    string
+		expected []string
+	}{
+		{
+			value:    "nil",
+			expected: nil,
+		},
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value:    " ",
+			expected: nil,
+		},
+		{
+			value:    "single",
+			expected: []string{"single"},
+		},
+		{
+			value:    "single ",
+			expected: []string{"single"},
+		},
+		{
+			value:    "one,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one ,two",
+			expected: []string{"one", "two"},
+		},
+		{
+			value:    "one, two",
+			expected: []string{"one", "two"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			c := &HookConfig{
+				SwarmResource: func() *string {
+					if tc.value == "nil" {
+						return nil
+					}
+					return &tc.value
+				}(),
+			}
+
+			envvars := c.getSwarmResourceEnvvars()
+			require.EqualValues(t, tc.expected, envvars)
+		})
+	}
+}

cmd/nvidia-container-runtime-hook/hook_test.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsPrivileged(t *testing.T) {
+	var tests = []struct {
+		spec     string
+		expected bool
+	}{
+		{
+			`
+			{
+				"ociVersion": "1.0.0",
+				"process": {
+					"capabilities": {
+						"bounding": [ "CAP_SYS_ADMIN" ]
+					}
+				}
+			}
+			`,
+			true,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0",
+				"process": {
+					"capabilities": {
+						"bounding": [ "CAP_SYS_OTHER" ]
+					}
+				}
+			}
+			`,
+			false,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0",
+				"process": {}
+			}
+			`,
+			false,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0-rc2-dev",
+				"process": {
+					"capabilities": [ "CAP_SYS_ADMIN" ]
+				}
+			}
+			`,
+			true,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0-rc2-dev",
+				"process": {
+					"capabilities": [ "CAP_SYS_OTHER" ]
+				}
+			}
+			`,
+			false,
+		},
+		{
+			`
+			{
+				"ociVersion": "1.0.0-rc2-dev",
+				"process": {}
+			}
+			`,
+			false,
+		},
+	}
+	for i, tc := range tests {
+		var spec Spec
+		_ = json.Unmarshal([]byte(tc.spec), &spec)
+		privileged := isPrivileged(&spec)
+
+		require.Equal(t, tc.expected, privileged, "%d: %v", i, tc)
+	}
+}

cmd/nvidia-container-runtime-hook/main.go

@@ -6,20 +6,21 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
 	"strings"
 	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 var (
-	debugflag  = flag.Bool("debug", false, "enable debug output")
-	configflag = flag.String("config", "", "configuration file")
-
-	defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
+	debugflag   = flag.Bool("debug", false, "enable debug output")
+	versionflag = flag.Bool("version", false, "enable version output")
+	configflag  = flag.String("config", "", "configuration file")
 )
 
 func exit() {
@@ -35,28 +36,16 @@ func exit() {
 	os.Exit(0)
 }
 
-func getPATH(config CLIConfig) string {
-	dirs := filepath.SplitList(os.Getenv("PATH"))
-	// directories from the hook environment have higher precedence
-	dirs = append(dirs, defaultPATH...)
-
-	if config.Root != nil {
-		rootDirs := []string{}
-		for _, dir := range dirs {
-			rootDirs = append(rootDirs, path.Join(*config.Root, dir))
-		}
-		// directories with the root prefix have higher precedence
-		dirs = append(rootDirs, dirs...)
-	}
-	return strings.Join(dirs, ":")
-}
-
 func getCLIPath(config CLIConfig) string {
 	if config.Path != nil {
 		return *config.Path
 	}
 
-	if err := os.Setenv("PATH", getPATH(config)); err != nil {
+	var root string
+	if config.Root != nil {
+		root = *config.Root
+	}
+	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -85,6 +74,10 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
+	if info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
+	}
+
 	container := getContainerConfig(hook)
 	nvidia := container.Nvidia
 	if nvidia == nil {
@@ -167,6 +160,11 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	if *versionflag {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime Hook", info.GetVersionString())
+		return
+	}
+
 	args := flag.Args()
 	if len(args) == 0 {
 		flag.Usage()
@@ -186,3 +184,12 @@ func main() {
 		os.Exit(2)
 	}
 }
+
+// logInterceptor implements the info.Logger interface to allow for logging from this function.
+type logInterceptor struct{}
+
+func (l *logInterceptor) Infof(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
+
+func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-container-runtime/README.md

@@ -0,0 +1,87 @@
+# The NVIDIA Container Runtime
+
+The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
+
+## Configuration
+
+The NVIDIA Container Runtime uses file-based configuration, with the config stored in `/etc/nvidia-container-runtime/config.toml`. The `/etc` path can be overridden using the `XDG_CONFIG_HOME` environment variable with the `${XDG_CONFIG_HOME}/nvidia-container-runtime/config.toml` file used instead if this environment variable is set.
+
+This config file may contain options for other components of the NVIDIA container stack and for the NVIDIA Container Runtime, the relevant config section is `nvidia-container-runtime`
+
+### Logging
+
+The `log-level` config option (default: `"info"`) specifies the log level to use and the `debug` option, if set, specifies a log file to which logs for the NVIDIA Container Runtime must be written.
+
+In addition to this, the NVIDIA Container Runtime considers the value of `--log` and `--log-format` flags that may be passed to it by a container runtime such as docker or containerd. If the `--debug` flag is present the log-level specified in the config file is overridden as `"debug"`.
+
+### Low-level Runtime Path
+
+The `runtimes` config option allows for the low-level runtime to be specified. The first entry in this list that is an existing executable file is used as the low-level runtime. If the entry is not a path, the `PATH` is searched for a matching executable. If the entry is a path this is checked instead.
+
+The default value for this setting is:
+```toml
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+```
+
+and if, for example, `crun` is to be used instead this can be changed to:
+```toml
+runtimes = [
+    "crun",
+]
+```
+
+### Runtime Mode
+
+The `mode` config option (default `"auto"`) controls the high-level behaviour of the runtime.
+
+#### Auto Mode
+
+When `mode` is set to `"auto"`, the runtime employs heuristics to determine which mode to use based on, for example, the platform where the runtime is being run.
+
+#### Legacy Mode
+
+When `mode` is set to `"legacy"`, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from the [libnvidia-container](https://github.com/NVIDIA/libnvidia-container) project.
+
+#### CSV Mode
+
+When `mode` is set to `"csv"`, CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` define the devices and mounts that are to be injected into a container when it is created. The search path for the files can be overridden by modifying the `nvidia-container-runtime.modes.csv.mount-spec-path` in the config as below:
+
+```toml
+[nvidia-container-runtime]
+    [nvidia-container-runtime.modes.csv]
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+```
+
+This mode is primarily targeted at Tegra-based systems without NVML available.
+
+### Notes on using the docker CLI
+
+Note that only the `"legacy"` NVIDIA Container Runtime mode is directly compatible with the `--gpus` flag implemented by the `docker` CLI (assuming the NVIDIA Container Runtime is not used). The reason for this is that `docker` inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification.
+
+
+If a different mode is explicitly set or detected, the NVIDIA Container Runtime Hook will raise the following error when `--gpus` is set:
+```
+$ docker run --rm --gpus all ubuntu:18.04
+docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'csv'
+invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.: unknown.
+```
+Here NVIDIA Container Runtime must be used explicitly. The recommended way to do this is to specify the `--runtime=nvidia` command line argument as part of the `docker run` commmand as follows:
+```
+$ docker run --rm --gpus all --runtime=nvidia ubuntu:18.04
+```
+
+Alternatively the NVIDIA Container Runtime can be set as the default runtime for docker. This can be done by modifying the `/etc/docker/daemon.json` file as follows:
+```json
+{
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}
+```

cmd/nvidia-container-runtime/logger.go

@@ -0,0 +1,239 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Logger adds a way to manage output to a log file to a logrus.Logger
+type Logger struct {
+	*logrus.Logger
+	previousLogger *logrus.Logger
+	logFiles       []*os.File
+}
+
+// NewLogger creates an empty logger
+func NewLogger() *Logger {
+	return &Logger{
+		Logger: logrus.New(),
+	}
+}
+
+// UpdateLogger constructs a Logger with a preddefined formatter
+func UpdateLogger(filename string, logLevel string, argv []string) (*Logger, error) {
+	configFromArgs := parseArgs(argv)
+
+	level, logLevelError := configFromArgs.getLevel(logLevel)
+
+	var logFiles []*os.File
+	var argLogFileError error
+
+	// We don't create log files if the version argument is supplied
+	if !configFromArgs.version {
+		configLogFile, err := createLogFile(filename)
+		if err != nil {
+			return logger, fmt.Errorf("error opening debug log file: %v", err)
+		}
+		if configLogFile != nil {
+			logFiles = append(logFiles, configLogFile)
+		}
+
+		argLogFile, err := createLogFile(configFromArgs.file)
+		if argLogFile != nil {
+			logFiles = append(logFiles, argLogFile)
+		}
+		argLogFileError = err
+	}
+
+	l := &Logger{
+		Logger:         logrus.New(),
+		previousLogger: logger.Logger,
+		logFiles:       logFiles,
+	}
+
+	l.SetLevel(level)
+	if level == logrus.DebugLevel {
+		logrus.SetReportCaller(true)
+		// Shorten function and file names reported by the logger, by
+		// trimming common "github.com/opencontainers/runc" prefix.
+		// This is only done for text formatter.
+		_, file, _, _ := runtime.Caller(0)
+		prefix := filepath.Dir(file) + "/"
+		logrus.SetFormatter(&logrus.TextFormatter{
+			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+				function := strings.TrimPrefix(f.Function, prefix) + "()"
+				fileLine := strings.TrimPrefix(f.File, prefix) + ":" + strconv.Itoa(f.Line)
+				return function, fileLine
+			},
+		})
+	}
+
+	if configFromArgs.format == "json" {
+		l.SetFormatter(new(logrus.JSONFormatter))
+	}
+
+	if len(logFiles) == 0 {
+		l.SetOutput(io.Discard)
+	} else if len(logFiles) == 1 {
+		l.SetOutput(logFiles[0])
+	} else if len(logFiles) > 1 {
+		var writers []io.Writer
+		for _, f := range logFiles {
+			writers = append(writers, f)
+		}
+		l.SetOutput(io.MultiWriter(writers...))
+	}
+
+	if logLevelError != nil {
+		l.Warn(logLevelError)
+	}
+
+	if argLogFileError != nil {
+		l.Warnf("Failed to open log file: %v", argLogFileError)
+	}
+
+	return l, nil
+}
+
+// Reset closes the log file (if any) and resets the logger output to what it
+// was before UpdateLogger was called.
+func (l *Logger) Reset() error {
+	defer func() {
+		previous := l.previousLogger
+		if previous == nil {
+			previous = logrus.New()
+		}
+		logger = &Logger{Logger: previous}
+	}()
+
+	var errs []error
+	for _, f := range l.logFiles {
+		err := f.Close()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	var err error
+	for _, e := range errs {
+		if err == nil {
+			err = e
+			continue
+		}
+		return fmt.Errorf("%v; %w", e, err)
+	}
+
+	return err
+}
+
+func createLogFile(filename string) (*os.File, error) {
+	if filename != "" && filename != os.DevNull {
+		return os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	}
+
+	return nil, nil
+}
+
+type loggerConfig struct {
+	file    string
+	format  string
+	debug   bool
+	version bool
+}
+
+func (c loggerConfig) getLevel(logLevel string) (logrus.Level, error) {
+	if c.debug {
+		return logrus.DebugLevel, nil
+	}
+
+	if logLevel, err := logrus.ParseLevel(logLevel); err == nil {
+		return logLevel, nil
+	}
+
+	return logrus.InfoLevel, fmt.Errorf("invalid log-level '%v'", logLevel)
+}
+
+// Informed by Taken from https://github.com/opencontainers/runc/blob/7fd8b57001f5bfa102e89cb434d96bf71f7c1d35/main.go#L182
+func parseArgs(args []string) loggerConfig {
+	c := loggerConfig{}
+
+	expected := map[string]*string{
+		"log-format": &c.format,
+		"log":        &c.file,
+	}
+
+	found := make(map[string]bool)
+
+	for i := 0; i < len(args); i++ {
+		if len(found) == 4 {
+			break
+		}
+
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		trimmed := strings.TrimLeft(parts[0], "-")
+		// If this is not a flag we continue
+		if parts[0] == trimmed {
+			continue
+		}
+
+		// Check the version flag
+		if trimmed == "version" {
+			c.version = true
+			found["version"] = true
+			// For the version flag we don't process any other flags
+			continue
+		}
+
+		// Check the debug flag
+		if trimmed == "debug" {
+			c.debug = true
+			found["debug"] = true
+			continue
+		}
+
+		destination, exists := expected[trimmed]
+		if !exists {
+			continue
+		}
+
+		var value string
+		if len(parts) == 2 {
+			value = parts[2]
+		} else if i+1 < len(args) {
+			value = args[i+1]
+			i++
+		} else {
+			continue
+		}
+
+		*destination = value
+		found[trimmed] = true
+	}
+
+	return c
+}

cmd/nvidia-container-runtime/main.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// version must be set by go build's -X main.version= option in the Makefile.
+var version = "unknown"
+
+// gitCommit will be the hash that the binary was built from
+// and will be populated by the Makefile
+var gitCommit = ""
+
+var logger = NewLogger()
+
+func main() {
+	err := run(os.Args)
+	if err != nil {
+		logger.Errorf("%v", err)
+		os.Exit(1)
+	}
+}
+
+// run is an entry point that allows for idiomatic handling of errors
+// when calling from the main function.
+func run(argv []string) (rerr error) {
+	printVersion := hasVersionFlag(argv)
+	if printVersion {
+		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
+	}
+
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return fmt.Errorf("error loading config: %v", err)
+	}
+
+	logger, err = UpdateLogger(
+		cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
+		cfg.NVIDIAContainerRuntimeConfig.LogLevel,
+		argv,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to set up logger: %v", err)
+	}
+	defer func() {
+		if rerr != nil {
+			logger.Errorf("%v", rerr)
+		}
+		logger.Reset()
+	}()
+
+	logger.Debugf("Command line arguments: %v", argv)
+	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
+	if err != nil {
+		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
+	}
+
+	if printVersion {
+		fmt.Print("\n")
+	}
+	return runtime.Exec(argv)
+}
+
+// TODO: This should be refactored / combined with parseArgs in logger.
+func hasVersionFlag(args []string) bool {
+	for i := 0; i < len(args); i++ {
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		trimmed := strings.TrimLeft(parts[0], "-")
+		// If this is not a flag we continue
+		if parts[0] == trimmed {
+			continue
+		}
+
+		// Check the version flag
+		if trimmed == "version" {
+			return true
+		}
+	}
+
+	return false
+}

cmd/nvidia-container-runtime/main_test.go

@@ -0,0 +1,246 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	nvidiaRuntime            = "nvidia-container-runtime"
+	nvidiaHook               = "nvidia-container-runtime-hook"
+	bundlePathSuffix         = "test/output/bundle/"
+	specFile                 = "config.json"
+	unmodifiedSpecFileSuffix = "test/input/test_spec.json"
+)
+
+const (
+	runcExecutableName = "runc"
+)
+
+type testConfig struct {
+	root    string
+	binPath string
+}
+
+var cfg *testConfig
+
+func TestMain(m *testing.M) {
+	// TEST SETUP
+	// Determine the module root and the test binary path
+	var err error
+	moduleRoot, err := test.GetModuleRoot()
+	if err != nil {
+		logger.Fatalf("error in test setup: could not get module root: %v", err)
+	}
+	testBinPath := filepath.Join(moduleRoot, "test", "bin")
+	testInputPath := filepath.Join(moduleRoot, "test", "input")
+
+	// Set the environment variables for the test
+	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
+	os.Setenv("XDG_CONFIG_HOME", testInputPath)
+
+	// Confirm that the environment is configured correctly
+	runcPath, err := exec.LookPath(runcExecutableName)
+	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
+		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+	}
+	hookPath, err := exec.LookPath(nvidiaHook)
+	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
+		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+	}
+
+	// Store the root and binary paths in the test Config
+	cfg = &testConfig{
+		root:    moduleRoot,
+		binPath: testBinPath,
+	}
+
+	// RUN TESTS
+	exitCode := m.Run()
+
+	// TEST CLEANUP
+	os.Remove(specFile)
+
+	os.Exit(exitCode)
+}
+
+// case 1) nvidia-container-runtime run --bundle
+// case 2) nvidia-container-runtime create --bundle
+//		- Confirm the runtime handles bad input correctly
+func TestBadInput(t *testing.T) {
+	err := cfg.generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.Error(t, err, "runtime should return an error")
+}
+
+// case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
+//		- Confirm the runtime runs with no errors
+// case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
+//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+func TestGoodInput(t *testing.T) {
+	err := cfg.generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatalf("error generating runtime spec: %v", err)
+	}
+
+	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle", cfg.bundlePath(), "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
+	output, err := cmdRun.CombinedOutput()
+	require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
+
+	// Check config.json and confirm there are no hooks
+	spec, err := cfg.getRuntimeSpec()
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
+
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.NoError(t, err, "runtime should not return an error")
+
+	// Check config.json for NVIDIA prestart hook
+	spec, err = cfg.getRuntimeSpec()
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
+	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+}
+
+// NVIDIA prestart hook already present in config file
+func TestDuplicateHook(t *testing.T) {
+	err := cfg.generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var spec specs.Spec
+	spec, err = cfg.getRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("inserting nvidia prestart hook to config.json")
+	if err = addNVIDIAHook(&spec); err != nil {
+		t.Fatal(err)
+	}
+
+	jsonOutput, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jsonFile, err := os.OpenFile(cfg.specFilePath(), os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = jsonFile.WriteAt(jsonOutput, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test how runtime handles already existing prestart hook in config.json
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	output, err := cmdCreate.CombinedOutput()
+	require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
+
+	// Check config.json for NVIDIA prestart hook
+	spec, err = cfg.getRuntimeSpec()
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
+	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+}
+
+// addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
+// testing.
+func addNVIDIAHook(spec *specs.Spec) error {
+	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	return m.Modify(spec)
+}
+
+func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
+	filePath := c.specFilePath()
+
+	var spec specs.Spec
+	jsonFile, err := os.OpenFile(filePath, os.O_RDWR, 0644)
+	if err != nil {
+		return spec, err
+	}
+	defer jsonFile.Close()
+
+	jsonContent, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return spec, err
+	} else if json.Valid(jsonContent) {
+		err = json.Unmarshal(jsonContent, &spec)
+		if err != nil {
+			return spec, err
+		}
+	} else {
+		err = json.NewDecoder(bytes.NewReader(jsonContent)).Decode(&spec)
+		if err != nil {
+			return spec, err
+		}
+	}
+
+	return spec, err
+}
+
+func (c testConfig) bundlePath() string {
+	return filepath.Join(c.root, bundlePathSuffix)
+}
+
+func (c testConfig) specFilePath() string {
+	return filepath.Join(c.bundlePath(), specFile)
+}
+
+func (c testConfig) unmodifiedSpecFile() string {
+	return filepath.Join(c.root, unmodifiedSpecFileSuffix)
+}
+
+func (c testConfig) generateNewRuntimeSpec() error {
+	var err error
+
+	err = os.MkdirAll(c.bundlePath(), 0755)
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command("cp", c.unmodifiedSpecFile(), c.specFilePath())
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Return number of valid NVIDIA prestart hooks in runtime spec
+func nvidiaHookCount(hooks *specs.Hooks) int {
+	if hooks == nil {
+		return 0
+	}
+
+	count := 0
+	for _, hook := range hooks.Prestart {
+		if strings.Contains(hook.Path, nvidiaHook) {
+			count++
+		}
+	}
+	return count
+}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -0,0 +1,111 @@
+/*
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+	"github.com/sirupsen/logrus"
+)
+
+// newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger
+func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) {
+	lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, cfg.NVIDIAContainerRuntimeConfig.Runtimes)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
+	}
+
+	if !oci.HasCreateSubcommand(argv) {
+		logger.Debugf("Skipping modifier for non-create subcommand")
+		return lowLevelRuntime, nil
+	}
+
+	ociSpec, err := oci.NewSpec(logger, argv)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
+	}
+
+	specModifier, err := newSpecModifier(logger, cfg, ociSpec, argv)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
+	}
+
+	// Create the wrapping runtime with the specified modifier
+	r := runtime.NewModifyingRuntimeWrapper(
+		logger,
+		lowLevelRuntime,
+		ociSpec,
+		specModifier,
+	)
+
+	return r, nil
+}
+
+// newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
+func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
+	modeModifier, err := newModeModifier(logger, cfg, ociSpec, argv)
+	if err != nil {
+		return nil, err
+	}
+
+	graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	gdsModifier, err := modifier.NewGDSModifier(logger, cfg, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	mofedModifier, err := modifier.NewMOFEDModifier(logger, cfg, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	tegraModifier, err := modifier.NewTegraPlatformFiles(logger)
+	if err != nil {
+		return nil, err
+	}
+
+	modifiers := modifier.Merge(
+		modeModifier,
+		graphicsModifier,
+		gdsModifier,
+		mofedModifier,
+		tegraModifier,
+	)
+	return modifiers, nil
+}
+
+func newModeModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) {
+	switch info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode) {
+	case "legacy":
+		return modifier.NewStableRuntimeModifier(logger), nil
+	case "csv":
+		return modifier.NewCSVModifier(logger, cfg, ociSpec)
+	case "cdi":
+		return modifier.NewCDIModifier(logger, cfg, ociSpec)
+	}
+
+	return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)
+}

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -0,0 +1,129 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFactoryMethod(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		cfg           *config.Config
+		spec          *specs.Spec
+		expectedError bool
+	}{
+		{
+			description: "empty config raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
+			},
+			expectedError: true,
+		},
+		{
+			description: "config with runtime raises no error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "legacy",
+				},
+			},
+		},
+		{
+			description: "csv mode is supported",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "csv",
+				},
+			},
+			spec: &specs.Spec{
+				Process: &specs.Process{
+					Env: []string{
+						"NVIDIA_VISIBLE_DEVICES=all",
+					},
+				},
+			},
+		},
+		{
+			description: "non-legacy discover mode raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "non-legacy",
+				},
+			},
+			expectedError: true,
+		},
+		{
+			description: "legacy discover mode returns modifier",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "legacy",
+				},
+			},
+		},
+		{
+			description: "csv discover mode returns modifier",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+					Mode:     "csv",
+				},
+			},
+		},
+		{
+			description: "empty mode raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Runtimes: []string{"runc"},
+				},
+			},
+			expectedError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			bundleDir := t.TempDir()
+
+			specFile, err := os.Create(filepath.Join(bundleDir, "config.json"))
+			require.NoError(t, err)
+			require.NoError(t, json.NewEncoder(specFile).Encode(tc.spec))
+
+			argv := []string{"--bundle", bundleDir, "create"}
+
+			_, err = newNVIDIAContainerRuntime(logger, tc.cfg, argv)
+			if tc.expectedError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

cmd/nvidia-ctk/README.md

@@ -0,0 +1,48 @@
+# NVIDIA Container Toolkit CLI
+
+The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
+
+## Functionality
+
+### Configure runtimes
+
+The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
+and management of supported container engines.
+
+For example, running the following command:
+```bash
+nvidia-ctk runtime configure --set-as-default
+```
+will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
+engine.
+
+### Generate CDI specifications
+
+The [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface) provides
+a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
+used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
+available NVIDIA GPUs in a system.
+
+In order to generate the CDI specification for the available devices, run the following command:\
+```bash
+nvidia-ctk cdi generate
+```
+
+The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
+
+The specification will contain a device entries as follows (where applicable):
+* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
+* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
+* A special device called `nvidia.com/gpu=all` which represents all available devices.
+
+For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
+
+With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
+```bash
+podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
+```

cmd/nvidia-ctk/cdi/cdi.go

@@ -0,0 +1,50 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "cdi",
+		Usage: "Provide tools for interacting with Container Device Interface specifications",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		generate.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/cdi/generate/common.go

@@ -0,0 +1,63 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// NewCommonDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
+// This includes driver libraries and meta devices, for example.
+func NewCommonDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
+	metaDevices := discover.NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{
+			"/dev/nvidia-modeset",
+			"/dev/nvidia-uvm-tools",
+			"/dev/nvidia-uvm",
+			"/dev/nvidiactl",
+		},
+	)
+
+	graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing discoverer for graphics mounts: %v", err)
+	}
+
+	driverFiles, err := NewDriverDiscoverer(logger, root, nvidiaCTKPath, nvmllib)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err)
+	}
+
+	d := discover.Merge(
+		metaDevices,
+		graphicsMounts,
+		driverFiles,
+	)
+
+	return d, nil
+}

cmd/nvidia-ctk/cdi/generate/device-folder-permissions.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+type deviceFolderPermissions struct {
+	logger        *logrus.Logger
+	root          string
+	nvidiaCTKPath string
+	folders       []string
+}
+
+var _ discover.Discover = (*deviceFolderPermissions)(nil)
+
+// NewDeviceFolderPermissionHookDiscoverer creates a discoverer that can be used to update the permissions for the parent folders of nested device nodes from the specified set of device specs.
+// This works around an issue with rootless podman when using crun as a low-level runtime.
+// See https://github.com/containers/crun/issues/1047
+// The nested devices that are applicable to the NVIDIA GPU devices are:
+//   - DRM devices at /dev/dri/*
+//   - NVIDIA Caps devices at /dev/nvidia-caps/*
+func NewDeviceFolderPermissionHookDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, deviceSpecs []specs.Device) (discover.Discover, error) {
+	var folders []string
+	seen := make(map[string]bool)
+	for _, device := range deviceSpecs {
+		for _, dn := range device.ContainerEdits.DeviceNodes {
+			df := filepath.Dir(dn.Path)
+			if seen[df] {
+				continue
+			}
+			// We only consider the special case paths
+			if df != "/dev/dri" && df != "/dev/nvidia-caps" {
+				continue
+			}
+			folders = append(folders, df)
+			seen[df] = true
+		}
+		if len(folders) == 2 {
+			break
+		}
+	}
+
+	if len(folders) == 0 {
+		return discover.None{}, nil
+	}
+
+	d := &deviceFolderPermissions{
+		logger:        logger,
+		root:          root,
+		nvidiaCTKPath: nvidiaCTKPath,
+		folders:       folders,
+	}
+
+	return d, nil
+}
+
+// Devices are empty for this discoverer
+func (d *deviceFolderPermissions) Devices() ([]discover.Device, error) {
+	return nil, nil
+}
+
+// Hooks returns a set of hooks that sets the file mode to 755 of parent folders for nested device nodes.
+func (d *deviceFolderPermissions) Hooks() ([]discover.Hook, error) {
+	if len(d.folders) == 0 {
+		return nil, nil
+	}
+
+	args := []string{"--mode", "755"}
+	for _, folder := range d.folders {
+		args = append(args, "--path", folder)
+	}
+
+	hook := discover.CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"chmod",
+		args...,
+	)
+
+	return []discover.Hook{hook}, nil
+}
+
+// Mounts are empty for this discoverer
+func (d *deviceFolderPermissions) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}

cmd/nvidia-ctk/cdi/generate/driver.go

@@ -0,0 +1,156 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
+// The supplied NVML Library is used to query the expected driver version.
+func NewDriverDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, nvmllib nvml.Interface) (discover.Discover, error) {
+	version, r := nvmllib.SystemGetDriverVersion()
+	if r != nvml.SUCCESS {
+		return nil, fmt.Errorf("failed to determine driver version: %v", r)
+	}
+
+	libraries, err := NewDriverLibraryDiscoverer(logger, root, nvidiaCTKPath, version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for driver libraries: %v", err)
+	}
+
+	firmwares := NewDriverFirmwareDiscoverer(logger, root, version)
+
+	binaries := NewDriverBinariesDiscoverer(logger, root)
+
+	d := discover.Merge(
+		libraries,
+		firmwares,
+		binaries,
+	)
+
+	return d, nil
+}
+
+// NewDriverLibraryDiscoverer creates a discoverer for the libraries associated with the specified driver version.
+func NewDriverLibraryDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, version string) (discover.Discover, error) {
+	libraryPaths, err := getVersionLibs(logger, root, version)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get libraries for driver version: %v", err)
+	}
+
+	libraries := discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		libraryPaths,
+	)
+
+	cfg := &discover.Config{
+		Root:          root,
+		NvidiaCTKPath: nvidiaCTKPath,
+	}
+	hooks, _ := discover.NewLDCacheUpdateHook(logger, libraries, cfg)
+
+	d := discover.Merge(
+		libraries,
+		hooks,
+	)
+
+	return d, nil
+}
+
+// NewDriverFirmwareDiscoverer creates a discoverer for GSP firmware associated with the specified driver version.
+func NewDriverFirmwareDiscoverer(logger *logrus.Logger, root string, version string) discover.Discover {
+	gspFirmwarePath := filepath.Join("/lib/firmware/nvidia", version, "gsp.bin")
+	return discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{gspFirmwarePath},
+	)
+}
+
+// NewDriverBinariesDiscoverer creates a discoverer for GSP firmware associated with the GPU driver.
+func NewDriverBinariesDiscoverer(logger *logrus.Logger, root string) discover.Discover {
+	return discover.NewMounts(
+		logger,
+		lookup.NewExecutableLocator(logger, root),
+		root,
+		[]string{
+			"nvidia-smi",              /* System management interface */
+			"nvidia-debugdump",        /* GPU coredump utility */
+			"nvidia-persistenced",     /* Persistence mode utility */
+			"nvidia-cuda-mps-control", /* Multi process service CLI */
+			"nvidia-cuda-mps-server",  /* Multi process service server */
+		},
+	)
+}
+
+// getVersionLibs checks the LDCache for libraries ending in the specified driver version.
+// Although the ldcache at the specified root is queried, the paths are returned relative to this root.
+// This allows the standard mount location logic to be used for resolving the mounts.
+func getVersionLibs(logger *logrus.Logger, root string, version string) ([]string, error) {
+	logger.Infof("Using driver version %v", version)
+
+	cache, err := ldcache.New(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load ldcache: %v", err)
+	}
+
+	libs32, libs64 := cache.List()
+
+	var libs []string
+	for _, l := range libs64 {
+		if strings.HasSuffix(l, version) {
+			logger.Infof("found 64-bit driver lib: %v", l)
+			libs = append(libs, l)
+		}
+	}
+
+	for _, l := range libs32 {
+		if strings.HasSuffix(l, version) {
+			logger.Infof("found 32-bit driver lib: %v", l)
+			libs = append(libs, l)
+		}
+	}
+
+	if root == "/" || root == "" {
+		return libs, nil
+	}
+
+	var relative []string
+	for _, l := range libs {
+		relative = append(relative, strings.TrimPrefix(l, root))
+	}
+
+	return relative, nil
+}

cmd/nvidia-ctk/cdi/generate/full-gpu.go

@@ -0,0 +1,160 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// byPathHookDiscoverer discovers the entities required for injecting by-path DRM device links
+type byPathHookDiscoverer struct {
+	logger        *logrus.Logger
+	root          string
+	nvidiaCTKPath string
+	pciBusID      string
+}
+
+var _ discover.Discover = (*byPathHookDiscoverer)(nil)
+
+// NewFullGPUDiscoverer creates a discoverer for the full GPU defined by the specified device.
+func NewFullGPUDiscoverer(logger *logrus.Logger, root string, nvidiaCTKPath string, d device.Device) (discover.Discover, error) {
+	// TODO: The functionality to get device paths should be integrated into the go-nvlib/pkg/device.Device interface.
+	// This will allow reuse here and in other code where the paths are queried such as the NVIDIA device plugin.
+	minor, ret := d.GetMinorNumber()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
+	}
+	path := fmt.Sprintf("/dev/nvidia%d", minor)
+
+	pciInfo, ret := d.GetPciInfo()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting PCI info for device: %v", ret)
+	}
+	pciBusID := getBusID(pciInfo)
+
+	drmDeviceNodes, err := drm.GetDeviceNodesByBusID(pciBusID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", pciBusID, err)
+	}
+
+	deviceNodePaths := append([]string{path}, drmDeviceNodes...)
+
+	deviceNodes := discover.NewCharDeviceDiscoverer(
+		logger,
+		deviceNodePaths,
+		root,
+	)
+
+	byPathHooks := &byPathHookDiscoverer{
+		logger:        logger,
+		root:          root,
+		nvidiaCTKPath: nvidiaCTKPath,
+		pciBusID:      pciBusID,
+	}
+
+	dd := discover.Merge(
+		deviceNodes,
+		byPathHooks,
+	)
+
+	return dd, nil
+}
+
+// Devices returns the empty list for the by-path hook discoverer
+func (d *byPathHookDiscoverer) Devices() ([]discover.Device, error) {
+	return nil, nil
+}
+
+// Hooks returns the hooks for the GPU device.
+// The following hooks are detected:
+//  1. A hook to create /dev/dri/by-path symlinks
+func (d *byPathHookDiscoverer) Hooks() ([]discover.Hook, error) {
+	links, err := d.deviceNodeLinks()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover DRA device links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	hook := discover.CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []discover.Hook{hook}, nil
+}
+
+// Mounts returns an empty slice for a full GPU
+func (d *byPathHookDiscoverer) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}
+
+func (d *byPathHookDiscoverer) deviceNodeLinks() ([]string, error) {
+	candidates := []string{
+		fmt.Sprintf("/dev/dri/by-path/pci-%s-card", d.pciBusID),
+		fmt.Sprintf("/dev/dri/by-path/pci-%s-render", d.pciBusID),
+	}
+
+	var links []string
+	for _, c := range candidates {
+		linkPath := filepath.Join(d.root, c)
+		device, err := os.Readlink(linkPath)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", linkPath)
+			continue
+		}
+
+		d.logger.Debugf("adding device symlink %v -> %v", linkPath, device)
+		links = append(links, fmt.Sprintf("%v::%v", device, linkPath))
+	}
+
+	return links, nil
+}
+
+// getBusID provides a utility function that returns the string representation of the bus ID.
+func getBusID(p nvml.PciInfo) string {
+	var bytes []byte
+	for _, b := range p.BusId {
+		if byte(b) == '\x00' {
+			break
+		}
+		bytes = append(bytes, byte(b))
+	}
+	id := strings.ToLower(string(bytes))
+
+	if id != "0000" {
+		id = strings.TrimPrefix(id, "0000")
+	}
+
+	return id
+}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -0,0 +1,383 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	formatJSON = "json"
+	formatYAML = "yaml"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	output             string
+	format             string
+	deviceNameStrategy string
+	root               string
+	nvidiaCTKPath      string
+}
+
+// NewCommand constructs a generate-cdi command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build creates the CLI command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'generate-cdi' command
+	c := cli.Command{
+		Name:  "generate",
+		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "output",
+			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
+			Destination: &cfg.output,
+		},
+		&cli.StringFlag{
+			Name:        "format",
+			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
+			Value:       formatYAML,
+			Destination: &cfg.format,
+		},
+		&cli.StringFlag{
+			Name:        "device-name-strategy",
+			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
+			Value:       deviceNameStrategyIndex,
+			Destination: &cfg.deviceNameStrategy,
+		},
+		&cli.StringFlag{
+			Name:        "root",
+			Usage:       "Specify the root to use when discovering the entities that should be included in the CDI specification.",
+			Destination: &cfg.root,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-ctk-path",
+			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
+			Destination: &cfg.nvidiaCTKPath,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	cfg.format = strings.ToLower(cfg.format)
+	switch cfg.format {
+	case formatJSON:
+	case formatYAML:
+	default:
+		return fmt.Errorf("invalid output format: %v", cfg.format)
+	}
+
+	_, err := newDeviceNamer(cfg.deviceNameStrategy)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	deviceNamer, err := newDeviceNamer(cfg.deviceNameStrategy)
+	if err != nil {
+		return fmt.Errorf("failed to create device namer: %v", err)
+	}
+
+	spec, err := m.generateSpec(
+		cfg.root,
+		discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath),
+		deviceNamer,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %v", err)
+	}
+
+	var outputTo io.Writer
+	if cfg.output == "" {
+		outputTo = os.Stdout
+	} else {
+		err := createParentDirsIfRequired(cfg.output)
+		if err != nil {
+			return fmt.Errorf("failed to create parent folders for output file: %v", err)
+		}
+		outputFile, err := os.Create(cfg.output)
+		if err != nil {
+			return fmt.Errorf("failed to create output file: %v", err)
+		}
+		defer outputFile.Close()
+		outputTo = outputFile
+	}
+
+	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
+		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
+		if !c.IsSet("format") {
+			cfg.format = outputFileFormat
+		} else if outputFileFormat != cfg.format {
+			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", cfg.format, outputFileFormat)
+		}
+	}
+
+	data, err := yaml.Marshal(spec)
+	if err != nil {
+		return fmt.Errorf("failed to marshal CDI spec: %v", err)
+	}
+
+	if strings.ToLower(cfg.format) == formatJSON {
+		data, err = yaml.YAMLToJSONStrict(data)
+		if err != nil {
+			return fmt.Errorf("failed to convert CDI spec from YAML to JSON: %v", err)
+		}
+	}
+
+	err = writeToOutput(cfg.format, data, outputTo)
+	if err != nil {
+		return fmt.Errorf("failed to write output: %v", err)
+	}
+
+	return nil
+}
+
+func formatFromFilename(filename string) string {
+	ext := filepath.Ext(filename)
+	switch strings.ToLower(ext) {
+	case ".json":
+		return formatJSON
+	case ".yaml":
+		return formatYAML
+	case ".yml":
+		return formatYAML
+	}
+
+	return ""
+}
+
+func writeToOutput(format string, data []byte, output io.Writer) error {
+	if format == formatYAML {
+		_, err := output.Write([]byte("---\n"))
+		if err != nil {
+			return fmt.Errorf("failed to write YAML separator: %v", err)
+		}
+	}
+	_, err := output.Write(data)
+	if err != nil {
+		return fmt.Errorf("failed to write data: %v", err)
+	}
+
+	return nil
+}
+
+func (m command) generateSpec(root string, nvidiaCTKPath string, namer deviceNamer) (*specs.Spec, error) {
+	nvmllib := nvml.New()
+	if r := nvmllib.Init(); r != nvml.SUCCESS {
+		return nil, r
+	}
+	defer nvmllib.Shutdown()
+
+	devicelib := device.New(device.WithNvml(nvmllib))
+
+	deviceSpecs, err := m.generateDeviceSpecs(devicelib, root, nvidiaCTKPath, namer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
+	}
+
+	allDevice := createAllDevice(deviceSpecs)
+
+	deviceSpecs = append(deviceSpecs, allDevice)
+
+	allEdits := edits.NewContainerEdits()
+
+	ipcs, err := NewIPCDiscoverer(m.logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for IPC sockets: %v", err)
+	}
+
+	ipcEdits, err := edits.FromDiscoverer(ipcs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for IPC sockets: %v", err)
+	}
+	// TODO: We should not have to update this after the fact
+	for _, s := range ipcEdits.Mounts {
+		s.Options = append(s.Options, "noexec")
+	}
+
+	allEdits.Append(ipcEdits)
+
+	common, err := NewCommonDiscoverer(m.logger, root, nvidiaCTKPath, nvmllib)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for common entities: %v", err)
+	}
+
+	deviceFolderPermissionHooks, err := NewDeviceFolderPermissionHookDiscoverer(m.logger, root, nvidiaCTKPath, deviceSpecs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generated permission hooks for device nodes: %v", err)
+	}
+
+	commonEdits, err := edits.FromDiscoverer(discover.Merge(common, deviceFolderPermissionHooks))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for common entities: %v", err)
+	}
+
+	allEdits.Append(commonEdits)
+
+	// We construct the spec and determine the minimum required version based on the specification.
+	spec := specs.Spec{
+		Version:        "NOT_SET",
+		Kind:           "nvidia.com/gpu",
+		Devices:        deviceSpecs,
+		ContainerEdits: *allEdits.ContainerEdits,
+	}
+
+	minVersion, err := cdi.MinimumRequiredVersion(&spec)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get minumum required CDI spec version: %v", err)
+	}
+	m.logger.Infof("Using minimum required CDI spec version: %s", minVersion)
+
+	spec.Version = minVersion
+
+	return &spec, nil
+}
+
+func (m command) generateDeviceSpecs(devicelib device.Interface, root string, nvidiaCTKPath string, namer deviceNamer) ([]specs.Device, error) {
+	var deviceSpecs []specs.Device
+
+	err := devicelib.VisitDevices(func(i int, d device.Device) error {
+		isMigEnabled, err := d.IsMigEnabled()
+		if err != nil {
+			return fmt.Errorf("failed to check whether device is MIG device: %v", err)
+		}
+		if isMigEnabled {
+			return nil
+		}
+		device, err := NewFullGPUDiscoverer(m.logger, root, nvidiaCTKPath, d)
+		if err != nil {
+			return fmt.Errorf("failed to create device: %v", err)
+		}
+
+		deviceEdits, err := edits.FromDiscoverer(device)
+		if err != nil {
+			return fmt.Errorf("failed to create container edits for device: %v", err)
+		}
+
+		deviceName, err := namer.GetDeviceName(i, d)
+		if err != nil {
+			return fmt.Errorf("failed to get device name: %v", err)
+		}
+		deviceSpec := specs.Device{
+			Name:           deviceName,
+			ContainerEdits: *deviceEdits.ContainerEdits,
+		}
+
+		deviceSpecs = append(deviceSpecs, deviceSpec)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate CDI spec for GPU devices: %v", err)
+	}
+
+	err = devicelib.VisitMigDevices(func(i int, d device.Device, j int, mig device.MigDevice) error {
+		device, err := NewMigDeviceDiscoverer(m.logger, "", d, mig)
+		if err != nil {
+			return fmt.Errorf("failed to create MIG device: %v", err)
+		}
+
+		deviceEdits, err := edits.FromDiscoverer(device)
+		if err != nil {
+			return fmt.Errorf("failed to create container edits for MIG device: %v", err)
+		}
+
+		deviceName, err := namer.GetMigDeviceName(i, j, mig)
+		if err != nil {
+			return fmt.Errorf("failed to get device name: %v", err)
+		}
+		deviceSpec := specs.Device{
+			Name:           deviceName,
+			ContainerEdits: *deviceEdits.ContainerEdits,
+		}
+
+		deviceSpecs = append(deviceSpecs, deviceSpec)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("falied to generate CDI spec for MIG devices: %v", err)
+	}
+
+	return deviceSpecs, nil
+}
+
+// createAllDevice creates an 'all' device which combines the edits from the previous devices
+func createAllDevice(deviceSpecs []specs.Device) specs.Device {
+	edits := edits.NewContainerEdits()
+
+	for _, d := range deviceSpecs {
+		edit := cdi.ContainerEdits{
+			ContainerEdits: &d.ContainerEdits,
+		}
+		edits.Append(&edit)
+	}
+
+	all := specs.Device{
+		Name:           "all",
+		ContainerEdits: *edits.ContainerEdits,
+	}
+	return all
+}
+
+// createParentDirsIfRequired creates the parent folders of the specified path if requried.
+// Note that MkdirAll does not specifically check whether the specified path is non-empty and raises an error if it is.
+// The path will be empty if filename in the current folder is specified, for example
+func createParentDirsIfRequired(filename string) error {
+	dir := filepath.Dir(filename)
+	if dir == "" {
+		return nil
+	}
+	return os.MkdirAll(dir, 0755)
+}

cmd/nvidia-ctk/cdi/generate/ipc.go

@@ -0,0 +1,42 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets.
+func NewIPCDiscoverer(logger *logrus.Logger, root string) (discover.Discover, error) {
+	d := discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{
+			"/var/run/nvidia-persistenced/socket",
+			"/var/run/nvidia-fabricmanager/socket",
+			"/tmp/nvidia-mps",
+		},
+	)
+
+	return d, nil
+}

cmd/nvidia-ctk/cdi/generate/mig-device.go

@@ -0,0 +1,75 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+// NewMigDeviceDiscoverer creates a discoverer for the specified mig device and its parent.
+func NewMigDeviceDiscoverer(logger *logrus.Logger, root string, parent device.Device, d device.MigDevice) (discover.Discover, error) {
+	minor, ret := parent.GetMinorNumber()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
+	}
+	parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
+
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
+	}
+
+	gi, ret := d.GetGpuInstanceId()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
+	}
+
+	ci, ret := d.GetComputeInstanceId()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
+	}
+
+	giCap := nvcaps.NewGPUInstanceCap(minor, gi)
+	giCapDevicePath, err := migCaps.GetCapDevicePath(giCap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GI cap device path: %v", err)
+	}
+
+	ciCap := nvcaps.NewComputeInstanceCap(minor, gi, ci)
+	ciCapDevicePath, err := migCaps.GetCapDevicePath(ciCap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get CI cap device path: %v", err)
+	}
+
+	deviceNodes := discover.NewCharDeviceDiscoverer(
+		logger,
+		[]string{
+			parentPath,
+			giCapDevicePath,
+			ciCapDevicePath,
+		},
+		root,
+	)
+
+	return deviceNodes, nil
+}

cmd/nvidia-ctk/cdi/generate/namer.go

@@ -0,0 +1,84 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package generate
+
+import (
+	"fmt"
+
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvml"
+)
+
+type deviceNamer interface {
+	GetDeviceName(int, device.Device) (string, error)
+	GetMigDeviceName(int, int, device.MigDevice) (string, error)
+}
+
+const (
+	deviceNameStrategyIndex     = "index"
+	deviceNameStrategyTypeIndex = "type-index"
+	deviceNameStrategyUUID      = "uuid"
+)
+
+type deviceNameIndex struct {
+	gpuPrefix string
+	migPrefix string
+}
+type deviceNameUUID struct{}
+
+// newDeviceNamer creates a Device Namer based on the supplied strategy.
+// This namer can be used to construct the names for MIG and GPU devices when generating the CDI spec.
+func newDeviceNamer(strategy string) (deviceNamer, error) {
+	switch strategy {
+	case deviceNameStrategyIndex:
+		return deviceNameIndex{}, nil
+	case deviceNameStrategyTypeIndex:
+		return deviceNameIndex{gpuPrefix: "gpu", migPrefix: "mig"}, nil
+	case deviceNameStrategyUUID:
+		return deviceNameUUID{}, nil
+	}
+
+	return nil, fmt.Errorf("invalid device name strategy: %v", strategy)
+}
+
+// GetDeviceName returns the name for the specified device based on the naming strategy
+func (s deviceNameIndex) GetDeviceName(i int, d device.Device) (string, error) {
+	return fmt.Sprintf("%s%d", s.gpuPrefix, i), nil
+}
+
+// GetMigDeviceName returns the name for the specified device based on the naming strategy
+func (s deviceNameIndex) GetMigDeviceName(i int, j int, d device.MigDevice) (string, error) {
+	return fmt.Sprintf("%s%d:%d", s.migPrefix, i, j), nil
+}
+
+// GetDeviceName returns the name for the specified device based on the naming strategy
+func (s deviceNameUUID) GetDeviceName(i int, d device.Device) (string, error) {
+	uuid, ret := d.GetUUID()
+	if ret != nvml.SUCCESS {
+		return "", fmt.Errorf("failed to get device UUID: %v", ret)
+	}
+	return uuid, nil
+}
+
+// GetMigDeviceName returns the name for the specified device based on the naming strategy
+func (s deviceNameUUID) GetMigDeviceName(i int, j int, d device.MigDevice) (string, error) {
+	uuid, ret := d.GetUUID()
+	if ret != nvml.SUCCESS {
+		return "", fmt.Errorf("failed to get device UUID: %v", ret)
+	}
+	return uuid, nil
+}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -0,0 +1,140 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package chmod
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	paths         cli.StringSlice
+	mode          string
+	containerSpec string
+}
+
+// NewCommand constructs a chmod command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the chmod command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'chmod' command
+	c := cli.Command{
+		Name:  "chmod",
+		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
+		Before: func(c *cli.Context) error {
+			return validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "path",
+			Usage:       "Specifiy a path to apply the specified mode to",
+			Destination: &cfg.paths,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "Specify the file mode",
+			Destination: &cfg.mode,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func validateFlags(c *cli.Context, cfg *config) error {
+	if strings.TrimSpace(cfg.mode) == "" {
+		return fmt.Errorf("a non-empty mode must be specified")
+	}
+
+	for _, p := range cfg.paths.Value() {
+		if strings.TrimSpace(p) == "" {
+			return fmt.Errorf("paths must not be empty")
+		}
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+	if containerRoot == "" {
+		return fmt.Errorf("empty container root detected")
+	}
+
+	paths := m.getPaths(containerRoot, cfg.paths.Value())
+	if len(paths) == 0 {
+		m.logger.Debugf("No paths specified; exiting")
+		return nil
+	}
+
+	locator := lookup.NewExecutableLocator(m.logger, "")
+	targets, err := locator.Locate("chmod")
+	if err != nil {
+		return fmt.Errorf("failed to locate chmod: %v", err)
+	}
+	chmodPath := targets[0]
+
+	args := append([]string{filepath.Base(chmodPath), cfg.mode}, paths...)
+
+	return syscall.Exec(chmodPath, args, nil)
+}
+
+// getPaths updates the specified paths relative to the root.
+func (m command) getPaths(root string, paths []string) []string {
+	var pathsInRoot []string
+	for _, f := range paths {
+		pathsInRoot = append(pathsInRoot, filepath.Join(root, f))
+	}
+
+	return pathsInRoot
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -0,0 +1,229 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package symlinks
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	hostRoot      string
+	filenames     cli.StringSlice
+	links         cli.StringSlice
+	containerSpec string
+}
+
+// NewCommand constructs a hook command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the '' command
+	c := cli.Command{
+		Name:  "create-symlinks",
+		Usage: "A hook to create symlinks in the container. This can be used to proces CSV mount specs",
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "The root on the host filesystem to use to resolve symlinks",
+			Destination: &cfg.hostRoot,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv-filename",
+			Usage:       "Specify a (CSV) filename to process",
+			Destination: &cfg.filenames,
+		},
+		&cli.StringSliceFlag{
+			Name:        "link",
+			Usage:       "Specify a specific link to create. The link is specified as target::link",
+			Destination: &cfg.links,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	csvFiles := cfg.filenames.Value()
+
+	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
+
+	var candidates []string
+	for _, file := range csvFiles {
+		mountSpecs, err := csv.NewCSVFileParser(m.logger, file).Parse()
+		if err != nil {
+			m.logger.Debugf("Skipping CSV file %v: %v", file, err)
+			continue
+		}
+
+		for _, ms := range mountSpecs {
+			if ms.Type != csv.MountSpecSym {
+				continue
+			}
+			targets, err := chainLocator.Locate(ms.Path)
+			if err != nil {
+				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
+			}
+			candidates = append(candidates, targets...)
+		}
+	}
+
+	created := make(map[string]bool)
+	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
+	for _, candidate := range candidates {
+		targets, err := m.Locate(candidate)
+		if err != nil {
+			m.logger.Debugf("Skipping invalid link: %v", err)
+			continue
+		} else if len(targets) != 1 {
+			m.logger.Debugf("Unexepected number of targets: %v", targets)
+			continue
+		} else if targets[0] == candidate {
+			m.logger.Debugf("%v is not a symlink", candidate)
+			continue
+		}
+
+		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
+		if err != nil {
+			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
+		}
+	}
+
+	links := cfg.links.Value()
+	for _, l := range links {
+		parts := strings.Split(l, "::")
+		if len(parts) != 2 {
+			m.logger.Warnf("Invalid link specification %v", l)
+			continue
+		}
+
+		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
+		if err != nil {
+			m.logger.Warnf("Failed to create link %v: %v", parts, err)
+		}
+	}
+
+	return nil
+
+}
+
+func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
+	linkPath, err := changeRoot(hostRoot, containerRoot, link)
+	if err != nil {
+		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
+	}
+	if created[linkPath] {
+		m.logger.Debugf("Link %v already created", linkPath)
+		return nil
+	}
+
+	targetPath, err := changeRoot(hostRoot, "/", target)
+	if err != nil {
+		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
+	}
+
+	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)
+	err = os.MkdirAll(filepath.Dir(linkPath), 0755)
+	if err != nil {
+		return fmt.Errorf("failed to create directory: %v", err)
+	}
+	err = os.Symlink(target, linkPath)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink: %v", err)
+	}
+
+	return nil
+}
+
+func changeRoot(current string, new string, path string) (string, error) {
+	if !filepath.IsAbs(path) {
+		return path, nil
+	}
+
+	relative := path
+	if current != "" {
+		r, err := filepath.Rel(current, path)
+		if err != nil {
+			return "", err
+		}
+		relative = r
+	}
+
+	return filepath.Join(new, relative), nil
+}
+
+// Locate returns the link target of the specified filename or an empty slice if the
+// specified filename is not a symlink.
+func (m command) Locate(filename string) ([]string, error) {
+	info, err := os.Lstat(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get file info: %v", info)
+	}
+	if info.Mode()&os.ModeSymlink == 0 {
+		m.logger.Debugf("%v is not a symlink", filename)
+		return nil, nil
+	}
+
+	target, err := os.Readlink(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error checking symlink: %v", err)
+	}
+
+	m.logger.Debugf("Resolved link: '%v' => '%v'", filename, target)
+
+	return []string{target}, nil
+}

cmd/nvidia-ctk/hook/hook.go

@@ -0,0 +1,55 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package hook
+
+import (
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+
+	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type hookCommand struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a hook command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := hookCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m hookCommand) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "hook",
+		Usage: "A collection of hooks that may be injected into an OCI spec",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		ldcache.NewCommand(m.logger),
+		symlinks.NewCommand(m.logger),
+		chmod.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -0,0 +1,129 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package ldcache
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	folders       cli.StringSlice
+	containerSpec string
+}
+
+// NewCommand constructs an update-ldcache command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the update-ldcache command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'update-ldcache' command
+	c := cli.Command{
+		Name:  "update-ldcache",
+		Usage: "Update ldcache in a container by running ldconfig",
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "folder",
+			Usage:       "Specifiy a folder to add to /etc/ld.so.conf before updating the ld cache",
+			Destination: &cfg.folders,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	err = m.createConfig(containerRoot, cfg.folders.Value())
+	if err != nil {
+		return fmt.Errorf("failed to update ld.so.conf: %v", err)
+	}
+
+	args := []string{"/sbin/ldconfig"}
+	if containerRoot != "" {
+		args = append(args, "-r", containerRoot)
+	}
+
+	return syscall.Exec(args[0], args, nil)
+}
+
+// createConfig creates (or updates) /etc/ld.so.conf.d/nvcr-<RANDOM_STRING>.conf in the container
+// to include the required paths.
+func (m command) createConfig(root string, folders []string) error {
+	if len(folders) == 0 {
+		m.logger.Debugf("No folders to add to /etc/ld.so.conf")
+		return nil
+	}
+
+	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
+	if err != nil {
+		return fmt.Errorf("failed to create config file: %v", err)
+	}
+	defer configFile.Close()
+
+	m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name())
+
+	configured := make(map[string]bool)
+	for _, folder := range folders {
+		if configured[folder] {
+			continue
+		}
+		_, err = configFile.WriteString(fmt.Sprintf("%s\n", folder))
+		if err != nil {
+			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
+		}
+		configured[folder] = true
+	}
+
+	return nil
+}

cmd/nvidia-ctk/info/info.go

@@ -0,0 +1,47 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package info
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an info command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "info",
+		Usage: "Provide information about the system",
+	}
+
+	hook.Subcommands = []*cli.Command{}
+
+	return &hook
+}

cmd/nvidia-ctk/main.go

@@ -0,0 +1,90 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+var logger = log.New()
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line flags
+type config struct {
+	// Debug indicates whether the CLI is started in "debug" mode
+	Debug bool
+}
+
+func main() {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "NVIDIA Container Toolkit CLI"
+	c.UseShortOptionHandling = true
+	c.EnableBashCompletion = true
+	c.Usage = "Tools to configure the NVIDIA Container Toolkit"
+	c.Version = info.GetVersionString()
+
+	// Setup the flags for this command
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "debug",
+			Aliases:     []string{"d"},
+			Usage:       "Enable debug-level logging",
+			Destination: &config.Debug,
+			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
+		},
+	}
+
+	// Set log-level for all subcommands
+	c.Before = func(c *cli.Context) error {
+		logLevel := log.InfoLevel
+		if config.Debug {
+			logLevel = log.DebugLevel
+		}
+		logger.SetLevel(logLevel)
+		return nil
+	}
+
+	// Define the subcommands
+	c.Commands = []*cli.Command{
+		hook.NewCommand(logger),
+		runtime.NewCommand(logger),
+		infoCLI.NewCommand(logger),
+		cdi.NewCommand(logger),
+		system.NewCommand(logger),
+	}
+
+	// Run the CLI
+	err := c.Run(os.Args)
+	if err != nil {
+		log.Errorf("%v", err)
+		log.Exit(1)
+	}
+}

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -0,0 +1,203 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package configure
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/crio"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/docker"
+	"github.com/pelletier/go-toml"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultRuntime = "docker"
+
+	defaultDockerConfigFilePath = "/etc/docker/daemon.json"
+	defaultCrioConfigFilePath   = "/etc/crio/crio.conf"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs an configure command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line config
+type config struct {
+	dryRun         bool
+	runtime        string
+	configFilePath string
+	nvidiaOptions  nvidia.Options
+}
+
+func (m command) build() *cli.Command {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the 'configure' command
+	configure := cli.Command{
+		Name:  "configure",
+		Usage: "Add a runtime to the specified container engine",
+		Action: func(c *cli.Context) error {
+			return m.configureWrapper(c, &config)
+		},
+	}
+
+	configure.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "update the runtime configuration as required but don't write changes to disk",
+			Destination: &config.dryRun,
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Usage:       "the target runtime engine. One of [crio, docker]",
+			Value:       defaultRuntime,
+			Destination: &config.runtime,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to the config file for the target runtime",
+			Destination: &config.configFilePath,
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-runtime-name",
+			Usage:       "specify the name of the NVIDIA runtime that will be added",
+			Value:       nvidia.RuntimeName,
+			Destination: &config.nvidiaOptions.RuntimeName,
+		},
+		&cli.StringFlag{
+			Name:        "runtime-path",
+			Usage:       "specify the path to the NVIDIA runtime executable",
+			Value:       nvidia.RuntimeExecutable,
+			Destination: &config.nvidiaOptions.RuntimePath,
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Usage:       "set the specified runtime as the default runtime",
+			Destination: &config.nvidiaOptions.SetAsDefault,
+		},
+	}
+
+	return &configure
+}
+
+func (m command) configureWrapper(c *cli.Context, config *config) error {
+	switch config.runtime {
+	case "crio":
+		return m.configureCrio(c, config)
+	case "docker":
+		return m.configureDocker(c, config)
+	}
+
+	return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
+}
+
+// configureDocker updates the docker config to enable the NVIDIA Container Runtime
+func (m command) configureDocker(c *cli.Context, config *config) error {
+	configFilePath := config.configFilePath
+	if configFilePath == "" {
+		configFilePath = defaultDockerConfigFilePath
+	}
+
+	cfg, err := docker.LoadConfig(configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = docker.UpdateConfig(
+		cfg,
+		config.nvidiaOptions.RuntimeName,
+		config.nvidiaOptions.RuntimePath,
+		config.nvidiaOptions.SetAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.dryRun {
+		output, err := json.MarshalIndent(cfg, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to convert to JSON: %v", err)
+		}
+		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
+		return nil
+	}
+	err = docker.FlushConfig(cfg, configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	m.logger.Infof("It is recommended that the docker daemon be restarted.")
+
+	return nil
+}
+
+// configureCrio updates the crio config to enable the NVIDIA Container Runtime
+func (m command) configureCrio(c *cli.Context, config *config) error {
+	configFilePath := config.configFilePath
+	if configFilePath == "" {
+		configFilePath = defaultCrioConfigFilePath
+	}
+
+	cfg, err := crio.LoadConfig(configFilePath)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = crio.UpdateConfig(
+		cfg,
+		config.nvidiaOptions.RuntimeName,
+		config.nvidiaOptions.RuntimePath,
+		config.nvidiaOptions.SetAsDefault,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	if config.dryRun {
+		output, err := toml.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("unable to convert to TOML: %v", err)
+		}
+		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
+		return nil
+	}
+	err = crio.FlushConfig(configFilePath, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	m.logger.Infof("Wrote updated config to %v", configFilePath)
+	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
+
+	return nil
+}

cmd/nvidia-ctk/runtime/nvidia/nvidia.go

@@ -0,0 +1,75 @@
+/*
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package nvidia
+
+const (
+	// RuntimeName is the default name to use in configs for the NVIDIA Container Runtime
+	RuntimeName = "nvidia"
+	// RuntimeExecutable is the default NVIDIA Container Runtime executable file name
+	RuntimeExecutable = "nvidia-container-runtime"
+)
+
+// Options specifies the options for the NVIDIA Container Runtime w.r.t a container engine such as docker.
+type Options struct {
+	SetAsDefault bool
+	RuntimeName  string
+	RuntimePath  string
+}
+
+// Runtime defines an NVIDIA runtime with a name and a executable
+type Runtime struct {
+	Name string
+	Path string
+}
+
+// DefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o Options) DefaultRuntime() string {
+	if !o.SetAsDefault {
+		return ""
+	}
+
+	return o.RuntimeName
+}
+
+// Runtime creates a runtime struct based on the options.
+func (o Options) Runtime() Runtime {
+	path := o.RuntimePath
+
+	if o.RuntimePath == "" {
+		path = RuntimeExecutable
+	}
+
+	r := Runtime{
+		Name: o.RuntimeName,
+		Path: path,
+	}
+
+	return r
+}
+
+// DockerRuntimesConfig generatest the expected docker config for the specified runtime
+func (r Runtime) DockerRuntimesConfig() map[string]interface{} {
+	runtimes := make(map[string]interface{})
+	runtimes[r.Name] = map[string]interface{}{
+		"path": r.Path,
+		"args": []string{},
+	}
+
+	return runtimes
+}

cmd/nvidia-ctk/runtime/runtime.go

@@ -0,0 +1,49 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package runtime
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type runtimeCommand struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := runtimeCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m runtimeCommand) build() *cli.Command {
+	// Create the 'runtime' command
+	runtime := cli.Command{
+		Name:  "runtime",
+		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	runtime.Subcommands = []*cli.Command{
+		configure.NewCommand(m.logger),
+	}
+
+	return &runtime
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -0,0 +1,175 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
+	"github.com/sirupsen/logrus"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
+)
+
+type allPossible struct {
+	logger       *logrus.Logger
+	driverRoot   string
+	deviceMajors devices.Devices
+	migCaps      nvcaps.MigCaps
+}
+
+// newAllPossible returns a new allPossible device node lister.
+// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error) {
+	deviceMajors, err := devices.GetNVIDIADevices()
+	if err != nil {
+		return nil, fmt.Errorf("failed reading device majors: %v", err)
+	}
+	migCaps, err := nvcaps.NewMigCaps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
+	}
+	if migCaps == nil {
+		migCaps = make(nvcaps.MigCaps)
+	}
+
+	l := allPossible{
+		logger:       logger,
+		driverRoot:   driverRoot,
+		deviceMajors: deviceMajors,
+		migCaps:      migCaps,
+	}
+
+	return l, nil
+}
+
+// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
+func (m allPossible) DeviceNodes() ([]deviceNode, error) {
+	gpus, err := nvpci.NewFrom(
+		filepath.Join(m.driverRoot, nvpci.PCIDevicesRoot),
+	).GetGPUs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get GPU information: %v", err)
+	}
+
+	count := len(gpus)
+	if count == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		return nil, nil
+	}
+
+	deviceNodes, err := m.getControlDeviceNodes()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
+	}
+
+	for gpu := 0; gpu < count; gpu++ {
+		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
+		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
+	}
+
+	return deviceNodes, nil
+}
+
+// getControlDeviceNodes generates a list of control devices
+func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
+	var deviceNodes []deviceNode
+
+	// Define the control devices for standard GPUs.
+	controlDevices := []deviceNode{
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
+		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
+		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
+	}
+
+	deviceNodes = append(deviceNodes, controlDevices...)
+
+	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
+		migControlMinor, exist := m.migCaps[migControlDevice]
+		if !exist {
+			continue
+		}
+
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			migControlMinor.DevicePath(),
+			int(migControlMinor),
+		)
+
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes, nil
+}
+
+// getGPUDeviceNodes generates a list of device nodes for a given GPU.
+func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
+	d := m.newDeviceNode(
+		devices.NVIDIAGPU,
+		fmt.Sprintf("/dev/nvidia%d", gpu),
+		gpu,
+	)
+
+	return []deviceNode{d}
+}
+
+// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
+func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
+	var selectedCapMinors []nvcaps.MigMinor
+	for gi := 0; ; gi++ {
+		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
+		giMinor, exist := m.migCaps[giCap]
+		if !exist {
+			break
+		}
+		selectedCapMinors = append(selectedCapMinors, giMinor)
+		for ci := 0; ; ci++ {
+			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
+			ciMinor, exist := m.migCaps[ciCap]
+			if !exist {
+				break
+			}
+			selectedCapMinors = append(selectedCapMinors, ciMinor)
+		}
+	}
+
+	var deviceNodes []deviceNode
+	for _, capMinor := range selectedCapMinors {
+		d := m.newDeviceNode(
+			devices.NVIDIACaps,
+			capMinor.DevicePath(),
+			int(capMinor),
+		)
+		deviceNodes = append(deviceNodes, d)
+	}
+
+	return deviceNodes
+}
+
+// newDeviceNode creates a new device node with the specified path and major/minor numbers.
+// The path is adjusted for the specified driver root.
+func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
+	major, _ := m.deviceMajors.Get(deviceName)
+
+	return deviceNode{
+		path:  filepath.Join(m.driverRoot, path),
+		major: uint32(major),
+		minor: uint32(minor),
+	}
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -0,0 +1,332 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultDevCharPath = "/dev/char"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	devCharPath string
+	driverRoot  string
+	dryRun      bool
+	watch       bool
+	createAll   bool
+}
+
+// NewCommand constructs a command sub-command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'create-dev-char-symlinks' command
+	c := cli.Command{
+		Name:  "create-dev-char-symlinks",
+		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "dev-char-path",
+			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
+			Value:       defaultDevCharPath,
+			Destination: &cfg.devCharPath,
+			EnvVars:     []string{"DEV_CHAR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "driver-root",
+			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
+			Value:       "/",
+			Destination: &cfg.driverRoot,
+			EnvVars:     []string{"DRIVER_ROOT"},
+		},
+		&cli.BoolFlag{
+			Name:        "watch",
+			Usage:       "If set, the command will watch for changes to the driver root and recreate the symlinks when changes are detected.",
+			Value:       false,
+			Destination: &cfg.watch,
+			EnvVars:     []string{"WATCH"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-all",
+			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
+			Destination: &cfg.createAll,
+			EnvVars:     []string{"CREATE_ALL"},
+		},
+		&cli.BoolFlag{
+			Name:        "dry-run",
+			Usage:       "If set, the command will not create any symlinks.",
+			Value:       false,
+			Destination: &cfg.dryRun,
+			EnvVars:     []string{"DRY_RUN"},
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(r *cli.Context, cfg *config) error {
+	if cfg.createAll && cfg.watch {
+		return fmt.Errorf("create-all and watch are mutually exclusive")
+	}
+
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	var watcher *fsnotify.Watcher
+	var sigs chan os.Signal
+
+	if cfg.watch {
+		watcher, err := newFSWatcher(filepath.Join(cfg.driverRoot, "dev"))
+		if err != nil {
+			return fmt.Errorf("failed to create FS watcher: %v", err)
+		}
+		defer watcher.Close()
+
+		sigs = newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	}
+
+	l, err := NewSymlinkCreator(
+		WithLogger(m.logger),
+		WithDevCharPath(cfg.devCharPath),
+		WithDriverRoot(cfg.driverRoot),
+		WithDryRun(cfg.dryRun),
+		WithCreateAll(cfg.createAll),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create symlink creator: %v", err)
+	}
+
+create:
+	err = l.CreateLinks()
+	if err != nil {
+		return fmt.Errorf("failed to create links: %v", err)
+	}
+	if !cfg.watch {
+		return nil
+	}
+	for {
+		select {
+
+		case event := <-watcher.Events:
+			deviceNode := filepath.Base(event.Name)
+			if !strings.HasPrefix(deviceNode, "nvidia") {
+				continue
+			}
+			if event.Op&fsnotify.Create == fsnotify.Create {
+				m.logger.Infof("%s created, restarting.", event.Name)
+				goto create
+			}
+			if event.Op&fsnotify.Create == fsnotify.Remove {
+				m.logger.Infof("%s removed. Ignoring", event.Name)
+
+			}
+
+		// Watch for any other fs errors and log them.
+		case err := <-watcher.Errors:
+			m.logger.Errorf("inotify: %s", err)
+
+		// React to signals
+		case s := <-sigs:
+			switch s {
+			case syscall.SIGHUP:
+				m.logger.Infof("Received SIGHUP, recreating symlinks.")
+				goto create
+			default:
+				m.logger.Infof("Received signal %q, shutting down.", s)
+				return nil
+			}
+		}
+	}
+}
+
+type linkCreator struct {
+	logger      *logrus.Logger
+	lister      nodeLister
+	driverRoot  string
+	devCharPath string
+	dryRun      bool
+	createAll   bool
+}
+
+// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
+type Creator interface {
+	CreateLinks() error
+}
+
+// Option is a functional option for configuring the linkCreator.
+type Option func(*linkCreator)
+
+// NewSymlinkCreator creates a new linkCreator.
+func NewSymlinkCreator(opts ...Option) (Creator, error) {
+	c := linkCreator{}
+	for _, opt := range opts {
+		opt(&c)
+	}
+	if c.logger == nil {
+		c.logger = logrus.StandardLogger()
+	}
+	if c.driverRoot == "" {
+		c.driverRoot = "/"
+	}
+	if c.devCharPath == "" {
+		c.devCharPath = defaultDevCharPath
+	}
+
+	if c.createAll {
+		lister, err := newAllPossible(c.logger, c.driverRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
+		}
+		c.lister = lister
+	} else {
+		c.lister = existing{c.logger, c.driverRoot}
+	}
+	return c, nil
+}
+
+// WithDriverRoot sets the driver root path.
+func WithDriverRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.driverRoot = root
+	}
+}
+
+// WithDevCharPath sets the path at which the symlinks will be created.
+func WithDevCharPath(path string) Option {
+	return func(c *linkCreator) {
+		c.devCharPath = path
+	}
+}
+
+// WithDryRun sets the dry run flag.
+func WithDryRun(dryRun bool) Option {
+	return func(c *linkCreator) {
+		c.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger.
+func WithLogger(logger *logrus.Logger) Option {
+	return func(c *linkCreator) {
+		c.logger = logger
+	}
+}
+
+// WithCreateAll sets the createAll flag for the linkCreator.
+func WithCreateAll(createAll bool) Option {
+	return func(lc *linkCreator) {
+		lc.createAll = createAll
+	}
+}
+
+// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
+func (m linkCreator) CreateLinks() error {
+	deviceNodes, err := m.lister.DeviceNodes()
+	if err != nil {
+		return fmt.Errorf("failed to get device nodes: %v", err)
+	}
+
+	if len(deviceNodes) != 0 && !m.dryRun {
+		err := os.MkdirAll(m.devCharPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
+		}
+	}
+
+	for _, deviceNode := range deviceNodes {
+		target := deviceNode.path
+		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
+
+		m.logger.Infof("Creating link %s => %s", linkPath, target)
+		if m.dryRun {
+			continue
+		}
+
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warnf("Could not create symlink: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type deviceNode struct {
+	path  string
+	major uint32
+	minor uint32
+}
+
+func (d deviceNode) devCharName() string {
+	return fmt.Sprintf("%d:%d", d.major, d.minor)
+}
+
+func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, f := range files {
+		err = watcher.Add(f)
+		if err != nil {
+			watcher.Close()
+			return nil, err
+		}
+	}
+
+	return watcher, nil
+}
+
+func newOSWatcher(sigs ...os.Signal) chan os.Signal {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, sigs...)
+
+	return sigChan
+}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -0,0 +1,95 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package devchar
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+type nodeLister interface {
+	DeviceNodes() ([]deviceNode, error)
+}
+
+type existing struct {
+	logger     *logrus.Logger
+	driverRoot string
+}
+
+// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
+// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
+func (m existing) DeviceNodes() ([]deviceNode, error) {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(m.logger),
+		lookup.WithRoot(m.driverRoot),
+		lookup.WithOptional(true),
+	)
+
+	devices, err := locator.Locate("/dev/nvidia*")
+	if err != nil {
+		m.logger.Warnf("Error while locating device: %v", err)
+	}
+
+	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
+	if err != nil {
+		m.logger.Warnf("Error while locating caps device: %v", err)
+	}
+
+	if len(devices) == 0 && len(capDevices) == 0 {
+		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		return nil, nil
+	}
+
+	var deviceNodes []deviceNode
+	for _, d := range append(devices, capDevices...) {
+		if m.nodeIsBlocked(d) {
+			continue
+		}
+
+		var stat unix.Stat_t
+		err := unix.Stat(d, &stat)
+		if err != nil {
+			m.logger.Warnf("Could not stat device: %v", err)
+			continue
+		}
+		deviceNode := deviceNode{
+			path:  d,
+			major: unix.Major(uint64(stat.Rdev)),
+			minor: unix.Minor(uint64(stat.Rdev)),
+		}
+
+		deviceNodes = append(deviceNodes, deviceNode)
+	}
+
+	return deviceNodes, nil
+}
+
+// nodeIsBlocked returns true if the specified device node should be ignored.
+func (m existing) nodeIsBlocked(path string) bool {
+	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
+	nodeName := filepath.Base(path)
+	for _, prefix := range blockedPrefixes {
+		if strings.HasPrefix(nodeName, prefix) {
+			return true
+		}
+	}
+	return false
+}

cmd/nvidia-ctk/system/system.go

@@ -0,0 +1,49 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package system
+
+import (
+	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a runtime command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+func (m command) build() *cli.Command {
+	// Create the 'system' command
+	system := cli.Command{
+		Name:  "system",
+		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
+	}
+
+	system.Subcommands = []*cli.Command{
+		devchar.NewCommand(m.logger),
+	}
+
+	return &system
+}

config/config.toml.amzn

@@ -1,17 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-#alpha-merge-visible-devices-envvars = false
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"

config/config.toml.centos

@@ -1,17 +0,0 @@
-disable-require = false
-#swarm-resource = "DOCKER_RESOURCE_GPU"
-
-[nvidia-container-cli]
-#root = "/run/nvidia/driver"
-#path = "/usr/bin/nvidia-container-cli"
-environment = []
-#debug = "/var/log/nvidia-container-toolkit.log"
-#ldcache = "/etc/ld.so.cache"
-load-kmods = true
-#no-cgroups = false
-#user = "root:video"
-ldconfig = "@/sbin/ldconfig"
-#alpha-merge-visible-devices-envvars = false
-
-[nvidia-container-runtime]
-#debug = "/var/log/nvidia-container-runtime.log"

config/config.toml.debian

@@ -1,5 +1,7 @@
 disable-require = false
 #swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
 
 [nvidia-container-cli]
 #root = "/run/nvidia/driver"
@@ -11,7 +13,20 @@ load-kmods = true
 #no-cgroups = false
 #user = "root:video"
 ldconfig = "@/sbin/ldconfig"
-#alpha-merge-visible-devices-envvars = false
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.opensuse-leap

@@ -1,5 +1,7 @@
 disable-require = false
 #swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
 
 [nvidia-container-cli]
 #root = "/run/nvidia/driver"
@@ -11,7 +13,20 @@ load-kmods = true
 #no-cgroups = false
 user = "root:video"
 ldconfig = "@/sbin/ldconfig"
-#alpha-merge-visible-devices-envvars = false
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.rpm-yum

@@ -0,0 +1,32 @@
+disable-require = false
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+#user = "root:video"
+ldconfig = "@/sbin/ldconfig"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu

@@ -1,5 +1,7 @@
 disable-require = false
 #swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
 
 [nvidia-container-cli]
 #root = "/run/nvidia/driver"
@@ -11,7 +13,20 @@ load-kmods = true
 #no-cgroups = false
 #user = "root:video"
 ldconfig = "@/sbin/ldconfig.real"
-#alpha-merge-visible-devices-envvars = false
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

container_config_test.go

@@ -1,131 +0,0 @@
-package main
-
-import (
-	"github.com/stretchr/testify/require"
-	"sort"
-	"strings"
-	"testing"
-)
-
-func TestMergeVisibleDevicesEnvvars(t *testing.T) {
-	var tests = []struct {
-		name        string
-		input       []string
-		expected    string
-		enableMerge bool
-	}{
-		{
-			"Simple Merge Enabled",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-			},
-			"0,1,2,3,4,5",
-			true,
-		},
-		{
-			"Simple Merge Disabled",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-			},
-			"",
-			false,
-		},
-		{
-			"Merge No Override (Enabled)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES=all",
-			},
-			"all",
-			true,
-		},
-		{
-			"Merge No Override (Disabled)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES=all",
-			},
-			"all",
-			false,
-		},
-		{
-			"Merge Override (Enabled, Before)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES=all",
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-			},
-			"0,1,2,3,4,5",
-			true,
-		},
-		{
-			"Merge Override (Enabled, After)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-				"NVIDIA_VISIBLE_DEVICES=all",
-			},
-			"0,1,2,3,4,5",
-			true,
-		},
-		{
-			"Merge Override (Enabled, In Between)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES=all",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-			},
-			"0,1,2,3,4,5",
-			true,
-		},
-		{
-			"Merge Override (Disabled, Before)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES=all",
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-			},
-			"all",
-			false,
-		},
-		{
-			"Merge Override (Disabled, After)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-				"NVIDIA_VISIBLE_DEVICES=all",
-			},
-			"all",
-			false,
-		},
-		{
-			"Merge Override (Disabled, In Between)",
-			[]string{
-				"NVIDIA_VISIBLE_DEVICES_0=0,1",
-				"NVIDIA_VISIBLE_DEVICES_1=2,3",
-				"NVIDIA_VISIBLE_DEVICES=all",
-				"NVIDIA_VISIBLE_DEVICES_WHATEVER=4,5",
-			},
-			"all",
-			false,
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			config := CLIConfig{
-				AlphaMergeVisibleDevicesEnvvars: tc.enableMerge,
-			}
-			envvars := getEnvMap(tc.input, config)
-			devices := strings.Split(envvars[envNVVisibleDevices], ",")
-			sort.Strings(devices)
-			require.Equal(t, tc.expected, strings.Join(devices, ","))
-		})
-	}
-}

docker.mk

@@ -1,128 +0,0 @@
-# Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
-
-# Supported OSs by architecture
-AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
-X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux1 amazonlinux2 opensuse-leap15.1
-PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
-ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := centos8 rhel8
-
-# Define top-level build targets
-docker%: SHELL:=/bin/bash
-
-# Native targets
-PLATFORM ?= $(shell uname -m)
-ifeq ($(PLATFORM),x86_64)
-NATIVE_TARGETS := $(AMD64_TARGETS) $(X86_64_TARGETS)
-$(AMD64_TARGETS): %: %-amd64
-$(X86_64_TARGETS): %: %-x86_64
-else ifeq ($(PLATFORM),ppc64le)
-NATIVE_TARGETS := $(PPC64LE_TARGETS)
-$(PPC64LE_TARGETS): %: %-ppc64le
-else ifeq ($(PLATFORM),aarch64)
-NATIVE_TARGETS := $(ARM64_TARGETS) $(AARCH64_TARGETS)
-$(ARM64_TARGETS): %: %-arm64
-$(AARCH64_TARGETS): %: %-aarch64
-endif
-docker-native: $(NATIVE_TARGETS)
-
-# amd64 targets
-AMD64_TARGETS := $(patsubst %, %-amd64, $(AMD64_TARGETS))
-$(AMD64_TARGETS): ARCH := amd64
-$(AMD64_TARGETS): %: --%
-docker-amd64: $(AMD64_TARGETS)
-
-# x86_64 targets
-X86_64_TARGETS := $(patsubst %, %-x86_64, $(X86_64_TARGETS))
-$(X86_64_TARGETS): ARCH := x86_64
-$(X86_64_TARGETS): %: --%
-docker-x86_64: $(X86_64_TARGETS)
-
-# arm64 targets
-ARM64_TARGETS := $(patsubst %, %-arm64, $(ARM64_TARGETS))
-$(ARM64_TARGETS): ARCH := arm64
-$(ARM64_TARGETS): %: --%
-docker-arm64: $(ARM64_TARGETS)
-
-# aarch64 targets
-AARCH64_TARGETS := $(patsubst %, %-aarch64, $(AARCH64_TARGETS))
-$(AARCH64_TARGETS): ARCH := aarch64
-$(AARCH64_TARGETS): %: --%
-docker-aarch64: $(AARCH64_TARGETS)
-
-# ppc64le targets
-PPC64LE_TARGETS := $(patsubst %, %-ppc64le, $(PPC64LE_TARGETS))
-$(PPC64LE_TARGETS): ARCH := ppc64le
-$(PPC64LE_TARGETS): WITH_LIBELF := yes
-$(PPC64LE_TARGETS): %: --%
-docker-ppc64le: $(PPC64LE_TARGETS)
-
-# docker target to build for all os/arch combinations
-docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
-            $(ARM64_TARGETS) $(AARCH64_TARGETS) \
-            $(PPC64LE_TARGETS)
-
-# Default variables for all private '--' targets below.
-# One private target is defined for each OS we support.
---%: TARGET_PLATFORM = $(*)
---%: VERSION = $(patsubst $(OS)%-$(ARCH),%,$(TARGET_PLATFORM))
---%: BASEIMAGE = $(OS):$(VERSION)
---%: BUILDIMAGE = nvidia/$(LIB_NAME)/$(OS)$(VERSION)-$(ARCH)
---%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.$(OS)
---%: ARTIFACTS_DIR = $(DIST_DIR)/$(OS)$(VERSION)/$(ARCH)
---%: docker-build-%
-	@
-
-# private ubuntu target
---ubuntu%: OS := ubuntu
---ubuntu%: PKG_REV := 1
-
-# private debian target
---debian%: OS := debian
---debian%: PKG_REV := 1
-
-# private centos target
---centos%: OS := centos
---centos%: PKG_REV := 2
-
-# private amazonlinux target
---amazonlinux%: OS := amazonlinux
---amazonlinux%: PKG_REV = 2.amzn$(VERSION)
-
-# private opensuse-leap target
---opensuse-leap%: OS = opensuse-leap
---opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
---opensuse-leap%: PKG_REV := 1
-
-# private rhel target (actually built on centos)
---rhel%: OS := centos
---rhel%: PKG_REV := 2
---rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
---rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
-
-docker-build-%:
-	@echo "Building for $(TARGET_PLATFORM)"
-	docker pull --platform=linux/$(ARCH) $(BASEIMAGE)
-	DOCKER_BUILDKIT=1 \
-	$(DOCKER) build \
-	    --progress=plain \
-	    --build-arg BASEIMAGE=$(BASEIMAGE) \
-	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-	    --build-arg PKG_VERS="$(LIB_VERSION)" \
-	    --build-arg PKG_REV="$(PKG_REV)" \
-	    --tag $(BUILDIMAGE) \
-	    --file $(DOCKERFILE) .
-	$(DOCKER) run \
-	    -e DISTRIB \
-	    -e SECTION \
-	    -v $(ARTIFACTS_DIR):/dist \
-	    $(BUILDIMAGE)
-
-docker-clean:
-	IMAGES=$$(docker images "nvidia/$(LIB_NAME)/*" --format="{{.ID}}"); \
-	if [ "$${IMAGES}" != "" ]; then \
-	    docker rmi -f $${IMAGES}; \
-	fi
-
-distclean:
-	rm -rf $(DIST_DIR)

docker/Dockerfile.amazonlinux

@@ -1,65 +0,0 @@
-ARG BASEIMAGE
-FROM ${BASEIMAGE}
-
-RUN yum install -y \
-        ca-certificates \
-        wget \
-        git \
-        rpm-build \
-        make && \
-    rm -rf /var/cache/yum/*
-
-ARG GOLANG_VERSION=0.0.0
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture"; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
-
-# packaging
-ARG PKG_VERS
-ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
-
-# output directory
-ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
-RUN mkdir -p $DIST_DIR /dist
-
-# nvidia-container-toolkit
-WORKDIR $GOPATH/src/nvidia-container-toolkit
-COPY . .
-
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
-
-COPY config/config.toml.amzn $DIST_DIR/config.toml
-
-# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
-# This might not be useful on Amazon Linux, but it's simpler to keep the RHEL
-# and Amazon Linux packages identical.
-COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
-
-# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
-COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
-
-WORKDIR $DIST_DIR/..
-COPY packaging/rpm .
-
-CMD arch=$(uname -m) && \
-    rpmbuild --clean --target=$arch -bb \
-             -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "release $RELEASE" \
-             SPECS/nvidia-container-toolkit.spec && \
-    mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -32,6 +32,7 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -48,10 +49,13 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.debian $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Debian Jessie still had ldconfig.real
 RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
@@ -61,9 +65,17 @@ RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
+RUN dch --create --package="${PKG_NAME}" \
+        --newversion "${REVISION}" \
+            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER1_VERSION}" && \
+    dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
-    mv /tmp/nvidia-container-toolkit_*.deb /dist
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
+    --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
+    mv /tmp/*.deb /dist

docker/Dockerfile.devel

@@ -0,0 +1,21 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG GOLANG_VERSION=x.x.x
+FROM golang:${GOLANG_VERSION}
+
+RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
+RUN go install github.com/matryer/moq@latest
+RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
+RUN go install github.com/client9/misspell/cmd/misspell@latest
+RUN go install github.com/google/go-licenses@latest

docker/Dockerfile.opensuse-leap

@@ -25,11 +25,12 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -39,8 +40,9 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
+RUN make PREFIX=${DIST_DIR} cmds
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
@@ -48,15 +50,23 @@ COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
 # Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
 COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 
-COPY config/config.toml.opensuse-leap $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "release $RELEASE" \
+             -D "release_date $(date +'%a %b %d %Y')" \
+             -D "git_commit ${GIT_COMMIT}" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.rpm-yum

@@ -1,8 +1,25 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the dockerfile for building packages on yum-based RPM systems.
+
 ARG BASEIMAGE
 FROM ${BASEIMAGE}
 
 RUN yum install -y \
         ca-certificates \
+        gcc \
         wget \
         git \
         make \
@@ -26,11 +43,12 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-
-ENV VERSION $PKG_VERS
-ENV RELEASE $PKG_REV
+ENV PKG_NAME ${PKG_NAME}
+ENV PKG_VERS ${PKG_VERS}
+ENV PKG_REV ${PKG_REV}
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -40,10 +58,13 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.centos $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
@@ -54,10 +75,16 @@ COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "version $VERSION" \
-             -D "release $RELEASE" \
+             -D "release_date $(date +'%a %b %d %Y')" \
+             -D "git_commit ${GIT_COMMIT}" \
+             -D "version ${PKG_VERS}" \
+             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
+             -D "release ${PKG_REV}" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -30,6 +30,7 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
+ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -46,17 +47,28 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+ARG GIT_COMMIT
+ENV GIT_COMMIT ${GIT_COMMIT}
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.ubuntu $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
+ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
+
+RUN dch --create --package="${PKG_NAME}" \
+        --newversion "${REVISION}" \
+            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
+    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" && \
+    dch -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
+    --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -0,0 +1,179 @@
+# Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Supported OSs by architecture
+AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
+X86_64_TARGETS := fedora35 centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
+PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
+ARM64_TARGETS := ubuntu20.04 ubuntu18.04
+AARCH64_TARGETS := fedora35 centos8 rhel8 amazonlinux2
+
+# Define top-level build targets
+docker%: SHELL:=/bin/bash
+
+# Native targets
+PLATFORM ?= $(shell uname -m)
+ifeq ($(PLATFORM),x86_64)
+NATIVE_TARGETS := $(AMD64_TARGETS) $(X86_64_TARGETS)
+$(AMD64_TARGETS): %: %-amd64
+$(X86_64_TARGETS): %: %-x86_64
+else ifeq ($(PLATFORM),ppc64le)
+NATIVE_TARGETS := $(PPC64LE_TARGETS)
+$(PPC64LE_TARGETS): %: %-ppc64le
+else ifeq ($(PLATFORM),aarch64)
+NATIVE_TARGETS := $(ARM64_TARGETS) $(AARCH64_TARGETS)
+$(ARM64_TARGETS): %: %-arm64
+$(AARCH64_TARGETS): %: %-aarch64
+endif
+docker-native: $(NATIVE_TARGETS)
+
+# amd64 targets
+AMD64_TARGETS := $(patsubst %, %-amd64, $(AMD64_TARGETS))
+$(AMD64_TARGETS): ARCH := amd64
+$(AMD64_TARGETS): %: --%
+docker-amd64: $(AMD64_TARGETS)
+
+# x86_64 targets
+X86_64_TARGETS := $(patsubst %, %-x86_64, $(X86_64_TARGETS))
+$(X86_64_TARGETS): ARCH := x86_64
+$(X86_64_TARGETS): %: --%
+docker-x86_64: $(X86_64_TARGETS)
+
+# arm64 targets
+ARM64_TARGETS := $(patsubst %, %-arm64, $(ARM64_TARGETS))
+$(ARM64_TARGETS): ARCH := arm64
+$(ARM64_TARGETS): %: --%
+docker-arm64: $(ARM64_TARGETS)
+
+# aarch64 targets
+AARCH64_TARGETS := $(patsubst %, %-aarch64, $(AARCH64_TARGETS))
+$(AARCH64_TARGETS): ARCH := aarch64
+$(AARCH64_TARGETS): %: --%
+docker-aarch64: $(AARCH64_TARGETS)
+
+# ppc64le targets
+PPC64LE_TARGETS := $(patsubst %, %-ppc64le, $(PPC64LE_TARGETS))
+$(PPC64LE_TARGETS): ARCH := ppc64le
+$(PPC64LE_TARGETS): WITH_LIBELF := yes
+$(PPC64LE_TARGETS): %: --%
+docker-ppc64le: $(PPC64LE_TARGETS)
+
+# docker target to build for all os/arch combinations
+docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
+            $(ARM64_TARGETS) $(AARCH64_TARGETS) \
+            $(PPC64LE_TARGETS)
+
+# Default variables for all private '--' targets below.
+# One private target is defined for each OS we support.
+--%: TARGET_PLATFORM = $(*)
+--%: VERSION = $(patsubst $(OS)%-$(ARCH),%,$(TARGET_PLATFORM))
+--%: BASEIMAGE = $(OS):$(VERSION)
+--%: BUILDIMAGE = nvidia/$(LIB_NAME)/$(OS)$(VERSION)-$(ARCH)
+--%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.$(OS)
+--%: ARTIFACTS_DIR = $(DIST_DIR)/$(OS)$(VERSION)/$(ARCH)
+--%: docker-build-%
+	@
+
+LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
+LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
+
+# private ubuntu target
+--ubuntu%: OS := ubuntu
+--ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+--ubuntu%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+--ubuntu%: PKG_REV := 1
+
+# private debian target
+--debian%: OS := debian
+--debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+--debian%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
+--debian%: PKG_REV := 1
+
+# private centos target
+--centos%: OS := centos
+--centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
+--centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--centos%: CONFIG_TOML_SUFFIX := rpm-yum
+--centos8%: BASEIMAGE = quay.io/centos/centos:stream8
+
+# private fedora target
+--fedora%: OS := fedora
+--fedora%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--fedora%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
+--fedora%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--fedora%: CONFIG_TOML_SUFFIX := rpm-yum
+# The fedora(35) base image has very slow performance when building aarch64 packages.
+# Since our primary concern here is glibc versions, we use the older glibc version available in centos8.
+--fedora35%: BASEIMAGE = quay.io/centos/centos:stream8
+
+# private amazonlinux target
+--amazonlinux%: OS := amazonlinux
+--amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
+--amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum
+
+# private opensuse-leap target
+--opensuse-leap%: OS = opensuse-leap
+--opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
+--opensuse-leap%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
+--opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+# private rhel target (actually built on centos)
+--rhel%: OS := centos
+--rhel%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1)
+--rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
+--rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
+--rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
+--rhel%: CONFIG_TOML_SUFFIX := rpm-yum
+--rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
+
+
+# We allow the CONFIG_TOML_SUFFIX to be overridden.
+CONFIG_TOML_SUFFIX ?= $(OS)
+
+docker-build-%:
+	@echo "Building for $(TARGET_PLATFORM)"
+	docker pull --platform=linux/$(ARCH) $(BASEIMAGE)
+	DOCKER_BUILDKIT=1 \
+	$(DOCKER) build \
+	    --platform=linux/$(ARCH) \
+	    --progress=plain \
+	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
+	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+	    --build-arg PKG_NAME="$(LIB_NAME)" \
+	    --build-arg PKG_VERS="$(LIB_VERSION)" \
+	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
+	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
+	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \
+	    --tag $(BUILDIMAGE) \
+	    --file $(DOCKERFILE) .
+	$(DOCKER) run \
+	    --platform=linux/$(ARCH) \
+	    -e DISTRIB \
+	    -e SECTION \
+	    -v $(ARTIFACTS_DIR):/dist \
+	    $(BUILDIMAGE)
+
+docker-clean:
+	IMAGES=$$(docker images "nvidia/$(LIB_NAME)/*" --format="{{.ID}}"); \
+	if [ "$${IMAGES}" != "" ]; then \
+	    docker rmi -f $${IMAGES}; \
+	fi
+
+distclean:
+	rm -rf $(DIST_DIR)

go.mod

@@ -1,9 +1,36 @@
-module github.com/NVIDIA/container-toolkit
+module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.14
+go 1.18
 
 require (
-	github.com/BurntSushi/toml v0.3.1
-	github.com/stretchr/testify v1.6.0
-	golang.org/x/mod v0.3.0
+	github.com/BurntSushi/toml v1.0.0
+	github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82
+	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
+	github.com/fsnotify/fsnotify v1.5.4
+	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
+	github.com/pelletier/go-toml v1.9.4
+	github.com/sirupsen/logrus v1.9.0
+	github.com/stretchr/testify v1.7.0
+	github.com/urfave/cli/v2 v2.3.0
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342
+	golang.org/x/mod v0.5.0
+	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
+	sigs.k8s.io/yaml v1.3.0
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.10.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,25 +1,123 @@
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
+github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82 h1:x751Xx1tdxkiA/sdkv2J769n21UbYKzVOpe9S/h1M3k=
+github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
+github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
+github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a h1:sP3PcgyIkRlHqfF3Jfpe/7G8kf/qpzG4C8r94y9hLbE=
+github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a/go.mod h1:xMRa4fJgXzSDFUCURSimOUgoSc+odohvO3uXT9xjqH0=
+github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
+github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
+github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
+github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg=
+github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
+github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb h1:1xSVPOd7/UA+39/hXEGnBJ13p6JFB0E1EvQFlrRDOXI=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
+github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w=
+github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
+github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho=
-github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342 h1:083n9fJt2dWOpJd/X/q9Xgl5XtQLL22uSFYbzVqJssg=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
+golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
+golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6 h1:cy1ko5847T/lJ45eyg/7uLprIE/amW5IXxGtEnQdYMI=
+golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

internal/config/cli.go

@@ -0,0 +1,48 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"github.com/pelletier/go-toml"
+)
+
+// ContainerCLIConfig stores the options for the nvidia-container-cli
+type ContainerCLIConfig struct {
+	Root string
+}
+
+// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
+	cfg := getDefaultContainerCLIConfig()
+
+	if toml == nil {
+		return cfg
+	}
+
+	cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
+
+	return cfg
+}
+
+// getDefaultContainerCLIConfig defines the default values for the config
+func getDefaultContainerCLIConfig() *ContainerCLIConfig {
+	c := ContainerCLIConfig{
+		Root: "",
+	}
+
+	return &c
+}

internal/config/config.go

@@ -0,0 +1,114 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/pelletier/go-toml"
+)
+
+const (
+	configOverride = "XDG_CONFIG_HOME"
+	configFilePath = "nvidia-container-runtime/config.toml"
+)
+
+var (
+	// DefaultExecutableDir specifies the default path to use for executables if they cannot be located in the path.
+	DefaultExecutableDir = "/usr/bin"
+
+	// NVIDIAContainerRuntimeHookExecutable is the executable name for the NVIDIA Container Runtime Hook
+	NVIDIAContainerRuntimeHookExecutable = "nvidia-container-runtime-hook"
+	// NVIDIAContainerToolkitExecutable is the executable name for the NVIDIA Container Toolkit (an alias for the NVIDIA Container Runtime Hook)
+	NVIDIAContainerToolkitExecutable = "nvidia-container-toolkit"
+
+	configDir = "/etc/"
+)
+
+// Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
+// Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
+type Config struct {
+	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+}
+
+// GetConfig sets up the config struct. Values are read from a toml file
+// or set via the environment.
+func GetConfig() (*Config, error) {
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		configDir = XDGConfigDir
+	}
+
+	configFilePath := path.Join(configDir, configFilePath)
+
+	tomlFile, err := os.Open(configFilePath)
+	if err != nil {
+		return getDefaultConfig(), nil
+	}
+	defer tomlFile.Close()
+
+	cfg, err := loadConfigFrom(tomlFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config values: %v", err)
+	}
+
+	return cfg, nil
+}
+
+// loadRuntimeConfigFrom reads the config from the specified Reader
+func loadConfigFrom(reader io.Reader) (*Config, error) {
+	toml, err := toml.LoadReader(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	return getConfigFrom(toml)
+}
+
+// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getConfigFrom(toml *toml.Tree) (*Config, error) {
+	cfg := getDefaultConfig()
+
+	if toml == nil {
+		return cfg, nil
+	}
+
+	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
+	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
+	runtimeConfig, err := getRuntimeConfigFrom(toml)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
+	}
+	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
+
+	return cfg, nil
+}
+
+// getDefaultConfig defines the default values for the config
+func getDefaultConfig() *Config {
+	c := Config{
+		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
+		NVIDIACTKConfig:              *getDefaultCTKConfig(),
+		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
+	}
+
+	return &c
+}

internal/config/config_test.go

@@ -0,0 +1,165 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetConfigWithCustomConfig(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// By default debug is disabled
+	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
+	testDir := filepath.Join(wd, "test")
+	filename := filepath.Join(testDir, configFilePath)
+
+	os.Setenv(configOverride, testDir)
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
+
+	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+
+	cfg, err := GetConfig()
+	require.NoError(t, err)
+	require.Equal(t, cfg.NVIDIAContainerRuntimeConfig.DebugFilePath, "/nvidia-container-toolkit.log")
+}
+
+func TestGetConfig(t *testing.T) {
+	testCases := []struct {
+		description    string
+		contents       []string
+		expectedError  error
+		expectedConfig *Config
+	}{
+		{
+			description: "empty config is default",
+			expectedConfig: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root: "",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					LogLevel:      "info",
+					Runtimes:      []string{"docker-runc", "runc"},
+					Mode:          "auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+					},
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+			},
+		},
+		{
+			description: "config options set inline",
+			contents: []string{
+				"nvidia-container-cli.root = \"/bar/baz\"",
+				"nvidia-container-runtime.debug = \"/foo/bar\"",
+				"nvidia-container-runtime.experimental = true",
+				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
+				"nvidia-container-runtime.log-level = \"debug\"",
+				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
+				"nvidia-container-runtime.mode = \"not-auto\"",
+				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
+			},
+			expectedConfig: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root: "/bar/baz",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/foo/bar",
+					LogLevel:      "debug",
+					Runtimes:      []string{"/some/runtime"},
+					Mode:          "not-auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+					},
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "/foo/bar/nvidia-ctk",
+				},
+			},
+		},
+		{
+			description: "config options set in section",
+			contents: []string{
+				"[nvidia-container-cli]",
+				"root = \"/bar/baz\"",
+				"[nvidia-container-runtime]",
+				"debug = \"/foo/bar\"",
+				"experimental = true",
+				"discover-mode = \"not-legacy\"",
+				"log-level = \"debug\"",
+				"runtimes = [\"/some/runtime\",]",
+				"mode = \"not-auto\"",
+				"[nvidia-container-runtime.modes.csv]",
+				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
+				"[nvidia-ctk]",
+				"path = \"/foo/bar/nvidia-ctk\"",
+			},
+			expectedConfig: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root: "/bar/baz",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/foo/bar",
+					LogLevel:      "debug",
+					Runtimes:      []string{"/some/runtime"},
+					Mode:          "not-auto",
+					Modes: modesConfig{
+						CSV: csvModeConfig{
+							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
+						},
+					},
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "/foo/bar/nvidia-ctk",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
+
+			cfg, err := loadConfigFrom(reader)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.EqualValues(t, tc.expectedConfig, cfg)
+		})
+	}
+}

internal/config/crio/crio.go

@@ -0,0 +1,125 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package crio
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+// LoadConfig loads the cri-o config from disk
+func LoadConfig(config string) (*toml.Tree, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return cfg, nil
+}
+
+// UpdateConfig updates the cri-o config to include the NVIDIA Container Runtime
+func UpdateConfig(config *toml.Tree, runtimeClass string, runtimePath string, setAsDefault bool) error {
+	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass}, runc)
+	}
+
+	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_path"}, runtimePath)
+	config.SetPath([]string{"crio", "runtime", "runtimes", runtimeClass, "runtime_type"}, "oci")
+
+	if setAsDefault {
+		config.SetPath([]string{"crio", "runtime", "default_runtime"}, runtimeClass)
+	}
+
+	return nil
+}
+
+// RevertConfig reverts the cri-o config to remove the NVIDIA Container Runtime
+func RevertConfig(config *toml.Tree, runtimeClass string) error {
+	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
+		if runtimeClass == runtime {
+			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
+		}
+	}
+
+	runtimeClassPath := []string{"crio", "runtime", "runtimes", runtimeClass}
+	config.DeletePath(runtimeClassPath)
+	for i := 0; i < len(runtimeClassPath); i++ {
+		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
+		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
+			if len(entry.Keys()) != 0 {
+				break
+			}
+			config.DeletePath(remainingPath)
+		}
+	}
+
+	return nil
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(config string, cfg *toml.Tree) error {
+	log.Infof("Flushing config")
+
+	output, err := cfg.ToTomlString()
+	if err != nil {
+		return fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(config)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(config)
+		if err != nil {
+			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(output)
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}

internal/config/docker/docker.go

@@ -0,0 +1,117 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// LoadConfig loads the docker config from disk
+func LoadConfig(configFilePath string) (map[string]interface{}, error) {
+	log.Infof("Loading docker config from %v", configFilePath)
+
+	info, err := os.Stat(configFilePath)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(map[string]interface{})
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return cfg, nil
+}
+
+// UpdateConfig updates the docker config to include the nvidia runtimes
+func UpdateConfig(config map[string]interface{}, runtimeName string, runtimePath string, setAsDefault bool) error {
+	// Read the existing runtimes
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	// Add / update the runtime definitions
+	runtimes[runtimeName] = map[string]interface{}{
+		"path": runtimePath,
+		"args": []string{},
+	}
+
+	// Update the runtimes definition
+	if len(runtimes) > 0 {
+		config["runtimes"] = runtimes
+	}
+
+	if setAsDefault {
+		config["default-runtime"] = runtimeName
+	}
+
+	return nil
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(cfg map[string]interface{}, configFilePath string) error {
+	log.Infof("Flushing docker config to %v", configFilePath)
+
+	output, err := json.MarshalIndent(cfg, "", "    ")
+	if err != nil {
+		return fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(configFilePath)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(configFilePath)
+		if err != nil {
+			return fmt.Errorf("unable to open %v for writing: %v", configFilePath, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(string(output))
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}

internal/config/docker/docker_test.go

@@ -0,0 +1,215 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateConfigDefaultRuntime(t *testing.T) {
+	testCases := []struct {
+		config                     map[string]interface{}
+		runtimeName                string
+		setAsDefault               bool
+		expectedDefaultRuntimeName interface{}
+	}{
+		{
+			setAsDefault:               false,
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			runtimeName:                "NAME",
+			setAsDefault:               true,
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "ALREADY_SET",
+			},
+			runtimeName:                "NAME",
+			setAsDefault:               false,
+			expectedDefaultRuntimeName: "ALREADY_SET",
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "ALREADY_SET",
+			},
+			runtimeName:                "NAME",
+			setAsDefault:               true,
+			expectedDefaultRuntimeName: "NAME",
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			if tc.config == nil {
+				tc.config = make(map[string]interface{})
+			}
+			err := UpdateConfig(tc.config, tc.runtimeName, "", tc.setAsDefault)
+			require.NoError(t, err)
+
+			defaultRuntimeName := tc.config["default-runtime"]
+			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+		})
+	}
+}
+
+func TestUpdateConfigRuntimes(t *testing.T) {
+	testCases := []struct {
+		config         map[string]interface{}
+		runtimes       map[string]string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config: map[string]interface{}{},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+				"runtime2": "/test/runtime/dir/runtime2",
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+					"runtime2": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime2",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "runtime1",
+						"args": []string{},
+					},
+				},
+			},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+				"runtime2": "/test/runtime/dir/runtime2",
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+					"runtime2": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime2",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+				},
+			},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			runtimes: map[string]string{
+				"runtime1": "/test/runtime/dir/runtime1",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"runtime1": map[string]interface{}{
+						"path": "/test/runtime/dir/runtime1",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			for runtimeName, runtimePath := range tc.runtimes {
+				err := UpdateConfig(tc.config, runtimeName, runtimePath, false)
+				require.NoError(t, err)
+			}
+
+			configContent, err := json.MarshalIndent(tc.config, "", "    ")
+			require.NoError(t, err)
+
+			expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+			require.NoError(t, err)
+
+			require.EqualValues(t, string(expectedContent), string(configContent))
+		})
+
+	}
+}

internal/config/image/capabilities.go

@@ -0,0 +1,54 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+// DriverCapability represents the possible values of NVIDIA_DRIVER_CAPABILITIES
+type DriverCapability string
+
+// Constants for the supported driver capabilities
+const (
+	DriverCapabilityAll      DriverCapability = "all"
+	DriverCapabilityCompat32 DriverCapability = "compat32"
+	DriverCapabilityCompute  DriverCapability = "compute"
+	DriverCapabilityDisplay  DriverCapability = "display"
+	DriverCapabilityGraphics DriverCapability = "graphics"
+	DriverCapabilityNgx      DriverCapability = "ngx"
+	DriverCapabilityUtility  DriverCapability = "utility"
+	DriverCapabilityVideo    DriverCapability = "video"
+)
+
+// DriverCapabilities represents the NVIDIA_DRIVER_CAPABILITIES set for the specified image.
+type DriverCapabilities map[DriverCapability]bool
+
+// Has check whether the specified capability is selected.
+func (c DriverCapabilities) Has(capability DriverCapability) bool {
+	if c[DriverCapabilityAll] {
+		return true
+	}
+	return c[capability]
+}
+
+// Any checks whether any of the specified capabilites are set
+func (c DriverCapabilities) Any(capabilities ...DriverCapability) bool {
+	for _, cap := range capabilities {
+		if c.Has(cap) {
+			return true
+		}
+	}
+
+	return false
+}

internal/config/image/cuda_image.go

@@ -0,0 +1,190 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"golang.org/x/mod/semver"
+)
+
+const (
+	envCUDAVersion          = "CUDA_VERSION"
+	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
+	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
+	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
+	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
+	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
+)
+
+// CUDA represents a CUDA image that can be used for GPU computing. This wraps
+// a map of environment variable to values that can be used to perform lookups
+// such as requirements.
+type CUDA map[string]string
+
+// NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
+// The process environment is read (if present) to construc the CUDA Image.
+func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
+	if spec == nil || spec.Process == nil {
+		return NewCUDAImageFromEnv(nil)
+	}
+
+	return NewCUDAImageFromEnv(spec.Process.Env)
+}
+
+// NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
+// is a list of strings of the form ENVAR=VALUE.
+func NewCUDAImageFromEnv(env []string) (CUDA, error) {
+	c := make(CUDA)
+
+	for _, e := range env {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid environment variable: %v", e)
+		}
+		c[parts[0]] = parts[1]
+	}
+
+	return c, nil
+}
+
+// IsLegacy returns whether the associated CUDA image is a "legacy" image. An
+// image is considered legacy if it has a CUDA_VERSION environment variable defined
+// and no NVIDIA_REQUIRE_CUDA environment variable defined.
+func (i CUDA) IsLegacy() bool {
+	legacyCudaVersion := i[envCUDAVersion]
+	cudaRequire := i[envNVRequireCUDA]
+	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
+}
+
+// GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
+// variables.
+func (i CUDA) GetRequirements() ([]string, error) {
+	// TODO: We need not process this if disable require is set, but this will be done
+	// in a single follow-up to ensure that the behavioural change is accurately captured.
+	// if i.HasDisableRequire() {
+	// 	return nil, nil
+	// }
+
+	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
+	var requirements []string
+	for name, value := range i {
+		if strings.HasPrefix(name, envNVRequirePrefix) && !strings.HasPrefix(name, envNVRequireJetpack) {
+			requirements = append(requirements, value)
+		}
+	}
+	if i.IsLegacy() {
+		v, err := i.legacyVersion()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get version: %v", err)
+		}
+		cudaRequire := fmt.Sprintf("cuda>=%s", v)
+		requirements = append(requirements, cudaRequire)
+	}
+	return requirements, nil
+}
+
+// HasDisableRequire checks for the value of the NVIDIA_DISABLE_REQUIRE. If set
+// to a valid (true) boolean value this can be used to disable the requirement checks
+func (i CUDA) HasDisableRequire() bool {
+	if disable, exists := i[envNVDisableRequire]; exists {
+		// i.logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", disable)
+		d, _ := strconv.ParseBool(disable)
+		return d
+	}
+
+	return false
+}
+
+// DevicesFromEnvvars returns the devices requested by the image through environment variables
+func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+	// We concantenate all the devices from the specified envvars.
+	var isSet bool
+	var devices []string
+	requested := make(map[string]bool)
+	for _, envVar := range envVars {
+		if devs, ok := i[envVar]; ok {
+			isSet = true
+			for _, d := range strings.Split(devs, ",") {
+				trimmed := strings.TrimSpace(d)
+				if len(trimmed) == 0 {
+					continue
+				}
+				devices = append(devices, trimmed)
+				requested[trimmed] = true
+			}
+		}
+	}
+
+	// Environment variable unset with legacy image: default to "all".
+	if !isSet && len(devices) == 0 && i.IsLegacy() {
+		return NewVisibleDevices("all")
+	}
+
+	// Environment variable unset or empty or "void": return nil
+	if len(devices) == 0 || requested["void"] {
+		return NewVisibleDevices("void")
+	}
+
+	return NewVisibleDevices(devices...)
+}
+
+// GetDriverCapabilities returns the requested driver capabilities.
+func (i CUDA) GetDriverCapabilities() DriverCapabilities {
+	env := i[envNVDriverCapabilities]
+
+	capabilites := make(DriverCapabilities)
+	for _, c := range strings.Split(env, ",") {
+		capabilites[DriverCapability(c)] = true
+	}
+
+	return capabilites
+}
+
+func (i CUDA) legacyVersion() (string, error) {
+	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
+	if err != nil {
+		return "", fmt.Errorf("invalid CUDA version: %v", err)
+	}
+
+	return majorMinor, nil
+}
+
+func parseMajorMinorVersion(version string) (string, error) {
+	vVersion := "v" + strings.TrimPrefix(version, "v")
+
+	if !semver.IsValid(vVersion) {
+		return "", fmt.Errorf("invalid version string")
+	}
+
+	majorMinor := strings.TrimPrefix(semver.MajorMinor(vVersion), "v")
+	parts := strings.Split(majorMinor, ".")
+
+	var err error
+	_, err = strconv.ParseUint(parts[0], 10, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid major version")
+	}
+	_, err = strconv.ParseUint(parts[1], 10, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid minor version")
+	}
+	return majorMinor, nil
+}

internal/config/image/cuda_image_test.go

@@ -0,0 +1,123 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseMajorMinorVersionValid(t *testing.T) {
+	var tests = []struct {
+		version  string
+		expected string
+	}{
+		{"0", "0.0"},
+		{"8", "8.0"},
+		{"7.5", "7.5"},
+		{"9.0.116", "9.0"},
+		{"4294967295.4294967295.4294967295", "4294967295.4294967295"},
+		{"v11.6", "11.6"},
+	}
+	for _, c := range tests {
+		t.Run(c.version, func(t *testing.T) {
+			version, err := parseMajorMinorVersion(c.version)
+
+			require.NoError(t, err)
+			require.Equal(t, c.expected, version)
+		})
+	}
+}
+
+func TestParseMajorMinorVersionInvalid(t *testing.T) {
+	var tests = []string{
+		"foo",
+		"foo.5.10",
+		"9.0.116.50",
+		"9.0.116foo",
+		"7.foo",
+		"9.0.bar",
+		"9.4294967296",
+		"9.0.116.",
+		"9..0",
+		"9.",
+		".5.10",
+		"-9",
+		"+9",
+		"-9.1.116",
+		"-9.-1.-116",
+	}
+	for _, c := range tests {
+		t.Run(c, func(t *testing.T) {
+			_, err := parseMajorMinorVersion(c)
+			require.Error(t, err)
+		})
+	}
+}
+
+func TestGetRequirements(t *testing.T) {
+	testCases := []struct {
+		description  string
+		env          []string
+		requirements []string
+	}{
+		{
+			description:  "NVIDIA_REQUIRE_JETPACK is ignored",
+			env:          []string{"NVIDIA_REQUIRE_JETPACK=csv-mounts=all"},
+			requirements: nil,
+		},
+		{
+			description:  "NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS is ignored",
+			env:          []string{"NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS=base-only"},
+			requirements: nil,
+		},
+		{
+			description:  "single requirement set",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6"},
+			requirements: []string{"cuda>=11.6"},
+		},
+		{
+			description:  "requirements are concatenated requirement set",
+			env:          []string{"NVIDIA_REQUIRE_CUDA=cuda>=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
+			requirements: []string{"cuda>=11.6", "brand=tesla"},
+		},
+		{
+			description:  "legacy image",
+			env:          []string{"CUDA_VERSION=11.6"},
+			requirements: []string{"cuda>=11.6"},
+		},
+		{
+			description:  "legacy image with additional requirement",
+			env:          []string{"CUDA_VERSION=11.6", "NVIDIA_REQUIRE_BRAND=brand=tesla"},
+			requirements: []string{"cuda>=11.6", "brand=tesla"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			image, err := NewCUDAImageFromEnv(tc.env)
+			require.NoError(t, err)
+
+			requirements, err := image.GetRequirements()
+			require.NoError(t, err)
+			require.ElementsMatch(t, tc.requirements, requirements)
+
+		})
+
+	}
+}

internal/config/image/devices.go

@@ -0,0 +1,127 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package image
+
+import (
+	"strings"
+)
+
+// VisibleDevices represents the devices selected in a container image
+// through the NVIDIA_VISIBLE_DEVICES or other environment variables
+type VisibleDevices interface {
+	List() []string
+	Has(string) bool
+}
+
+var _ VisibleDevices = (*all)(nil)
+var _ VisibleDevices = (*none)(nil)
+var _ VisibleDevices = (*void)(nil)
+var _ VisibleDevices = (*devices)(nil)
+
+// NewVisibleDevices creates a VisibleDevices based on the value of the specified envvar.
+func NewVisibleDevices(envvars ...string) VisibleDevices {
+	for _, envvar := range envvars {
+		if envvar == "all" {
+			return all{}
+		}
+		if envvar == "none" {
+			return none{}
+		}
+		if envvar == "" || envvar == "void" {
+			return void{}
+		}
+	}
+
+	return newDevices(envvars...)
+}
+
+type all struct{}
+
+// List returns ["all"] for all devices
+func (a all) List() []string {
+	return []string{"all"}
+}
+
+// Has for all devices is true for any id except the empty ID
+func (a all) Has(id string) bool {
+	return id != ""
+}
+
+type none struct{}
+
+// List returns [""] for the none devices
+func (n none) List() []string {
+	return []string{""}
+}
+
+// Has for none devices is false for any id
+func (n none) Has(id string) bool {
+	return false
+}
+
+type void struct {
+	none
+}
+
+// List returns nil for the void devices
+func (v void) List() []string {
+	return nil
+}
+
+type devices struct {
+	len    int
+	lookup map[string]int
+}
+
+func newDevices(idOrCommaSeparated ...string) devices {
+	lookup := make(map[string]int)
+
+	i := 0
+	for _, commaSeparated := range idOrCommaSeparated {
+		for _, id := range strings.Split(commaSeparated, ",") {
+			lookup[id] = i
+			i++
+		}
+	}
+
+	d := devices{
+		len:    i,
+		lookup: lookup,
+	}
+	return d
+}
+
+// List returns the list of requested devices
+func (d devices) List() []string {
+	list := make([]string, d.len)
+
+	for id, i := range d.lookup {
+		list[i] = id
+	}
+
+	return list
+}
+
+// Has checks whether the specified ID is in the set of requested devices
+func (d devices) Has(id string) bool {
+	if id == "" {
+		return false
+	}
+
+	_, exist := d.lookup[id]
+	return exist
+}

internal/config/runtime.go

@@ -0,0 +1,101 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"fmt"
+
+	"github.com/pelletier/go-toml"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	dockerRuncExecutableName = "docker-runc"
+	runcExecutableName       = "runc"
+
+	auto = "auto"
+)
+
+// RuntimeConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeConfig struct {
+	DebugFilePath string `toml:"debug"`
+	// LogLevel defines the logging level for the application
+	LogLevel string `toml:"log-level"`
+	// Runtimes defines the candidates for the low-level runtime
+	Runtimes []string    `toml:"runtimes"`
+	Mode     string      `toml:"mode"`
+	Modes    modesConfig `toml:"modes"`
+}
+
+// modesConfig defines (optional) per-mode configs
+type modesConfig struct {
+	CSV csvModeConfig `toml:"csv"`
+	CDI cdiModeConfig `toml:"cdi"`
+}
+
+type cdiModeConfig struct {
+	// SpecDirs allows for the default spec dirs for CDI to be overridden
+	SpecDirs []string `toml:"spec-dirs"`
+}
+
+type csvModeConfig struct {
+	MountSpecPath string `toml:"mount-spec-path"`
+}
+
+// dummy allows us to unmarshal only a RuntimeConfig from a *toml.Tree
+type dummy struct {
+	Runtime RuntimeConfig `toml:"nvidia-container-runtime"`
+}
+
+// getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getRuntimeConfigFrom(toml *toml.Tree) (*RuntimeConfig, error) {
+	cfg := GetDefaultRuntimeConfig()
+
+	if toml == nil {
+		return cfg, nil
+	}
+
+	d := dummy{
+		Runtime: *cfg,
+	}
+
+	if err := toml.Unmarshal(&d); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
+	}
+
+	return &d.Runtime, nil
+}
+
+// GetDefaultRuntimeConfig defines the default values for the config
+func GetDefaultRuntimeConfig() *RuntimeConfig {
+	c := RuntimeConfig{
+		DebugFilePath: "/dev/null",
+		LogLevel:      logrus.InfoLevel.String(),
+		Runtimes: []string{
+			dockerRuncExecutableName,
+			runcExecutableName,
+		},
+		Mode: auto,
+		Modes: modesConfig{
+			CSV: csvModeConfig{
+				MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
+			},
+		},
+	}
+
+	return &c
+}

internal/config/toolkit-cli.go

@@ -0,0 +1,46 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import "github.com/pelletier/go-toml"
+
+// CTKConfig stores the config options for the NVIDIA Container Toolkit CLI (nvidia-ctk)
+type CTKConfig struct {
+	Path string `toml:"path"`
+}
+
+// getCTKConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getCTKConfigFrom(toml *toml.Tree) *CTKConfig {
+	cfg := getDefaultCTKConfig()
+
+	if toml == nil {
+		return cfg
+	}
+
+	cfg.Path = toml.GetDefault("nvidia-ctk.path", cfg.Path).(string)
+
+	return cfg
+}
+
+// getDefaultCTKConfig defines the default values for the config
+func getDefaultCTKConfig() *CTKConfig {
+	c := CTKConfig{
+		Path: "nvidia-ctk",
+	}
+
+	return &c
+}

internal/cuda/cuda.go

@@ -0,0 +1,137 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package cuda
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/go-nvml/pkg/dl"
+)
+
+/*
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
+
+#ifdef _WIN32
+#define CUDAAPI __stdcall
+#else
+#define CUDAAPI
+#endif
+
+typedef int CUdevice;
+
+typedef enum CUdevice_attribute_enum {
+    CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75,
+    CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR = 76
+} CUdevice_attribute;
+
+typedef enum cudaError_enum {
+	CUDA_SUCCESS = 0
+} CUresult;
+
+CUresult CUDAAPI cuInit(unsigned int Flags);
+CUresult CUDAAPI cuDriverGetVersion(int *driverVersion);
+CUresult CUDAAPI cuDeviceGet(CUdevice *device, int ordinal);
+CUresult CUDAAPI cuDeviceGetAttribute(int *pi, CUdevice_attribute attrib, CUdevice dev);
+*/
+import "C"
+
+const (
+	libraryName      = "libcuda.so.1"
+	libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+)
+
+// cuda stores a reference the cuda dynamic library
+var lib *dl.DynamicLibrary
+
+// Version returns the CUDA version of the driver as a string or an error if this
+// cannot be determined.
+func Version() (string, error) {
+	lib, err := load()
+	if err != nil {
+		return "", err
+	}
+	defer lib.Close()
+
+	if err := lib.Lookup("cuDriverGetVersion"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+
+	var version C.int
+	if result := C.cuDriverGetVersion(&version); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA version: result=%v", result)
+	}
+
+	major := version / 1000
+	minor := version % 100 / 10
+
+	return fmt.Sprintf("%d.%d", major, minor), nil
+}
+
+// ComputeCapability returns the CUDA compute capability of a device with the specified index as a string
+// or an error if this cannot be determined.
+func ComputeCapability(index int) (string, error) {
+	lib, err := load()
+	if err != nil {
+		return "", err
+	}
+	defer lib.Close()
+
+	if err := lib.Lookup("cuInit"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+	if err := lib.Lookup("cuDeviceGet"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+	if err := lib.Lookup("cuDeviceGetAttribute"); err != nil {
+		return "", fmt.Errorf("failed to lookup symbol: %v", err)
+	}
+
+	if result := C.cuInit(C.uint(0)); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to initialize CUDA: result=%v", result)
+	}
+
+	var device C.CUdevice
+	// NOTE: We only query the first device
+	if result := C.cuDeviceGet(&device, C.int(index)); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA device %v: result=%v", 0, result)
+	}
+
+	var major C.int
+	if result := C.cuDeviceGetAttribute(&major, C.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA compute capability major for device %v : result=%v", 0, result)
+	}
+
+	var minor C.int
+	if result := C.cuDeviceGetAttribute(&minor, C.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device); result != C.CUDA_SUCCESS {
+		return "", fmt.Errorf("failed to get CUDA compute capability minor for device %v: result=%v", 0, result)
+	}
+
+	return fmt.Sprintf("%d.%d", major, minor), nil
+}
+
+func load() (*dl.DynamicLibrary, error) {
+	lib := dl.New(libraryName, libraryLoadFlags)
+	if lib == nil {
+		return nil, fmt.Errorf("error instantiating DynamicLibrary for CUDA")
+	}
+	err := lib.Open()
+	if err != nil {
+		return nil, fmt.Errorf("error opening DynamicLibrary for CUDA: %v", err)
+	}
+
+	return lib, nil
+}

internal/discover/char_devices.go

@@ -0,0 +1,65 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
+func NewCharDeviceDiscoverer(logger *logrus.Logger, devices []string, root string) Discover {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(root),
+	)
+
+	return NewDeviceDiscoverer(logger, locator, root, devices)
+}
+
+// NewDeviceDiscoverer creates a discoverer which locates the specified set of device nodes using the specified locator.
+func NewDeviceDiscoverer(logger *logrus.Logger, locator lookup.Locator, root string, devices []string) Discover {
+	m := NewMounts(logger, locator, root, devices).(*mounts)
+
+	return (*charDevices)(m)
+}
+
+// Mounts returns the discovered mounts for the charDevices.
+// Since this explicitly specifies a device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices.
+// Here the device nodes are first discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	devicesAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range devicesAsMounts {
+		devices = append(devices, Device(mount))
+	}
+
+	return devices, nil
+}

internal/discover/char_devices_test.go

@@ -0,0 +1,83 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCharDevices(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description          string
+		input                *charDevices
+		expectedMounts       []Mount
+		expectedMountsError  error
+		expectedDevicesError error
+		expectedDevices      []Device
+	}{
+		{
+			description: "dev mounts are empty",
+			input: (*charDevices)(
+				&mounts{
+					lookup: &lookup.LocatorMock{
+						LocateFunc: func(string) ([]string, error) {
+							return []string{"located"}, nil
+						},
+					},
+					required: []string{"required"},
+				},
+			),
+			expectedDevices: []Device{{Path: "located", HostPath: "located"}},
+		},
+		{
+			description:          "dev devices returns error for nil lookup",
+			input:                &charDevices{},
+			expectedDevicesError: fmt.Errorf("no lookup defined"),
+		},
+	}
+
+	for _, tc := range testCases {
+		logHook.Reset()
+
+		t.Run(tc.description, func(t *testing.T) {
+			tc.input.logger = logger
+
+			mounts, err := tc.input.Mounts()
+			if tc.expectedMountsError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedMounts, mounts)
+
+			devices, err := tc.input.Devices()
+			if tc.expectedDevicesError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedDevices, devices)
+		})
+	}
+}

internal/discover/csv.go

@@ -0,0 +1,107 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
+// The constructed discoverer is comprised of a list, with each element in the list being associated with a
+// single CSV files.
+func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
+	if len(files) == 0 {
+		logger.Warnf("No CSV files specified")
+		return None{}, nil
+	}
+
+	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
+	locators := map[csv.MountSpecType]lookup.Locator{
+		csv.MountSpecDev: lookup.NewCharDeviceLocator(lookup.WithLogger(logger), lookup.WithRoot(root)),
+		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
+		// Libraries and symlinks are handled in the same way
+		csv.MountSpecLib: symlinkLocator,
+		csv.MountSpecSym: symlinkLocator,
+	}
+
+	var mountSpecs []*csv.MountSpec
+	for _, filename := range files {
+		targets, err := loadCSVFile(logger, filename)
+		if err != nil {
+			logger.Warnf("Skipping CSV file %v: %v", filename, err)
+			continue
+		}
+		mountSpecs = append(mountSpecs, targets...)
+	}
+
+	return newFromMountSpecs(logger, locators, root, mountSpecs)
+}
+
+// loadCSVFile loads the specified CSV file and returns the list of mount specs
+func loadCSVFile(logger *logrus.Logger, filename string) ([]*csv.MountSpec, error) {
+	// Create a discoverer for each file-kind combination
+	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CSV file: %v", err)
+	}
+	if len(targets) == 0 {
+		return nil, fmt.Errorf("CSV file is empty")
+	}
+
+	return targets, nil
+}
+
+// newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
+// A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
+func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, root string, targets []*csv.MountSpec) (Discover, error) {
+	if len(targets) == 0 {
+		return &None{}, nil
+	}
+
+	var discoverers []Discover
+	var mountSpecTypes []csv.MountSpecType
+	candidatesByType := make(map[csv.MountSpecType][]string)
+	for _, t := range targets {
+		if _, exists := candidatesByType[t.Type]; !exists {
+			mountSpecTypes = append(mountSpecTypes, t.Type)
+		}
+		candidatesByType[t.Type] = append(candidatesByType[t.Type], t.Path)
+	}
+
+	for _, t := range mountSpecTypes {
+		locator, exists := locators[t]
+		if !exists {
+			return nil, fmt.Errorf("no locator defined for '%v'", t)
+		}
+
+		var m Discover
+		switch t {
+		case csv.MountSpecDev:
+			m = NewDeviceDiscoverer(logger, locator, root, candidatesByType[t])
+		default:
+			m = NewMounts(logger, locator, root, candidatesByType[t])
+		}
+		discoverers = append(discoverers, m)
+
+	}
+
+	return &list{discoverers: discoverers}, nil
+}

internal/discover/csv/csv.go

@@ -0,0 +1,131 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// DefaultMountSpecPath is default location of CSV files that define the modifications required to the OCI spec
+	DefaultMountSpecPath = "/etc/nvidia-container-runtime/host-files-for-container.d"
+)
+
+// GetFileList returns the (non-recursive) list of CSV files in the specified
+// folder
+func GetFileList(root string) ([]string, error) {
+	contents, err := os.ReadDir(root)
+	if err != nil && errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to read the contents of %v: %v", root, err)
+	}
+
+	var csvFilePaths []string
+	for _, c := range contents {
+		if c.IsDir() {
+			continue
+		}
+		if c.Name() == ".csv" {
+			continue
+		}
+		ext := strings.ToLower(filepath.Ext(c.Name()))
+		if ext != ".csv" {
+			continue
+		}
+
+		csvFilePaths = append(csvFilePaths, filepath.Join(root, c.Name()))
+	}
+
+	return csvFilePaths, nil
+}
+
+// BaseFilesOnly filters out non-base CSV files from the list of CSV files.
+func BaseFilesOnly(filenames []string) []string {
+	filter := map[string]bool{
+		"l4t.csv":     true,
+		"drivers.csv": true,
+		"devices.csv": true,
+	}
+
+	var selected []string
+	for _, file := range filenames {
+		base := filepath.Base(file)
+		if filter[base] {
+			selected = append(selected, file)
+		}
+	}
+
+	return selected
+}
+
+// Parser specifies an interface for parsing MountSpecs
+type Parser interface {
+	Parse() ([]*MountSpec, error)
+}
+
+type csv struct {
+	logger   *logrus.Logger
+	filename string
+}
+
+// NewCSVFileParser creates a new parser for reading MountSpecs from the specified CSV file
+func NewCSVFileParser(logger *logrus.Logger, filename string) Parser {
+	p := csv{
+		logger:   logger,
+		filename: filename,
+	}
+
+	return &p
+}
+
+// Parse parses the csv file and returns a list of MountSpecs in the file
+func (p csv) Parse() ([]*MountSpec, error) {
+	reader, err := os.Open(p.filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %v for reading: %v", p.filename, err)
+	}
+	defer reader.Close()
+
+	return p.parseFromReader(reader), nil
+}
+
+// parseFromReader parses the specified file and returns a list of required jetson mounts
+func (p csv) parseFromReader(reader io.Reader) []*MountSpec {
+	var targets []*MountSpec
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+		target, err := NewMountSpecFromLine(line)
+		if err != nil {
+			p.logger.Debugf("Skipping invalid mount spec '%v': %v", line, err)
+			continue
+		}
+		targets = append(targets, target)
+	}
+
+	return targets
+}

internal/discover/csv/csv_test.go

@@ -0,0 +1,83 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetFileList(t *testing.T) {
+	moduleRoot, _ := test.GetModuleRoot()
+
+	testCases := []struct {
+		description   string
+		root          string
+		files         []string
+		expectedError error
+	}{
+		{
+			description: "returns list of CSV files",
+			root:        "test/input/csv_samples/",
+			files: []string{
+				"jetson.csv",
+				"simple_wrong.csv",
+				"simple.csv",
+				"spaced.csv",
+			},
+		},
+		{
+			description: "handles empty folder",
+			root:        "test/input/csv_samples/empty",
+		},
+		{
+			description: "handles non-existent folder",
+			root:        "test/input/csv_samples/NONEXISTENT",
+		},
+		{
+			description: "handles non-existent folder root",
+			root:        "/NONEXISTENT/test/input/csv_samples/",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			root := filepath.Join(moduleRoot, tc.root)
+			files, err := GetFileList(root)
+
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				require.Empty(t, files)
+				return
+			}
+
+			require.NoError(t, err)
+
+			var foundFiles []string
+			for _, f := range files {
+				require.Equal(t, root, filepath.Dir(f))
+				require.Equal(t, ".csv", filepath.Ext(f))
+				foundFiles = append(foundFiles, filepath.Base(f))
+			}
+
+			require.ElementsMatch(t, tc.files, foundFiles)
+		})
+	}
+}

internal/discover/csv/mount_spec.go

@@ -0,0 +1,74 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"fmt"
+	"strings"
+)
+
+// MountSpecType defines the mount types allowed in a CSV file
+type MountSpecType string
+
+const (
+	// MountSpecDev is used for character devices
+	MountSpecDev = MountSpecType("dev")
+	// MountSpecDir is used for directories
+	MountSpecDir = MountSpecType("dir")
+	// MountSpecLib is used for libraries or regular files
+	MountSpecLib = MountSpecType("lib")
+	// MountSpecSym is used for symlinks.
+	MountSpecSym = MountSpecType("sym")
+)
+
+// MountSpec represents a Jetson mount consisting of a type and a path.
+type MountSpec struct {
+	Type MountSpecType
+	Path string
+}
+
+// NewMountSpecFromLine parses the specified line and returns the MountSpec or an error if the line is malformed
+func NewMountSpecFromLine(line string) (*MountSpec, error) {
+	parts := strings.SplitN(strings.TrimSpace(line), ",", 2)
+	if len(parts) < 2 {
+		return nil, fmt.Errorf("failed to parse line: %v", line)
+	}
+	mountType := strings.TrimSpace(parts[0])
+	path := strings.TrimSpace(parts[1])
+
+	return NewMountSpec(mountType, path)
+}
+
+// NewMountSpec creates a MountSpec with the specified type and path. An error is returned if the type is invalid.
+func NewMountSpec(mountType string, path string) (*MountSpec, error) {
+	mt := MountSpecType(mountType)
+	switch mt {
+	case MountSpecDev, MountSpecLib, MountSpecSym, MountSpecDir:
+	default:
+		return nil, fmt.Errorf("unexpected mount type: %v", mt)
+	}
+	if path == "" {
+		return nil, fmt.Errorf("invalid path: %v", path)
+	}
+
+	mount := MountSpec{
+		Type: mt,
+		Path: path,
+	}
+
+	return &mount, nil
+}

internal/discover/csv/mount_spec_test.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewMountSpecFromLine(t *testing.T) {
+	parseError := fmt.Errorf("failed to parse line")
+	unexpectedError := fmt.Errorf("unexpected mount type")
+
+	testCases := []struct {
+		line          string
+		expectedError error
+		expectedValue MountSpec
+	}{
+		{
+			line:          "",
+			expectedError: parseError,
+		},
+		{
+			line:          "\t",
+			expectedError: parseError,
+		},
+		{
+			line:          ",",
+			expectedError: parseError,
+		},
+		{
+			line:          "dev,",
+			expectedError: parseError,
+		},
+		{
+			line: "dev ,/a/path",
+			expectedValue: MountSpec{
+				Path: "/a/path",
+				Type: "dev",
+			},
+		},
+		{
+			line: "dev ,/a/path,with,commas",
+			expectedValue: MountSpec{
+				Path: "/a/path,with,commas",
+				Type: "dev",
+			},
+		},
+		{
+			line:          "not-dev ,/a/path",
+			expectedError: unexpectedError,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			target, err := NewMountSpecFromLine(tc.line)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.EqualValues(t, &tc.expectedValue, target)
+		})
+	}
+}

internal/discover/csv_test.go

@@ -0,0 +1,142 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewFromMountSpec(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	locators := map[csv.MountSpecType]lookup.Locator{
+		"dev": &lookup.LocatorMock{},
+		"lib": &lookup.LocatorMock{},
+	}
+
+	testCases := []struct {
+		description        string
+		root               string
+		targets            []*csv.MountSpec
+		expectedError      error
+		expectedDiscoverer Discover
+	}{
+		{
+			description:        "empty targets returns None discoverer list",
+			expectedDiscoverer: &None{},
+		},
+		{
+			description: "unexpected locator returns error",
+			targets: []*csv.MountSpec{
+				{
+					Type: "foo",
+					Path: "bar",
+				},
+			},
+			expectedError: fmt.Errorf("no locator defined for foo"),
+		},
+		{
+			description: "creates discoverers based on type",
+			targets: []*csv.MountSpec{
+				{
+					Type: "dev",
+					Path: "dev0",
+				},
+				{
+					Type: "lib",
+					Path: "lib0",
+				},
+				{
+					Type: "dev",
+					Path: "dev1",
+				},
+			},
+			expectedDiscoverer: &list{
+				discoverers: []Discover{
+					(*charDevices)(
+						&mounts{
+							logger:   logger,
+							lookup:   locators["dev"],
+							root:     "/",
+							required: []string{"dev0", "dev1"},
+						},
+					),
+					&mounts{
+						logger:   logger,
+						lookup:   locators["lib"],
+						root:     "/",
+						required: []string{"lib0"},
+					},
+				},
+			},
+		},
+		{
+			description: "sets root",
+			targets: []*csv.MountSpec{
+				{
+					Type: "dev",
+					Path: "dev0",
+				},
+				{
+					Type: "lib",
+					Path: "lib0",
+				},
+				{
+					Type: "dev",
+					Path: "dev1",
+				},
+			},
+			root: "/some/root",
+			expectedDiscoverer: &list{
+				discoverers: []Discover{
+					(*charDevices)(
+						&mounts{
+							logger:   logger,
+							lookup:   locators["dev"],
+							root:     "/some/root",
+							required: []string{"dev0", "dev1"},
+						},
+					),
+					&mounts{
+						logger:   logger,
+						lookup:   locators["lib"],
+						root:     "/some/root",
+						required: []string{"lib0"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			discoverer, err := newFromMountSpecs(logger, locators, tc.root, tc.targets)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expectedDiscoverer, discoverer)
+		})
+	}
+}

internal/discover/discover.go

@@ -0,0 +1,51 @@
+/*
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+// Config represents the configuration options for discovery
+type Config struct {
+	Root          string
+	NvidiaCTKPath string
+}
+
+// Device represents a discovered character device.
+type Device struct {
+	HostPath string
+	Path     string
+}
+
+// Mount represents a discovered mount.
+type Mount struct {
+	HostPath string
+	Path     string
+}
+
+// Hook represents a discovered hook.
+type Hook struct {
+	Lifecycle string
+	Path      string
+	Args      []string
+}
+
+// Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+//
+//go:generate moq -stub -out discover_mock.go . Discover
+type Discover interface {
+	Devices() ([]Device, error)
+	Mounts() ([]Mount, error)
+	Hooks() ([]Hook, error)
+}

internal/discover/discover_mock.go

@@ -0,0 +1,150 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package discover
+
+import (
+	"sync"
+)
+
+// Ensure, that DiscoverMock does implement Discover.
+// If this is not the case, regenerate this file with moq.
+var _ Discover = &DiscoverMock{}
+
+// DiscoverMock is a mock implementation of Discover.
+//
+// 	func TestSomethingThatUsesDiscover(t *testing.T) {
+//
+// 		// make and configure a mocked Discover
+// 		mockedDiscover := &DiscoverMock{
+// 			DevicesFunc: func() ([]Device, error) {
+// 				panic("mock out the Devices method")
+// 			},
+// 			HooksFunc: func() ([]Hook, error) {
+// 				panic("mock out the Hooks method")
+// 			},
+// 			MountsFunc: func() ([]Mount, error) {
+// 				panic("mock out the Mounts method")
+// 			},
+// 		}
+//
+// 		// use mockedDiscover in code that requires Discover
+// 		// and then make assertions.
+//
+// 	}
+type DiscoverMock struct {
+	// DevicesFunc mocks the Devices method.
+	DevicesFunc func() ([]Device, error)
+
+	// HooksFunc mocks the Hooks method.
+	HooksFunc func() ([]Hook, error)
+
+	// MountsFunc mocks the Mounts method.
+	MountsFunc func() ([]Mount, error)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Devices holds details about calls to the Devices method.
+		Devices []struct {
+		}
+		// Hooks holds details about calls to the Hooks method.
+		Hooks []struct {
+		}
+		// Mounts holds details about calls to the Mounts method.
+		Mounts []struct {
+		}
+	}
+	lockDevices sync.RWMutex
+	lockHooks   sync.RWMutex
+	lockMounts  sync.RWMutex
+}
+
+// Devices calls DevicesFunc.
+func (mock *DiscoverMock) Devices() ([]Device, error) {
+	callInfo := struct {
+	}{}
+	mock.lockDevices.Lock()
+	mock.calls.Devices = append(mock.calls.Devices, callInfo)
+	mock.lockDevices.Unlock()
+	if mock.DevicesFunc == nil {
+		var (
+			devicesOut []Device
+			errOut     error
+		)
+		return devicesOut, errOut
+	}
+	return mock.DevicesFunc()
+}
+
+// DevicesCalls gets all the calls that were made to Devices.
+// Check the length with:
+//     len(mockedDiscover.DevicesCalls())
+func (mock *DiscoverMock) DevicesCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockDevices.RLock()
+	calls = mock.calls.Devices
+	mock.lockDevices.RUnlock()
+	return calls
+}
+
+// Hooks calls HooksFunc.
+func (mock *DiscoverMock) Hooks() ([]Hook, error) {
+	callInfo := struct {
+	}{}
+	mock.lockHooks.Lock()
+	mock.calls.Hooks = append(mock.calls.Hooks, callInfo)
+	mock.lockHooks.Unlock()
+	if mock.HooksFunc == nil {
+		var (
+			hooksOut []Hook
+			errOut   error
+		)
+		return hooksOut, errOut
+	}
+	return mock.HooksFunc()
+}
+
+// HooksCalls gets all the calls that were made to Hooks.
+// Check the length with:
+//     len(mockedDiscover.HooksCalls())
+func (mock *DiscoverMock) HooksCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockHooks.RLock()
+	calls = mock.calls.Hooks
+	mock.lockHooks.RUnlock()
+	return calls
+}
+
+// Mounts calls MountsFunc.
+func (mock *DiscoverMock) Mounts() ([]Mount, error) {
+	callInfo := struct {
+	}{}
+	mock.lockMounts.Lock()
+	mock.calls.Mounts = append(mock.calls.Mounts, callInfo)
+	mock.lockMounts.Unlock()
+	if mock.MountsFunc == nil {
+		var (
+			mountsOut []Mount
+			errOut    error
+		)
+		return mountsOut, errOut
+	}
+	return mock.MountsFunc()
+}
+
+// MountsCalls gets all the calls that were made to Mounts.
+// Check the length with:
+//     len(mockedDiscover.MountsCalls())
+func (mock *DiscoverMock) MountsCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockMounts.RLock()
+	calls = mock.calls.Mounts
+	mock.lockMounts.RUnlock()
+	return calls
+}

internal/discover/filter.go

@@ -0,0 +1,62 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/sirupsen/logrus"
+
+// Filter defines an interface for filtering discovered entities
+type Filter interface {
+	DeviceIsSelected(device Device) bool
+}
+
+// filtered represents a filtered discoverer
+type filtered struct {
+	Discover
+	logger *logrus.Logger
+	filter Filter
+}
+
+// newFilteredDisoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
+func newFilteredDisoverer(logger *logrus.Logger, applyTo Discover, filter Filter) Discover {
+	return filtered{
+		Discover: applyTo,
+		logger:   logger,
+		filter:   filter,
+	}
+}
+
+// Devices returns a filtered list of devices based on the specified filter.
+func (d filtered) Devices() ([]Device, error) {
+	devices, err := d.Discover.Devices()
+	if err != nil {
+		return nil, err
+	}
+
+	if d.filter == nil {
+		return devices, nil
+	}
+
+	var selected []Device
+	for _, device := range devices {
+		if d.filter.DeviceIsSelected(device) {
+			selected = append(selected, device)
+		}
+		d.logger.Debugf("skipping device %v", device)
+	}
+
+	return selected, nil
+}

internal/discover/gds.go

@@ -0,0 +1,80 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+type gdsDeviceDiscoverer struct {
+	None
+	logger  *logrus.Logger
+	devices Discover
+	mounts  Discover
+}
+
+// NewGDSDiscoverer creates a discoverer for GPUDirect Storage devices and mounts.
+func NewGDSDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		[]string{"/dev/nvidia-fs*"},
+		root,
+	)
+
+	udev := NewMounts(
+		logger,
+		lookup.NewDirectoryLocator(logger, root),
+		root,
+		[]string{"/run/udev"},
+	)
+
+	cufile := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{"/etc/cufile.json"},
+	)
+
+	d := gdsDeviceDiscoverer{
+		logger:  logger,
+		devices: devices,
+		mounts:  Merge(udev, cufile),
+	}
+
+	return &d, nil
+}
+
+// Devices discovers the nvidia-fs device nodes for use with GPUDirect Storage
+func (d *gdsDeviceDiscoverer) Devices() ([]Device, error) {
+	return d.devices.Devices()
+}
+
+// Mounts discovers the required mounts for GPUDirect Storage.
+// If no devices are discovered the discovered mounts are empty
+func (d *gdsDeviceDiscoverer) Mounts() ([]Mount, error) {
+	devices, err := d.Devices()
+	if err != nil || len(devices) == 0 {
+		d.logger.Debugf("No nvidia-fs devices detected; skipping detection of mounts")
+		return nil, nil
+	}
+
+	return d.mounts.Mounts()
+}

internal/discover/graphics.go

@@ -0,0 +1,264 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/drm"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// NewGraphicsDiscoverer returns the discoverer for graphics tools such as Vulkan.
+func NewGraphicsDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, cfg *Config) (Discover, error) {
+	root := cfg.Root
+
+	mounts, err := NewGraphicsMountsDiscoverer(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)
+	}
+
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, cfg)
+
+	discover := Merge(
+		Merge(drmDeviceNodes, drmByPathSymlinks),
+		mounts,
+	)
+
+	return discover, nil
+}
+
+// NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
+func NewGraphicsMountsDiscoverer(logger *logrus.Logger, root string) (Discover, error) {
+	locator, err := lookup.NewLibraryLocator(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct library locator: %v", err)
+	}
+	libraries := NewMounts(
+		logger,
+		locator,
+		root,
+		[]string{
+			"libnvidia-egl-gbm.so",
+		},
+	)
+
+	jsonMounts := NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+			lookup.WithSearchPaths("/etc", "/usr/share"),
+		),
+		root,
+		[]string{
+			"glvnd/egl_vendor.d/10_nvidia.json",
+			"vulkan/icd.d/nvidia_icd.json",
+			"vulkan/implicit_layer.d/nvidia_layers.json",
+			"egl/egl_external_platform.d/15_nvidia_gbm.json",
+			"egl/egl_external_platform.d/10_nvidia_wayland.json",
+		},
+	)
+
+	discover := Merge(
+		libraries,
+		jsonMounts,
+	)
+
+	return discover, nil
+}
+
+type drmDevicesByPath struct {
+	None
+	logger        *logrus.Logger
+	nvidiaCTKPath string
+	root          string
+	devicesFrom   Discover
+}
+
+// newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
+func newCreateDRMByPathSymlinks(logger *logrus.Logger, devices Discover, cfg *Config) Discover {
+	d := drmDevicesByPath{
+		logger:        logger,
+		nvidiaCTKPath: FindNvidiaCTK(logger, cfg.NvidiaCTKPath),
+		root:          cfg.Root,
+		devicesFrom:   devices,
+	}
+
+	return &d
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d drmDevicesByPath) Hooks() ([]Hook, error) {
+	devices, err := d.devicesFrom.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices for by-path symlinks: %v", err)
+	}
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	links, err := d.getSpecificLinkArgs(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range links {
+		args = append(args, "--link", l)
+	}
+
+	hook := CreateNvidiaCTKHook(
+		d.nvidiaCTKPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []Hook{hook}, nil
+}
+
+// getSpecificLinkArgs returns the required specic links that need to be created
+func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error) {
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[filepath.Base(d.HostPath)] = true
+	}
+
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.root),
+	)
+	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
+	if err != nil {
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
+	}
+
+	var links []string
+	for _, c := range candidates {
+		device, err := os.Readlink(c)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", c)
+			continue
+		}
+
+		if selectedDevices[filepath.Base(device)] {
+			d.logger.Debugf("adding device symlink %v -> %v", c, device)
+			links = append(links, fmt.Sprintf("%v::%v", device, c))
+		}
+	}
+
+	return links, nil
+}
+
+// newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
+func newDRMDeviceDiscoverer(logger *logrus.Logger, devices image.VisibleDevices, root string) (Discover, error) {
+	allDevices := NewDeviceDiscoverer(
+		logger,
+		lookup.NewCharDeviceLocator(
+			lookup.WithLogger(logger),
+			lookup.WithRoot(root),
+		),
+		root,
+		[]string{
+			"/dev/dri/card*",
+			"/dev/dri/renderD*",
+		},
+	)
+
+	filter, err := newDRMDeviceFilter(logger, devices, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct DRM device filter: %v", err)
+	}
+
+	// We return a discoverer that applies the DRM device filter created above to all discovered DRM device nodes.
+	d := newFilteredDisoverer(
+		logger,
+		allDevices,
+		filter,
+	)
+
+	return d, err
+}
+
+// newDRMDeviceFilter creates a filter that matches DRM devices nodes for the visible devices.
+func newDRMDeviceFilter(logger *logrus.Logger, devices image.VisibleDevices, root string) (Filter, error) {
+	gpuInformationPaths, err := proc.GetInformationFilePaths(root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read GPU information: %v", err)
+	}
+
+	var selectedBusIds []string
+	for _, f := range gpuInformationPaths {
+		info, err := proc.ParseGPUInformationFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %v: %v", f, err)
+		}
+		uuid := info[proc.GPUInfoGPUUUID]
+		busID := info[proc.GPUInfoBusLocation]
+		minor := info[proc.GPUInfoDeviceMinor]
+
+		if devices.Has(minor) || devices.Has(uuid) || devices.Has(busID) {
+			selectedBusIds = append(selectedBusIds, busID)
+		}
+	}
+
+	filter := make(selectDeviceByPath)
+	for _, busID := range selectedBusIds {
+		drmDeviceNodes, err := drm.GetDeviceNodesByBusID(busID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", busID, err)
+		}
+		for _, drmDeviceNode := range drmDeviceNodes {
+			filter[filepath.Join(drmDeviceNode)] = true
+		}
+	}
+
+	return filter, nil
+}
+
+// selectDeviceByPath is a filter that allows devices to be selected by the path
+type selectDeviceByPath map[string]bool
+
+var _ Filter = (*selectDeviceByPath)(nil)
+
+// DeviceIsSelected determines whether the device's path has been selected
+func (s selectDeviceByPath) DeviceIsSelected(device Device) bool {
+	return s[device.Path]
+}
+
+// MountIsSelected is always true
+func (s selectDeviceByPath) MountIsSelected(Mount) bool {
+	return true
+}
+
+// HookIsSelected is always true
+func (s selectDeviceByPath) HookIsSelected(Hook) bool {
+	return true
+}

internal/discover/hooks.go

@@ -0,0 +1,63 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	nvidiaCTKExecutable      = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+)
+
+// CreateNvidiaCTKHook creates a hook which invokes the NVIDIA Container CLI hook subcommand.
+func CreateNvidiaCTKHook(executable string, hookName string, additionalArgs ...string) Hook {
+	return Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      executable,
+		Args:      append([]string{filepath.Base(executable), "hook", hookName}, additionalArgs...),
+	}
+}
+
+// FindNvidiaCTK locates the nvidia-ctk executable to be used in hooks.
+// If an override is specified, this is used instead.
+func FindNvidiaCTK(logger *logrus.Logger, override string) string {
+	if override != "" {
+		logger.Debugf("Using specified NVIDIA Container Toolkit CLI path %v", override)
+		return override
+	}
+
+	lookup := lookup.NewExecutableLocator(logger, "")
+	hookPath := nvidiaCTKDefaultFilePath
+	targets, err := lookup.Locate(nvidiaCTKExecutable)
+	if err != nil {
+		logger.Warnf("Failed to locate %v: %v", nvidiaCTKExecutable, err)
+	} else if len(targets) == 0 {
+		logger.Warnf("%v not found", nvidiaCTKExecutable)
+	} else {
+		logger.Debugf("Found %v candidates: %v", nvidiaCTKExecutable, targets)
+		hookPath = targets[0]
+	}
+	logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
+
+	return hookPath
+}

internal/discover/ldconfig.go

@@ -0,0 +1,125 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
+func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
+	d := ldconfig{
+		logger:        logger,
+		mountsFrom:    mounts,
+		lookup:        lookup.NewExecutableLocator(logger, cfg.Root),
+		nvidiaCTKPath: cfg.NvidiaCTKPath,
+	}
+
+	return &d, nil
+}
+
+type ldconfig struct {
+	None
+	logger        *logrus.Logger
+	mountsFrom    Discover
+	lookup        lookup.Locator
+	nvidiaCTKPath string
+}
+
+// Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
+func (d ldconfig) Hooks() ([]Hook, error) {
+	mounts, err := d.mountsFrom.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
+	}
+	h := CreateLDCacheUpdateHook(
+		d.nvidiaCTKPath,
+		getLibraryPaths(mounts),
+	)
+	return []Hook{h}, nil
+}
+
+// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
+func CreateLDCacheUpdateHook(executable string, libraries []string) Hook {
+	var args []string
+	for _, f := range uniqueFolders(libraries) {
+		args = append(args, "--folder", f)
+	}
+
+	hook := CreateNvidiaCTKHook(
+		executable,
+		"update-ldcache",
+		args...,
+	)
+
+	return hook
+
+}
+
+// getLibraryPaths extracts the library dirs from the specified mounts
+func getLibraryPaths(mounts []Mount) []string {
+	var paths []string
+	for _, m := range mounts {
+		if !isLibName(m.Path) {
+			continue
+		}
+		paths = append(paths, m.Path)
+	}
+	return paths
+}
+
+// isLibName checks if the specified filename is a library (i.e. ends in `.so*`)
+func isLibName(filename string) bool {
+
+	base := filepath.Base(filename)
+
+	isLib, err := filepath.Match("lib?*.so*", base)
+	if !isLib || err != nil {
+		return false
+	}
+
+	parts := strings.Split(base, ".so")
+	if len(parts) == 1 {
+		return true
+	}
+
+	return parts[len(parts)-1] == "" || strings.HasPrefix(parts[len(parts)-1], ".")
+}
+
+// uniqueFolders returns the unique set of folders for the specified files
+func uniqueFolders(libraries []string) []string {
+	var paths []string
+	checked := make(map[string]bool)
+
+	for _, l := range libraries {
+		dir := filepath.Dir(l)
+		if dir == "" {
+			continue
+		}
+		if checked[dir] {
+			continue
+		}
+		checked[dir] = true
+		paths = append(paths, dir)
+	}
+	return paths
+}