Merge branch 'add-experimental-config' into 'master'

Add commented experimental option to config files

See merge request nvidia/container-toolkit/container-toolkit!131

.common-ci.yml

@@ -18,9 +18,9 @@ default:
       command: ["--experimental"]
 
 variables:
-  IMAGE: "${CI_REGISTRY_IMAGE}"
-  IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
+  GIT_SUBMODULE_STRATEGY: recursive
   BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
+  BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
   - image
@@ -28,70 +28,225 @@ stages:
   - go-checks
   - go-build
   - unit-tests
-  - build
-  - build-long
+  - package-build
+  - image-build
+  - test
   - scan
   - release
 
-build-dev-image:
-  stage: image
-  script:
-    - apk --no-cache add make bash
-    - make .build-image
+# Define the distribution targets
+.dist-amazonlinux2:
+  variables:
+    DIST: amazonlinux2
+
+.dist-centos7:
+  variables:
+    DIST: centos7
+    CVE_UPDATES: "cyrus-sasl-lib"
+
+.dist-centos8:
+  variables:
+    DIST: centos8
+    CVE_UPDATES: "cyrus-sasl-lib"
+
+.dist-debian10:
+  variables:
+    DIST: debian10
+
+.dist-debian9:
+  variables:
+    DIST: debian9
+
+.dist-opensuse-leap15.1:
+  variables:
+    DIST: opensuse-leap15.1
+
+.dist-ubi8:
+  variables:
+    DIST: ubi8
+    CVE_UPDATES: "cyrus-sasl-lib"
+
+.dist-ubuntu16.04:
+  variables:
+    DIST: ubuntu16.04
+
+.dist-ubuntu18.04:
+  variables:
+    DIST: ubuntu18.04
+    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+
+.dist-ubuntu20.04:
+  variables:
+    DIST: ubuntu20.04
+    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
+
+.dist-packaging:
+  variables:
+    DIST: packaging
+
+# Define architecture targets
+.arch-aarch64:
+  variables:
+    ARCH: aarch64
+
+.arch-amd64:
+  variables:
+    ARCH: amd64
+
+.arch-arm64:
+  variables:
+    ARCH: arm64
+
+.arch-ppc64le:
+  variables:
+    ARCH: ppc64le
+
+.arch-x86_64:
+  variables:
+    ARCH: x86_64
+
+# Define the platform targets
+.platform-amd64:
+  variables:
+    PLATFORM: linux/amd64
+
+.platform-arm64:
+  variables:
+    PLATFORM: linux/arm64
+
+# Define test helpers
+.integration:
+  stage: test
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache make bash jq
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - make .push-build-image
-
-.requires-build-image:
-  image: "${BUILDIMAGE}"
-
-.go-check:
-  extends:
-    - .requires-build-image
-  stage: go-checks
-
-fmt:
-  extends:
-    - .go-check
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
   script:
-    - make assert-fmt
+    - make -f build/container/Makefile test-${DIST}
 
-vet:
+# Define the test targets
+test-packaging:
   extends:
-    - .go-check
-  script:
-    - make vet
+    - .integration
+    - .dist-packaging
+  needs:
+    - image-packaging
 
-lint:
+# Download the regctl binary for use in the release steps
+.regctl-setup:
+  before_script:
+    - export REGCTL_VERSION=v0.3.10
+    - apk add --no-cache curl
+    - mkdir -p bin
+    - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
+    - chmod a+x bin/regctl
+    - export PATH=$(pwd)/bin:${PATH}
+
+# .release forms the base of the deployment jobs which push images to the CI registry.
+# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
+# target os.
+.release:
+  stage: release
+  variables:
+    # Define the source image for the release
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    # OUT_IMAGE_VERSION is overridden for external releases
+    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - !reference [.regctl-setup, before_script]
+
+    # We ensure that the OUT_IMAGE_VERSION is set
+    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
+
+    # In the case where we are deploying a different version to the CI_COMMIT_SHA, we
+    # need to tag the image.
+    # Note: a leading 'v' is stripped from the version if present
+    - apk add --no-cache make bash
+  script:
+    # Log in to the "output" registry, tag the image and push the image
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
+    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"'
+    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"'
+
+    # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
+    # Target
+    - make -f build/container/Makefile push-${DIST}
+
+# Define a staging release step that pushes an image to an internal "staging" repository
+# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
+# outside of the release process.
+.release:staging:
   extends:
-    - .go-check
-  script:
-    - make lint
-  allow_failure: true
+    - .release
+  variables:
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
 
-ineffassign:
+# Define an external release step that pushes an image to an external repository.
+# This includes a devlopment image off master.
+.release:external:
   extends:
-    - .go-check
-  script:
-    - make ineffassign
-  allow_failure: true
+    - .release
+  rules:
+    - if: $CI_COMMIT_TAG
+      variables:
+        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
+    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
+      variables:
+        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
 
-misspell:
+# Define the release jobs
+release:staging-centos7:
   extends:
-    - .go-check
-  script:
-    - make misspell
+    - .release:staging
+    - .dist-centos7
+  needs:
+    - image-centos7
 
-go-build:
+release:staging-centos8:
   extends:
-    - .requires-build-image
-  stage: go-build
-  script:
-    - make build
+    - .release:staging
+    - .dist-centos8
+  needs:
+    - image-centos8
 
-unit-tests:
+release:staging-ubi8:
   extends:
-    - .requires-build-image
-  stage: unit-tests
-  script:
-    - make coverage
+    - .release:staging
+    - .dist-ubi8
+  needs:
+    - image-ubi8
 
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - test-toolkit-ubuntu18.04
+    - test-containerd-ubuntu18.04
+    - test-crio-ubuntu18.04
+    - test-docker-ubuntu18.04
+
+release:staging-ubuntu20.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu20.04
+  needs:
+    - test-toolkit-ubuntu20.04
+    - test-containerd-ubuntu20.04
+    - test-crio-ubuntu20.04
+    - test-docker-ubuntu20.04
+
+release:staging-packaging:
+  extends:
+    - .release:staging
+    - .dist-packaging
+  needs:
+    - test-packaging

.dockerignore

@@ -1,2 +1,2 @@
 .git
-dist
+/shared-*

.gitignore

@@ -1,7 +1,9 @@
 dist
 *.swp
 *.swo
-/coverage.out
+/coverage.out*
 /test/output/
 /nvidia-container-runtime
 /nvidia-container-toolkit
+/nvidia-ctk
+/shared-*

.gitlab-ci.yml

@@ -15,107 +15,369 @@
 include:
   - .common-ci.yml
 
-.build-setup:
+build-dev-image:
+  stage: image
+  script:
+    - apk --no-cache add make bash
+    - make .build-image
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - make .push-build-image
+
+.requires-build-image:
+  image: "${BUILDIMAGE}"
+
+.go-check:
+  extends:
+    - .requires-build-image
+  stage: go-checks
+
+fmt:
+  extends:
+    - .go-check
+  script:
+    - make assert-fmt
+
+vet:
+  extends:
+    - .go-check
+  script:
+    - make vet
+
+lint:
+  extends:
+    - .go-check
+  script:
+    - make lint
+  allow_failure: true
+
+ineffassign:
+  extends:
+    - .go-check
+  script:
+    - make ineffassign
+  allow_failure: true
+
+misspell:
+  extends:
+    - .go-check
+  script:
+    - make misspell
+
+go-build:
+  extends:
+    - .requires-build-image
+  stage: go-build
+  script:
+    - make build
+
+unit-tests:
+  extends:
+    - .requires-build-image
+  stage: unit-tests
+  script:
+    - make coverage
+
+# Define the package build helpers
+.multi-arch-build:
   before_script:
-  - apk update
-  - apk upgrade
-  - apk add coreutils build-base sed git bash make
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
-
-# build-one jobs build packages for a single OS / ARCH combination.
-#
-# They are run during the first stage of the pipeline as a smoke test to ensure
-# that we can successfully build packages on all of our architectures for a
-# single OS. They are triggered on any change to an MR. No artifacts are
-# produced as part of build-one jobs.
-.build-one-setup:
-  extends:
-    - .build-setup
-  stage: build
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-
-# build-all jobs build packages for every OS / ARCH combination we support.
-#
-# They are run under two conditions:
-# 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0)
-# 2) Manually by a reviewer just before merging a MR.
-#
-# Unlike build-one jobs, it takes a long time to build the full suite
-# OS / ARCH combinations, so this is optimized to only run once per MR
-# (assuming it all passes). A full set of artifacts including the packages
-# built for each OS / ARCH are produced as a result of these jobs.
-.build-all-setup:
-  extends:
-    - .build-setup
-  stage: build-long
-  timeout: 2h 30m
-  rules:
-    - if: $CI_COMMIT_TAG
-      when: always
-    - if: $CI_MERGE_REQUEST_ID
-      when: always
+    - apk add --no-cache coreutils build-base sed git bash make
+    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
 
+.package-artifacts:
   variables:
-    ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}"
-    ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}"
-    DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_DIR}"
+    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
+    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
+    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
+
+.package-build:
+  extends:
+    - .multi-arch-build
+    - .package-artifacts
+  stage: package-build
+  timeout: 2h 30m
+  script:
+    - ./scripts/release.sh ${DIST}-${ARCH}
 
   artifacts:
     name: ${ARTIFACTS_NAME}
     paths:
-      - ${ARTIFACTS_DIR}
+      - ${ARTIFACTS_ROOT}
 
-# The full set of build-one jobs organizes to build
-# ubuntu18.04 in parallel on each of our supported ARCHs.
-build-one-amd64:
+# Define the package build targets
+package-amazonlinux2-aarch64:
   extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-amd64
-  rules:
-    - when: always
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-aarch64
 
-build-one-ppc64le:
+package-amazonlinux2-x86_64:
   extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-ppc64le
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-x86_64
 
-build-one-arm64:
+package-centos7-ppc64le:
   extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-arm64
+    - .package-build
+    - .dist-centos7
+    - .arch-ppc64le
 
-# The full set of build-all jobs organized to
-# have builds for each ARCH run in parallel.
-build-all-amd64:
+package-centos7-x86_64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-amd64
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
 
-build-all-x86_64:
+package-centos8-aarch64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-x86_64
+    - .package-build
+    - .dist-centos8
+    - .arch-aarch64
 
-build-all-ppc64le:
+package-centos8-ppc64le:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-ppc64le
+    - .package-build
+    - .dist-centos8
+    - .arch-ppc64le
 
-build-all-arm64:
+package-centos8-x86_64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-arm64
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
 
-build-all-aarch64:
+package-debian10-amd64:
   extends:
-    - .build-all-setup
+    - .package-build
+    - .dist-debian10
+    - .arch-amd64
+
+package-debian9-amd64:
+  extends:
+    - .package-build
+    - .dist-debian9
+    - .arch-amd64
+
+package-opensuse-leap15.1-x86_64:
+  extends:
+    - .package-build
+    - .dist-opensuse-leap15.1
+    - .arch-x86_64
+
+package-ubuntu16.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-amd64
+
+package-ubuntu16.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-ppc64le
+
+package-ubuntu18.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+package-ubuntu18.04-arm64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+package-ubuntu18.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
+.buildx-setup:
+  before_script:
+    -  export BUILDX_VERSION=v0.6.3
+    -  apk add --no-cache curl
+    -  mkdir -p ~/.docker/cli-plugins
+    -  curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
+    -  chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+    -  docker buildx create --use --platform=linux/amd64,linux/arm64
+
+    -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
+
+# Define the image build targets
+.image-build:
+  stage: image-build
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PUSH_ON_BUILD: "true"
+  before_script:
+    - !reference [.buildx-setup, before_script]
+
+    - apk add --no-cache bash make
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
-    - make docker-aarch64
+    - make -f build/container/Makefile build-${DIST}
+
+image-centos7:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos7
+  needs:
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+
+image-centos8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos8
+  needs:
+    - package-centos8-aarch64
+    - package-centos8-x86_64
+    - package-centos8-ppc64le
+
+image-ubi8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubi8
+  needs:
+    # Note: The ubi8 image uses the centos8 packages
+    - package-centos8-aarch64
+    - package-centos8-x86_64
+    - package-centos8-ppc64le
+
+image-ubuntu18.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu18.04
+  needs:
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
+image-ubuntu20.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu20.04
+  needs:
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-packaging
+  needs:
+    - package-amazonlinux2-aarch64
+    - package-amazonlinux2-x86_64
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+    - package-centos8-aarch64
+    - package-centos8-ppc64le
+    - package-centos8-x86_64
+    - package-debian10-amd64
+    - package-debian9-amd64
+    - package-opensuse-leap15.1-x86_64
+    - package-ubuntu16.04-amd64
+    - package-ubuntu16.04-ppc64le
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
+# Define publish test helpers
+.test:toolkit:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "toolkit"
+
+.test:docker:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "docker"
+
+.test:containerd:
+  # TODO: The containerd tests fail due to issues with SIGHUP.
+  # Until this is resolved with retry up to twice and allow failure here.
+  retry: 2
+  allow_failure: true
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "containerd"
+
+.test:crio:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "crio"
+
+# Define the test targets
+test-toolkit-ubuntu18.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-containerd-ubuntu18.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-crio-ubuntu18.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-docker-ubuntu18.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-toolkit-ubuntu20.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu20.04
+  needs:
+    - image-ubuntu20.04
+
+test-containerd-ubuntu20.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu20.04
+  needs:
+    - image-ubuntu20.04
+
+test-crio-ubuntu20.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu20.04
+  needs:
+    - image-ubuntu20.04
+
+test-docker-ubuntu20.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu20.04
+  needs:
+    - image-ubuntu20.04
+

.nvidia-ci.yml

@@ -0,0 +1,309 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include:
+  - local: '.common-ci.yml'
+
+default:
+  tags:
+    - cnt
+    - container-dev
+    - docker/multi-arch
+    - docker/privileged
+    - os/linux
+    - type/docker
+
+variables:
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: "/certs"
+  # Release "devel"-tagged images off the master branch
+  RELEASE_DEVEL_BRANCH: "master"
+  DEVEL_RELEASE_IMAGE_VERSION: "devel"
+  # On the multi-arch builder we don't need the qemu setup.
+  SKIP_QEMU_SETUP: "1"
+  # Define the public staging registry
+  STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
+  STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+
+.image-pull:
+  stage: image-build
+  variables:
+    IN_REGISTRY: "${STAGING_REGISTRY}"
+    IN_IMAGE_NAME: container-toolkit
+    IN_VERSION: "${STAGING_VERSION}"
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PUSH_MULTIPLE_TAGS: "false"
+  # We delay the job start to allow the public pipeline to generate the required images.
+  when: delayed
+  start_in: 30 minutes
+  timeout: 30 minutes
+  retry:
+    max: 2
+    when:
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+  before_script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache make bash
+    - >
+      regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
+  script:
+    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
+    - make -f build/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
+
+image-centos7:
+  extends:
+    - .image-pull
+    - .dist-centos7
+
+image-centos8:
+  extends:
+    - .image-pull
+    - .dist-centos8
+
+image-ubi8:
+  extends:
+    - .image-pull
+    - .dist-ubi8
+
+image-ubuntu18.04:
+  extends:
+    - .image-pull
+    - .dist-ubuntu18.04
+
+image-ubuntu20.04:
+  extends:
+    - .image-pull
+    - .dist-ubuntu20.04
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .image-pull
+    - .dist-packaging
+
+# We skip the integration tests for the internal CI:
+.integration:
+  stage: test
+  before_script:
+    - echo "Skipped in internal CI"
+  script:
+    - echo "Skipped in internal CI"
+
+# The .scan step forms the base of the image scan operation performed before releasing
+# images.
+.scan:
+  stage: scan
+  image: "${PULSE_IMAGE}"
+  variables:
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
+    IMAGE_ARCHIVE: "container-toolkit.tar"
+  except:
+    variables:
+      - $SKIP_SCANS && $SKIP_SCANS == "yes"
+  before_script:
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    # TODO: We should specify the architecture here and scan all architectures
+    - docker pull --platform="${PLATFORM}" "${IMAGE}"
+    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
+    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+    - >
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+  script:
+    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+  artifacts:
+    when: always
+    expire_in: 1 week
+    paths:
+      - pulse-cli.log
+      - licenses.json
+      - sbom.json
+      - vulns.json
+      - policy_evaluation.json
+
+# Define the scan targets
+scan-centos7-amd64:
+  extends:
+    - .scan
+    - .dist-centos7
+    - .platform-amd64
+  needs:
+    - image-centos7
+
+scan-centos7-arm64:
+  extends:
+    - .scan
+    - .dist-centos7
+    - .platform-arm64
+  needs:
+    - image-centos7
+    - scan-centos7-amd64
+
+scan-centos8-amd64:
+  extends:
+    - .scan
+    - .dist-centos8
+    - .platform-amd64
+  needs:
+    - image-centos8
+
+scan-centos8-arm64:
+  extends:
+    - .scan
+    - .dist-centos8
+    - .platform-arm64
+  needs:
+    - image-centos8
+    - scan-centos8-amd64
+
+scan-ubuntu18.04-amd64:
+  extends:
+    - .scan
+    - .dist-ubuntu18.04
+    - .platform-amd64
+  needs:
+    - image-ubuntu18.04
+
+scan-ubuntu18.04-arm64:
+  extends:
+    - .scan
+    - .dist-ubuntu18.04
+    - .platform-arm64
+  needs:
+    - image-ubuntu18.04
+    - scan-ubuntu18.04-amd64
+
+scan-ubuntu20.04-amd64:
+  extends:
+    - .scan
+    - .dist-ubuntu20.04
+    - .platform-amd64
+  needs:
+    - image-ubuntu20.04
+
+scan-ubuntu20.04-arm64:
+  extends:
+    - .scan
+    - .dist-ubuntu20.04
+    - .platform-arm64
+  needs:
+    - image-ubuntu20.04
+    - scan-ubuntu20.04-amd64
+
+scan-ubi8-amd64:
+  extends:
+    - .scan
+    - .dist-ubi8
+    - .platform-amd64
+  needs:
+    - image-ubi8
+
+scan-ubi8-arm64:
+  extends:
+    - .scan
+    - .dist-ubi8
+    - .platform-arm64
+  needs:
+    - image-ubi8
+    - scan-ubi8-amd64
+
+# Define external release helpers
+.release:ngc:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+
+.release:dockerhub:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${DOCKERHUB_REGISTRY}"
+    OUT_IMAGE_NAME: "${REGISTRY_IMAGE}"
+
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+release:staging-ubuntu20.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu20.04
+  needs:
+    - image-ubuntu20.04
+
+# Define the external release targets
+# Release to NGC
+release:ngc-centos7:
+  extends:
+    - .release:ngc
+    - .dist-centos7
+
+release:ngc-centos8:
+  extends:
+    - .release:ngc
+    - .dist-centos8
+
+release:ngc-ubuntu18.04:
+  extends:
+    - .release:ngc
+    - .dist-ubuntu18.04
+
+release:ngc-ubuntu20.04:
+  extends:
+    - .release:ngc
+    - .dist-ubuntu20.04
+
+release:ngc-ubi8:
+  extends:
+    - .release:ngc
+    - .dist-ubi8
+
+# Release to Dockerhub
+release:dockerhub-centos7:
+  extends:
+    - .release:dockerhub
+    - .dist-centos7
+
+release:dockerhub-centos8:
+  extends:
+    - .release:dockerhub
+    - .dist-centos8
+
+release:dockerhub-ubuntu18.04:
+  extends:
+    - .release:dockerhub
+    - .dist-ubuntu18.04
+
+release:dockerhub-ubuntu20.04:
+  extends:
+    - .release:dockerhub
+    - .dist-ubuntu20.04
+
+release:dockerhub-ubi8:
+  extends:
+    - .release:dockerhub
+    - .dist-ubi8

Makefile

@@ -16,11 +16,8 @@ DOCKER   ?= docker
 MKDIR    ?= mkdir
 DIST_DIR ?= $(CURDIR)/dist
 
-LIB_NAME := nvidia-container-toolkit
-LIB_VERSION := 1.6.0
-LIB_TAG ?= rc.1
+include $(CURDIR)/versions.mk
 
-GOLANG_VERSION := 1.16.3
 MODULE := github.com/NVIDIA/nvidia-container-toolkit
 
 # By default run all native docker-based targets
@@ -93,11 +90,7 @@ ineffassign:
 
 lint:
 # We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
-	go list -f '{{.Dir}}' $(MODULE)/... | grep -v /internal/ | xargs golint -set_exit_status
-
-lint-internal:
-# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
-	go list -f '{{.Dir}}' $(MODULE)/internal/... | xargs golint -set_exit_status
+	go list -f '{{.Dir}}' $(MODULE)/... | xargs golint -set_exit_status
 
 misspell:
 	misspell $(MODULE)/...
@@ -145,3 +138,15 @@ $(DOCKER_TARGETS): docker-%: .build-image
 		--user $$(id -u):$$(id -g) \
 		$(BUILDIMAGE) \
 			make $(*)
+
+# Start an interactive shell using the development image.
+PHONY: .shell
+.shell:
+	$(DOCKER) run \
+		--rm \
+		-ti \
+		-e GOCACHE=/tmp/.cache \
+		-v $(PWD):$(PWD) \
+		-w $(PWD) \
+		--user $$(id -u):$$(id -g) \
+		$(BUILDIMAGE)

build/container/Dockerfile.centos

@@ -0,0 +1,97 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ARG BASE_DIST
+# See https://www.centos.org/centos-linux-eol/
+# and https://stackoverflow.com/a/70930049 for move to vault.centos.org
+# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud
+RUN [[ "${BASE_DIST}" != "centos8" ]] || \
+    ( \
+      sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
+      sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \
+    )
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+ARG ARTIFACTS_ROOT
+ARG PACKAGE_DIST
+COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
+
+WORKDIR /artifacts/packages
+
+ARG PACKAGE_VERSION
+ARG TARGETARCH
+ENV PACKAGE_ARCH ${TARGETARCH}
+RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
+    yum localinstall -y \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+COPY ./LICENSE /licenses/LICENSE
+
+# Install / upgrade packages here that are required to resolve CVEs
+ARG CVE_UPDATES
+RUN if [ -n "${CVE_UPDATES}" ]; then \
+        yum update -y ${CVE_UPDATES} && \
+        rm -rf /var/cache/yum/*; \
+    fi
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.packaging

@@ -0,0 +1,29 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
+
+ARG ARTIFACTS_ROOT
+COPY ${ARTIFACTS_ROOT} /artifacts/packages/
+
+WORKDIR /artifacts/packages
+
+COPY ./LICENSE /licenses/LICENSE

build/container/Dockerfile.ubuntu

@@ -0,0 +1,102 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libcap2 \
+    curl \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+ARG ARTIFACTS_ROOT
+ARG PACKAGE_DIST
+COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
+
+WORKDIR /artifacts/packages
+
+ARG PACKAGE_VERSION
+ARG TARGETARCH
+ENV PACKAGE_ARCH ${TARGETARCH}
+
+ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
+ARG LIBNVIDIA_CONTAINER0_VERSION
+RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
+        curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
+            --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \
+        dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \
+    fi
+
+RUN dpkg -i \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work/
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+COPY ./LICENSE /licenses/LICENSE
+
+# Install / upgrade packages here that are required to resolve CVEs
+ARG CVE_UPDATES
+RUN if [ -n "${CVE_UPDATES}" ]; then \
+        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -0,0 +1,152 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BUILD_MULTI_ARCH_IMAGES ?= false
+DOCKER ?= docker
+
+BUILDX =
+ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
+BUILDX = buildx
+endif
+
+MKDIR    ?= mkdir
+DIST_DIR ?= $(CURDIR)/dist
+
+##### Global variables #####
+include $(CURDIR)/versions.mk
+
+ifeq ($(IMAGE_NAME),)
+REGISTRY ?= nvidia
+IMAGE_NAME := $(REGISTRY)/container-toolkit
+endif
+
+VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+IMAGE_VERSION := $(VERSION)
+
+IMAGE_TAG ?= $(VERSION)-$(DIST)
+IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
+
+OUT_IMAGE_NAME ?= $(IMAGE_NAME)
+OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
+OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
+OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
+
+##### Public rules #####
+DEFAULT_PUSH_TARGET := ubuntu18.04
+DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+
+META_TARGETS := packaging
+
+BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
+PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
+TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+
+.PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
+
+ifneq ($(BUILD_MULTI_ARCH_IMAGES),true)
+include $(CURDIR)/build/container/native-only.mk
+else
+include $(CURDIR)/build/container/multi-arch.mk
+endif
+
+# For the default push target we also push a short tag equal to the version.
+# We skip this for the development release
+DEVEL_RELEASE_IMAGE_VERSION ?= devel
+PUSH_MULTIPLE_TAGS ?= true
+ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
+PUSH_MULTIPLE_TAGS = false
+endif
+ifeq ($(PUSH_MULTIPLE_TAGS),true)
+push-$(DEFAULT_PUSH_TARGET): push-short
+endif
+
+push-%: DIST = $(*)
+push-short: DIST = $(DEFAULT_PUSH_TARGET)
+
+build-%: DIST = $(*)
+build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
+
+ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+
+# Use a generic build target to build the relevant images
+$(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
+	DOCKER_BUILDKIT=1 \
+		$(DOCKER) $(BUILDX) build --pull \
+		$(DOCKER_BUILD_OPTIONS) \
+		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
+		--tag $(IMAGE) \
+		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
+		--build-arg BASE_DIST="$(BASE_DIST)" \
+		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
+		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+		--build-arg LIBNVIDIA_CONTAINER0_VERSION="$(LIBNVIDIA_CONTAINER0_DEPENDENCY)" \
+		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
+		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
+		--build-arg VERSION="$(VERSION)" \
+		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
+		-f $(DOCKERFILE) \
+		$(CURDIR)
+
+
+build-ubuntu%: BASE_DIST = $(*)
+build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
+build-ubuntu%: PACKAGE_DIST = ubuntu18.04
+build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
+
+build-ubi8: BASE_DIST := ubi8
+build-ubi8: DOCKERFILE_SUFFIX := centos
+build-ubi8: PACKAGE_DIST = centos8
+build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+build-centos%: BASE_DIST = $(*)
+build-centos%: DOCKERFILE_SUFFIX := centos
+build-centos%: PACKAGE_DIST = $(BASE_DIST)
+build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+build-packaging: BASE_DIST := ubuntu20.04
+build-packaging: DOCKERFILE_SUFFIX := packaging
+build-packaging: PACKAGE_ARCH := amd64
+build-packaging: PACKAGE_DIST = all
+build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+
+# Test targets
+test-%: DIST = $(*)
+
+TEST_CASES ?= toolkit docker crio containerd
+$(TEST_TARGETS): test-%:
+	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \
+		$(CURDIR)/shared-$(*) \
+		$(IMAGE) \
+			--no-cleanup-on-error
+
+.PHONY: test-packaging
+test-packaging: DIST = packaging
+test-packaging:
+	@echo "Testing package image contents"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/README.md

@@ -0,0 +1,4 @@
+# NVIDIA Container Toolkit Container
+
+This folder contains make and docker files for building the NVIDIA Container Toolkit Container.
+

build/container/multi-arch.mk

@@ -0,0 +1,34 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PUSH_ON_BUILD ?= false
+DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD)
+DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64
+
+REGCTL ?= regctl
+$(PUSH_TARGETS): push-%:
+	$(REGCTL) \
+	        image copy \
+	        $(IMAGE) $(OUT_IMAGE)
+
+push-short:
+	$(REGCTL) \
+	        image copy \
+	        $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)
+
+# We only have x86_64 packages for centos7
+build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
+# We only generate a single image for packaging targets
+build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64

build/container/native-only.mk

@@ -0,0 +1,23 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
+
+$(PUSH_TARGETS): push-%:
+	$(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)"
+	$(DOCKER) push "$(OUT_IMAGE)"
+
+push-short:
+	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"
+	$(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"

cmd/nvidia-container-runtime/README.md

@@ -0,0 +1,25 @@
+# The NVIDIA Container Runtime
+
+The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
+
+## Standard Mode
+
+In the standard mode configuration, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from project [libnvidia-container](https://github.com/NVIDIA/libnvidia-container).
+
+## Experimental Mode
+
+The NVIDIA Container Runtime can be configured in an experimental mode by setting the following options in the runtime's `config.toml` file:
+
+```toml
+[nvidia-container-runtime]
+experimental = true
+```
+
+When this setting is enabled, the modifications made to the OCI specification are controlled by the `nvidia-container-runtime.discover-mode` option, with the following mode supported:
+
+* `"legacy"`: This mode mirrors the behaviour of the standard mode, inserting the NVIDIA Container Runtime Hook as a `prestart` hook into the container's OCI specification.
+* `"csv"`: This mode uses CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` to define the devices and mounts that are to be injected into a container when it is created.
+
+### Notes on using the docker CLI
+
+The `docker` CLI supports the `--gpus` flag to select GPUs for inclusion in a container. Since specifying this flag inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification. When experimental mode is activated, the NVIDIA Container Runtime detects the presence of the hook and raises an error. This requirement will be relaxed in the near future.

cmd/nvidia-container-runtime/main.go

@@ -3,20 +3,9 @@ package main
 import (
 	"fmt"
 	"os"
-	"path"
 
-	"github.com/pelletier/go-toml"
-)
-
-const (
-	configOverride = "XDG_CONFIG_HOME"
-	configFilePath = "nvidia-container-runtime/config.toml"
-
-	hookDefaultFilePath = "/usr/bin/nvidia-container-runtime-hook"
-)
-
-var (
-	configDir = "/etc/"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/sirupsen/logrus"
 )
 
 var logger = NewLogger()
@@ -24,66 +13,42 @@ var logger = NewLogger()
 func main() {
 	err := run(os.Args)
 	if err != nil {
-		logger.Errorf("Error running %v: %v", os.Args, err)
+		logger.Errorf("%v", err)
 		os.Exit(1)
 	}
 }
 
 // run is an entry point that allows for idiomatic handling of errors
 // when calling from the main function.
-func run(argv []string) (err error) {
-	cfg, err := getConfig()
+func run(argv []string) (rerr error) {
+	logger.Debugf("Running %v", argv)
+	cfg, err := config.GetConfig()
 	if err != nil {
 		return fmt.Errorf("error loading config: %v", err)
 	}
 
-	err = logger.LogToFile(cfg.debugFilePath)
+	err = logger.LogToFile(cfg.NVIDIAContainerRuntimeConfig.DebugFilePath)
 	if err != nil {
 		return fmt.Errorf("error opening debug log file: %v", err)
 	}
 	defer func() {
 		// We capture and log a returning error before closing the log file.
-		if err != nil {
-			logger.Errorf("Error running %v: %v", argv, err)
+		if rerr != nil {
+			logger.Errorf("%v", rerr)
 		}
 		logger.CloseFile()
 	}()
 
-	r, err := newRuntime(argv)
+	if logLevel, err := logrus.ParseLevel(cfg.NVIDIAContainerRuntimeConfig.LogLevel); err == nil {
+		logger.SetLevel(logLevel)
+	} else {
+		logger.Warnf("Invalid log-level '%v'; using '%v'", cfg.NVIDIAContainerRuntimeConfig.LogLevel, logger.Level.String())
+	}
+
+	runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv)
 	if err != nil {
-		return fmt.Errorf("error creating runtime: %v", err)
+		return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
 	}
 
-	logger.Printf("Running %s\n", argv[0])
-	return r.Exec(argv)
-}
-
-type config struct {
-	debugFilePath string
-}
-
-// getConfig sets up the config struct. Values are read from a toml file
-// or set via the environment.
-func getConfig() (*config, error) {
-	cfg := &config{}
-
-	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
-		configDir = XDGConfigDir
-	}
-
-	configFilePath := path.Join(configDir, configFilePath)
-
-	tomlContent, err := os.ReadFile(configFilePath)
-	if err != nil {
-		return nil, err
-	}
-
-	toml, err := toml.Load(string(tomlContent))
-	if err != nil {
-		return nil, err
-	}
-
-	cfg.debugFilePath = toml.GetDefault("nvidia-container-runtime.debug", "/dev/null").(string)
-
-	return cfg, nil
+	return runtime.Exec(argv)
 }

cmd/nvidia-container-runtime/main_test.go

@@ -3,15 +3,15 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/require"
 )
@@ -35,7 +35,7 @@ func TestMain(m *testing.M) {
 	// TEST SETUP
 	// Determine the module root and the test binary path
 	var err error
-	moduleRoot, err := getModuleRoot()
+	moduleRoot, err := test.GetModuleRoot()
 	if err != nil {
 		logger.Fatalf("error in test setup: could not get module root: %v", err)
 	}
@@ -43,7 +43,7 @@ func TestMain(m *testing.M) {
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
 
 	// Set the environment variables for the test
-	os.Setenv("PATH", prependToPath(testBinPath, moduleRoot))
+	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
 	os.Setenv("XDG_CONFIG_HOME", testInputPath)
 
 	// Confirm that the environment is configured correctly
@@ -71,31 +71,6 @@ func TestMain(m *testing.M) {
 	os.Exit(exitCode)
 }
 
-func getModuleRoot() (string, error) {
-	_, filename, _, _ := runtime.Caller(0)
-
-	return hasGoMod(filename)
-}
-
-func hasGoMod(dir string) (string, error) {
-	if dir == "" || dir == "/" {
-		return "", fmt.Errorf("module root not found")
-	}
-
-	_, err := os.Stat(filepath.Join(dir, "go.mod"))
-	if err != nil {
-		return hasGoMod(filepath.Dir(dir))
-	}
-	return dir, nil
-}
-
-func prependToPath(additionalPaths ...string) string {
-	paths := strings.Split(os.Getenv("PATH"), ":")
-	paths = append(additionalPaths, paths...)
-
-	return strings.Join(paths, ":")
-}
-
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
 //		- Confirm the runtime handles bad input correctly
@@ -193,11 +168,11 @@ func TestDuplicateHook(t *testing.T) {
 	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
 }
 
-// addNVIDIAHook is a basic wrapper for nvidiaContainerRunime.addNVIDIAHook that is used for
+// addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	r := nvidiaContainerRuntime{logger: logger.Logger}
-	return r.addNVIDIAHook(spec)
+	m := modifier.NewStableRuntimeModifier(logger.Logger)
+	return m.Modify(spec)
 }
 
 func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
@@ -270,24 +245,3 @@ func nvidiaHookCount(hooks *specs.Hooks) int {
 	}
 	return count
 }
-
-func TestGetConfigWithCustomConfig(t *testing.T) {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
-
-	// By default debug is disabled
-	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
-	testDir := filepath.Join(wd, "test")
-	filename := filepath.Join(testDir, configFilePath)
-
-	os.Setenv(configOverride, testDir)
-
-	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
-	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
-
-	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
-
-	cfg, err := getConfig()
-	require.NoError(t, err)
-	require.Equal(t, cfg.debugFilePath, "/nvidia-container-toolkit.log")
-}

cmd/nvidia-container-runtime/modifier/experimental.go

@@ -0,0 +1,178 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// experiemental represents the modifications required by the experimental runtime
+type experimental struct {
+	logger     *logrus.Logger
+	discoverer discover.Discover
+}
+
+const (
+	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
+	visibleDevicesVoid   = "void"
+
+	nvidiaRequireJetpackEnvvar = "NVIDIA_REQUIRE_JETPACK"
+)
+
+// NewExperimentalModifier creates a modifier that applies the experimental
+// modications to an OCI spec if required by the runtime wrapper.
+func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	if err := ociSpec.Load(); err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	// In experimental mode, we check whether a modification is required at all and return the lowlevelRuntime directly
+	// if no modification is required.
+	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
+	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
+		logger.Infof("No modification required: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
+		return nil, nil
+	}
+	logger.Infof("Constructing modifier from config: %+v", cfg)
+
+	config := &discover.Config{
+		Root:                                    cfg.NVIDIAContainerCLIConfig.Root,
+		NVIDIAContainerToolkitCLIExecutablePath: cfg.NVIDIACTKConfig.Path,
+	}
+
+	var d discover.Discover
+
+	switch resolveAutoDiscoverMode(logger, cfg.NVIDIAContainerRuntimeConfig.DiscoverMode) {
+	case "legacy":
+		legacyDiscoverer, err := discover.NewLegacyDiscoverer(logger, config)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create legacy discoverer: %v", err)
+		}
+		d = legacyDiscoverer
+	case "csv":
+		csvFiles, err := csv.GetFileList(csv.DefaultMountSpecPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get list of CSV files: %v", err)
+		}
+
+		nvidiaRequireJetpack, _ := ociSpec.LookupEnv(nvidiaRequireJetpackEnvvar)
+		if nvidiaRequireJetpack != "csv-mounts=all" {
+			csvFiles = csv.BaseFilesOnly(csvFiles)
+		}
+
+		csvDiscoverer, err := discover.NewFromCSVFiles(logger, csvFiles, config.Root)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create CSV discoverer: %v", err)
+		}
+
+		ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(logger, csvDiscoverer, config)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err)
+		}
+
+		createSymlinksHook, err := discover.NewCreateSymlinksHook(logger, csvFiles, config)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create symlink hook discoverer: %v", err)
+		}
+
+		d = discover.NewList(csvDiscoverer, ldcacheUpdateHook, createSymlinksHook)
+	default:
+		return nil, fmt.Errorf("invalid discover mode: %v", cfg.NVIDIAContainerRuntimeConfig.DiscoverMode)
+	}
+
+	return newExperimentalModifierFromDiscoverer(logger, d)
+}
+
+// newExperimentalModifierFromDiscoverer created a modifier that aplies the discovered
+// modifications to an OCI spec if require by the runtime wrapper.
+func newExperimentalModifierFromDiscoverer(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	m := experimental{
+		logger:     logger,
+		discoverer: d,
+	}
+	return &m, nil
+}
+
+// Modify applies the required modifications to the incomming OCI spec. These modifications
+// are applied in-place.
+func (m experimental) Modify(spec *specs.Spec) error {
+	err := nvidiaContainerRuntimeHookRemover{m.logger}.Modify(spec)
+	if err != nil {
+		return fmt.Errorf("failed to remove existing hooks: %v", err)
+	}
+
+	specEdits, err := edits.NewSpecEdits(m.logger, m.discoverer)
+	if err != nil {
+		return fmt.Errorf("failed to get required container edits: %v", err)
+	}
+
+	return specEdits.Modify(spec)
+}
+
+// resolveAutoDiscoverMode determines the correct discover mode for the specified platform if set to "auto"
+func resolveAutoDiscoverMode(logger *logrus.Logger, mode string) (rmode string) {
+	if mode != "auto" {
+		return mode
+	}
+	defer func() {
+		logger.Infof("Auto-detected discover mode as '%v'", rmode)
+	}()
+
+	isTegra, reason := isTegraSystem()
+	logger.Debugf("Is Tegra-based system? %v: %v", isTegra, reason)
+
+	if isTegra {
+		return "csv"
+	}
+
+	return "legacy"
+}
+
+// isTegraSystem returns true if the system is detected as a Tegra-based system
+func isTegraSystem() (bool, string) {
+	const tegraReleaseFile = "/etc/nv_tegra_release"
+	const tegraFamilyFile = "/sys/devices/soc0/family"
+
+	if info, err := os.Stat(tegraReleaseFile); err == nil && !info.IsDir() {
+		return true, fmt.Sprintf("%v found", tegraReleaseFile)
+	}
+
+	if info, err := os.Stat(tegraFamilyFile); err != nil || !info.IsDir() {
+		return false, fmt.Sprintf("%v not found", tegraFamilyFile)
+	}
+
+	contents, err := os.ReadFile(tegraFamilyFile)
+	if err != nil {
+		return false, fmt.Sprintf("could not read %v", tegraFamilyFile)
+	}
+
+	if strings.HasPrefix(strings.ToLower(string(contents)), "tegra") {
+		return true, fmt.Sprintf("%v has 'tegra' prefix", tegraFamilyFile)
+	}
+
+	return false, fmt.Sprintf("%v has no 'tegra' prefix", tegraFamilyFile)
+}

cmd/nvidia-container-runtime/modifier/experimental_test.go

@@ -0,0 +1,349 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewExperimentalModifier(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description    string
+		cfg            *config.Config
+		spec           oci.Spec
+		visibleDevices string
+		expectedError  error
+		expectedNil    bool
+	}{
+		{
+			description: "spec load error returns error",
+			spec: &oci.SpecMock{
+				LoadFunc: func() error {
+					return fmt.Errorf("load failed")
+				},
+			},
+			expectedError: fmt.Errorf("load failed"),
+		},
+		{
+			description:    "visible devices not set returns nil",
+			visibleDevices: "NOT_SET",
+			expectedNil:    true,
+		},
+		{
+			description:    "visible devices empty returns nil",
+			visibleDevices: "",
+			expectedNil:    true,
+		},
+		{
+			description:    "visible devices 'void' returns nil",
+			visibleDevices: "void",
+			expectedNil:    true,
+		},
+		{
+			description: "empty config raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
+			},
+			visibleDevices: "all",
+			expectedError:  fmt.Errorf("invalid discover mode"),
+		},
+		{
+			description: "non-legacy discover mode raises error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					DiscoverMode: "non-legacy",
+				},
+			},
+			visibleDevices: "all",
+			expectedError:  fmt.Errorf("invalid discover mode"),
+		},
+		{
+			description: "legacy discover mode returns modifier",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					DiscoverMode: "legacy",
+				},
+			},
+			visibleDevices: "all",
+		},
+		{
+			description: "csv discover mode returns modifier",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					DiscoverMode: "csv",
+				},
+			},
+			visibleDevices: "all",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			spec := tc.spec
+			if spec == nil {
+				spec = &oci.SpecMock{
+					LookupEnvFunc: func(s string) (string, bool) {
+						if tc.visibleDevices != "NOT_SET" && s == visibleDevicesEnvvar {
+							return tc.visibleDevices, true
+						}
+						return "", false
+					},
+				}
+			}
+
+			m, err := NewExperimentalModifier(logger, tc.cfg, spec)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			if tc.expectedNil || tc.expectedError != nil {
+				require.Nil(t, m)
+			} else {
+				require.NotNil(t, m)
+			}
+		})
+	}
+}
+
+func TestExperimentalModifier(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		discover      *discover.DiscoverMock
+		spec          *specs.Spec
+		expectedError error
+		expectedSpec  *specs.Spec
+	}{
+		{
+			description: "empty discoverer does not modify spec",
+			discover:    &discover.DiscoverMock{},
+		},
+		{
+			description: "failed hooks discoverer returns error",
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					return nil, fmt.Errorf("discover.Hooks error")
+				},
+			},
+			expectedError: fmt.Errorf("discover.Hooks error"),
+		},
+		{
+			description: "discovered hooks are injected into spec",
+			spec:        &specs.Spec{},
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					hooks := []discover.Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/hook/a",
+							Args:      []string{"/hook/a", "arga"},
+						},
+						{
+							Lifecycle: "createContainer",
+							Path:      "/hook/b",
+							Args:      []string{"/hook/b", "argb"},
+						},
+					}
+					return hooks, nil
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+					},
+					CreateContainer: []specs.Hook{
+						{
+							Path: "/hook/b",
+							Args: []string{"/hook/b", "argb"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "existing hooks are maintained",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+					},
+				},
+			},
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					hooks := []discover.Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/hook/b",
+							Args:      []string{"/hook/b", "argb"},
+						},
+					}
+					return hooks, nil
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/a",
+							Args: []string{"/hook/a", "arga"},
+						},
+						{
+							Path: "/hook/b",
+							Args: []string{"/hook/b", "argb"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "modification removes existing nvidia-container-runtime-hook",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/path/to/nvidia-container-runtime-hook",
+							Args: []string{"/path/to/nvidia-container-runtime-hook", "prestart"},
+						},
+					},
+				},
+			},
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					hooks := []discover.Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/hook/b",
+							Args:      []string{"/hook/b", "argb"},
+						},
+					}
+					return hooks, nil
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/b",
+							Args: []string{"/hook/b", "argb"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "modification removes existing nvidia-container-toolkit",
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/path/to/nvidia-container-toolkit",
+							Args: []string{"/path/to/nvidia-container-toolkit", "prestart"},
+						},
+					},
+				},
+			},
+			discover: &discover.DiscoverMock{
+				HooksFunc: func() ([]discover.Hook, error) {
+					hooks := []discover.Hook{
+						{
+							Lifecycle: "prestart",
+							Path:      "/hook/b",
+							Args:      []string{"/hook/b", "argb"},
+						},
+					}
+					return hooks, nil
+				},
+			},
+			expectedSpec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "/hook/b",
+							Args: []string{"/hook/b", "argb"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			m, err := newExperimentalModifierFromDiscoverer(logger, tc.discover)
+			require.NoError(t, err)
+
+			err = m.Modify(tc.spec)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.EqualValues(t, tc.expectedSpec, tc.spec)
+		})
+	}
+}
+
+func TestResolveDiscoverMode(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description  string
+		mode         string
+		expectedMode string
+	}{
+		{
+			description:  "non-auto resolves to input",
+			mode:         "not-auto",
+			expectedMode: "not-auto",
+		},
+		// TODO: The following test is brittle in that it will break on Tegra-based systems.
+		// {
+		// 	description:  "auto resolves to legacy",
+		// 	mode:         "auto",
+		// 	expectedMode: "legacy",
+		// },
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mode := resolveAutoDiscoverMode(logger, tc.mode)
+			require.EqualValues(t, tc.expectedMode, mode)
+		})
+	}
+}

cmd/nvidia-container-runtime/modifier/hook_remover.go

@@ -0,0 +1,79 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// nvidiaContainerRuntimeHookRemover is a spec modifer that detects and removes inserted nvidia-container-runtime hooks
+type nvidiaContainerRuntimeHookRemover struct {
+	logger *logrus.Logger
+}
+
+var _ oci.SpecModifier = (*nvidiaContainerRuntimeHookRemover)(nil)
+
+// Modify removes any NVIDIA Container Runtime hooks from the provided spec
+func (m nvidiaContainerRuntimeHookRemover) Modify(spec *specs.Spec) error {
+	if spec == nil {
+		return nil
+	}
+
+	if spec.Hooks == nil {
+		return nil
+	}
+
+	if len(spec.Hooks.Prestart) == 0 {
+		return nil
+	}
+
+	var newPrestart []specs.Hook
+
+	for _, hook := range spec.Hooks.Prestart {
+		if isNVIDIAContainerRuntimeHook(&hook) {
+			m.logger.Debugf("Removing hook %v", hook)
+			continue
+		}
+		newPrestart = append(newPrestart, hook)
+	}
+
+	if len(newPrestart) != len(spec.Hooks.Prestart) {
+		m.logger.Debugf("Updating 'prestart' hooks to %v", newPrestart)
+		spec.Hooks.Prestart = newPrestart
+	}
+
+	return nil
+}
+
+// isNVIDIAContainerRuntimeHook checks if the provided hook is an nvidia-container-runtime-hook
+// or nvidia-container-toolkit hook. These are included, for example, by the non-experimental
+// nvidia-container-runtime or docker when specifying the --gpus flag.
+func isNVIDIAContainerRuntimeHook(hook *specs.Hook) bool {
+	bins := map[string]struct{}{
+		config.NVIDIAContainerRuntimeHookExecutable: {},
+		config.NVIDIAContainerToolkitExecutable:     {},
+	}
+
+	_, exists := bins[filepath.Base(hook.Path)]
+
+	return exists
+}

cmd/nvidia-container-runtime/modifier/stable.go

@@ -0,0 +1,77 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package modifier
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// NewStableRuntimeModifier creates an OCI spec modifier that inserts the NVIDIA Container Runtime Hook into an OCI
+// spec. The specified logger is used to capture log output.
+func NewStableRuntimeModifier(logger *logrus.Logger) oci.SpecModifier {
+	m := stableRuntimeModifier{logger: logger}
+
+	return &m
+}
+
+// stableRuntimeModifier modifies an OCI spec inplace, inserting the nvidia-container-runtime-hook as a
+// prestart hook. If the hook is already present, no modification is made.
+type stableRuntimeModifier struct {
+	logger *logrus.Logger
+}
+
+// Modify applies the required modification to the incoming OCI spec, inserting the nvidia-container-runtime-hook
+// as a prestart hook.
+func (m stableRuntimeModifier) Modify(spec *specs.Spec) error {
+	path, err := exec.LookPath(config.NVIDIAContainerRuntimeHookExecutable)
+	if err != nil {
+		path = filepath.Join(config.DefaultExecutableDir, config.NVIDIAContainerRuntimeHookExecutable)
+		_, err = os.Stat(path)
+		if err != nil {
+			return err
+		}
+	}
+
+	m.logger.Infof("Using prestart hook path: %s", path)
+
+	args := []string{path}
+	if spec.Hooks == nil {
+		spec.Hooks = &specs.Hooks{}
+	} else if len(spec.Hooks.Prestart) != 0 {
+		for _, hook := range spec.Hooks.Prestart {
+			if strings.Contains(hook.Path, config.NVIDIAContainerRuntimeHookExecutable) {
+				m.logger.Infof("existing nvidia prestart hook found in OCI spec")
+				return nil
+			}
+		}
+	}
+
+	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
+		Path: path,
+		Args: append(args, "prestart"),
+	})
+
+	return nil
+}

cmd/nvidia-container-runtime/modifier/stable_test.go

@@ -0,0 +1,170 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package modifier
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+type testConfig struct {
+	root    string
+	binPath string
+}
+
+var cfg *testConfig
+
+func TestMain(m *testing.M) {
+	// TEST SETUP
+	// Determine the module root and the test binary path
+	var err error
+	moduleRoot, err := test.GetModuleRoot()
+	if err != nil {
+		logrus.Fatalf("error in test setup: could not get module root: %v", err)
+	}
+	testBinPath := filepath.Join(moduleRoot, "test", "bin")
+
+	// Set the environment variables for the test
+	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
+
+	// Store the root and binary paths in the test Config
+	cfg = &testConfig{
+		root:    moduleRoot,
+		binPath: testBinPath,
+	}
+
+	// RUN TESTS
+	exitCode := m.Run()
+
+	os.Exit(exitCode)
+}
+
+func TestAddHookModifier(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+
+	testHookPath := filepath.Join(cfg.binPath, "nvidia-container-runtime-hook")
+
+	testCases := []struct {
+		description   string
+		spec          specs.Spec
+		expectedError error
+		expectedSpec  specs.Spec
+	}{
+		{
+			description: "empty spec adds hook",
+			spec:        specs.Spec{},
+			expectedSpec: specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: testHookPath,
+							Args: []string{testHookPath, "prestart"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "spec with empty hooks adds hook",
+			spec: specs.Spec{
+				Hooks: &specs.Hooks{},
+			},
+			expectedSpec: specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: testHookPath,
+							Args: []string{testHookPath, "prestart"},
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "hook is not replaced",
+			spec: specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "nvidia-container-runtime-hook",
+						},
+					},
+				},
+			},
+			expectedSpec: specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "nvidia-container-runtime-hook",
+						},
+					},
+				},
+			},
+		},
+		{
+			description: "other hooks are not replaced",
+			spec: specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "some-hook",
+						},
+					},
+				},
+			},
+			expectedSpec: specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{
+						{
+							Path: "some-hook",
+						},
+						{
+							Path: testHookPath,
+							Args: []string{testHookPath, "prestart"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		logHook.Reset()
+
+		t.Run(tc.description, func(t *testing.T) {
+
+			m := NewStableRuntimeModifier(logger)
+
+			err := m.Modify(&tc.spec)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.EqualValues(t, tc.expectedSpec, tc.spec)
+		})
+	}
+
+}

cmd/nvidia-container-runtime/nvcr.go

@@ -1,145 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	log "github.com/sirupsen/logrus"
-)
-
-// nvidiaContainerRuntime encapsulates the NVIDIA Container Runtime. It wraps the specified runtime, conditionally
-// modifying the specified OCI specification before invoking the runtime.
-type nvidiaContainerRuntime struct {
-	logger  *log.Logger
-	runtime oci.Runtime
-	ociSpec oci.Spec
-}
-
-var _ oci.Runtime = (*nvidiaContainerRuntime)(nil)
-
-// newNvidiaContainerRuntime is a constructor for a standard runtime shim.
-func newNvidiaContainerRuntimeWithLogger(logger *log.Logger, runtime oci.Runtime, ociSpec oci.Spec) (oci.Runtime, error) {
-	r := nvidiaContainerRuntime{
-		logger:  logger,
-		runtime: runtime,
-		ociSpec: ociSpec,
-	}
-
-	return &r, nil
-}
-
-// Exec defines the entrypoint for the NVIDIA Container Runtime. A check is performed to see whether modifications
-// to the OCI spec are required -- and applicable modifcations applied. The supplied arguments are then
-// forwarded to the underlying runtime's Exec method.
-func (r nvidiaContainerRuntime) Exec(args []string) error {
-	if r.modificationRequired(args) {
-		err := r.modifyOCISpec()
-		if err != nil {
-			return fmt.Errorf("error modifying OCI spec: %v", err)
-		}
-	}
-
-	r.logger.Println("Forwarding command to runtime")
-	return r.runtime.Exec(args)
-}
-
-// modificationRequired checks the intput arguments to determine whether a modification
-// to the OCI spec is required.
-func (r nvidiaContainerRuntime) modificationRequired(args []string) bool {
-	var previousWasBundle bool
-	for _, a := range args {
-		// We check for '--bundle create' explicitly to ensure that we
-		// don't inadvertently trigger a modification if the bundle directory
-		// is specified as `create`
-		if !previousWasBundle && isBundleFlag(a) {
-			previousWasBundle = true
-			continue
-		}
-
-		if !previousWasBundle && a == "create" {
-			r.logger.Infof("'create' command detected; modification required")
-			return true
-		}
-
-		previousWasBundle = false
-	}
-
-	r.logger.Infof("No modification required")
-	return false
-}
-
-// modifyOCISpec loads and modifies the OCI spec specified in the nvidiaContainerRuntime
-// struct. The spec is modified in-place and written to the same file as the input after
-// modifcationas are applied.
-func (r nvidiaContainerRuntime) modifyOCISpec() error {
-	err := r.ociSpec.Load()
-	if err != nil {
-		return fmt.Errorf("error loading OCI specification for modification: %v", err)
-	}
-
-	err = r.ociSpec.Modify(r.addNVIDIAHook)
-	if err != nil {
-		return fmt.Errorf("error injecting NVIDIA Container Runtime hook: %v", err)
-	}
-
-	err = r.ociSpec.Flush()
-	if err != nil {
-		return fmt.Errorf("error writing modified OCI specification: %v", err)
-	}
-	return nil
-}
-
-// addNVIDIAHook modifies the specified OCI specification in-place, inserting a
-// prestart hook.
-func (r nvidiaContainerRuntime) addNVIDIAHook(spec *specs.Spec) error {
-	path, err := exec.LookPath("nvidia-container-runtime-hook")
-	if err != nil {
-		path = hookDefaultFilePath
-		_, err = os.Stat(path)
-		if err != nil {
-			return err
-		}
-	}
-
-	r.logger.Printf("prestart hook path: %s\n", path)
-
-	args := []string{path}
-	if spec.Hooks == nil {
-		spec.Hooks = &specs.Hooks{}
-	} else if len(spec.Hooks.Prestart) != 0 {
-		for _, hook := range spec.Hooks.Prestart {
-			if !strings.Contains(hook.Path, "nvidia-container-runtime-hook") {
-				continue
-			}
-			r.logger.Println("existing nvidia prestart hook in OCI spec file")
-			return nil
-		}
-	}
-
-	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
-		Path: path,
-		Args: append(args, "prestart"),
-	})
-
-	return nil
-}

cmd/nvidia-container-runtime/nvcr_test.go

@@ -1,230 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestArgsGetConfigFilePath(t *testing.T) {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
-
-	testCases := []struct {
-		bundleDir   string
-		ociSpecPath string
-	}{
-		{
-			ociSpecPath: fmt.Sprintf("%v/config.json", wd),
-		},
-		{
-			bundleDir:   "/foo/bar",
-			ociSpecPath: "/foo/bar/config.json",
-		},
-		{
-			bundleDir:   "/foo/bar/",
-			ociSpecPath: "/foo/bar/config.json",
-		},
-	}
-
-	for i, tc := range testCases {
-		cp, err := getOCISpecFilePath(tc.bundleDir)
-
-		require.NoErrorf(t, err, "%d: %v", i, tc)
-		require.Equalf(t, tc.ociSpecPath, cp, "%d: %v", i, tc)
-	}
-}
-
-func TestAddNvidiaHook(t *testing.T) {
-	logger, logHook := testlog.NewNullLogger()
-	shim := nvidiaContainerRuntime{
-		logger: logger,
-	}
-
-	testCases := []struct {
-		spec         *specs.Spec
-		errorPrefix  string
-		shouldNotAdd bool
-	}{
-		{
-			spec: &specs.Spec{},
-		},
-		{
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{},
-			},
-		},
-		{
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{{
-						Path: "some-hook",
-					}},
-				},
-			},
-		},
-		{
-			spec: &specs.Spec{
-				Hooks: &specs.Hooks{
-					Prestart: []specs.Hook{{
-						Path: "nvidia-container-runtime-hook",
-					}},
-				},
-			},
-			shouldNotAdd: true,
-		},
-	}
-
-	for i, tc := range testCases {
-		logHook.Reset()
-
-		var numPrestartHooks int
-		if tc.spec.Hooks != nil {
-			numPrestartHooks = len(tc.spec.Hooks.Prestart)
-		}
-
-		err := shim.addNVIDIAHook(tc.spec)
-
-		if tc.errorPrefix == "" {
-			require.NoErrorf(t, err, "%d: %v", i, tc)
-		} else {
-			require.Truef(t, strings.HasPrefix(err.Error(), tc.errorPrefix), "%d: %v", i, tc)
-
-			require.NotNilf(t, tc.spec.Hooks, "%d: %v", i, tc)
-			require.Equalf(t, 1, nvidiaHookCount(tc.spec.Hooks), "%d: %v", i, tc)
-
-			if tc.shouldNotAdd {
-				require.Equal(t, numPrestartHooks+1, len(tc.spec.Hooks.Poststart), "%d: %v", i, tc)
-			} else {
-				require.Equal(t, numPrestartHooks+1, len(tc.spec.Hooks.Poststart), "%d: %v", i, tc)
-
-				nvidiaHook := tc.spec.Hooks.Poststart[len(tc.spec.Hooks.Poststart)-1]
-
-				// TODO: This assumes that the hook has been set up in the makefile
-				expectedPath := "/usr/bin/nvidia-container-runtime-hook"
-				require.Equalf(t, expectedPath, nvidiaHook.Path, "%d: %v", i, tc)
-				require.Equalf(t, []string{expectedPath, "prestart"}, nvidiaHook.Args, "%d: %v", i, tc)
-				require.Emptyf(t, nvidiaHook.Env, "%d: %v", i, tc)
-				require.Nilf(t, nvidiaHook.Timeout, "%d: %v", i, tc)
-			}
-		}
-	}
-}
-
-func TestNvidiaContainerRuntime(t *testing.T) {
-	logger, hook := testlog.NewNullLogger()
-
-	testCases := []struct {
-		shim         nvidiaContainerRuntime
-		shouldModify bool
-		args         []string
-		modifyError  error
-		writeError   error
-	}{
-		{
-			shim:         nvidiaContainerRuntime{},
-			shouldModify: false,
-		},
-		{
-			shim:         nvidiaContainerRuntime{},
-			args:         []string{"create"},
-			shouldModify: true,
-		},
-		{
-			shim:         nvidiaContainerRuntime{},
-			args:         []string{"--bundle=create"},
-			shouldModify: false,
-		},
-		{
-			shim:         nvidiaContainerRuntime{},
-			args:         []string{"--bundle", "create"},
-			shouldModify: false,
-		},
-		{
-			shim:         nvidiaContainerRuntime{},
-			args:         []string{"create"},
-			shouldModify: true,
-		},
-		{
-			shim:         nvidiaContainerRuntime{},
-			args:         []string{"create"},
-			modifyError:  fmt.Errorf("error modifying"),
-			shouldModify: true,
-		},
-		{
-			shim:         nvidiaContainerRuntime{},
-			args:         []string{"create"},
-			writeError:   fmt.Errorf("error writing"),
-			shouldModify: true,
-		},
-	}
-
-	for i, tc := range testCases {
-		tc.shim.logger = logger
-		hook.Reset()
-
-		spec := &specs.Spec{}
-		ociMock := oci.NewMockSpec(spec, tc.writeError, tc.modifyError)
-
-		require.Equal(t, tc.shouldModify, tc.shim.modificationRequired(tc.args), "%d: %v", i, tc)
-
-		tc.shim.ociSpec = ociMock
-		tc.shim.runtime = &MockShim{}
-
-		err := tc.shim.Exec(tc.args)
-		if tc.modifyError != nil || tc.writeError != nil {
-			require.Error(t, err, "%d: %v", i, tc)
-		} else {
-			require.NoError(t, err, "%d: %v", i, tc)
-		}
-
-		if tc.shouldModify {
-			require.Equal(t, 1, ociMock.MockModify.Callcount, "%d: %v", i, tc)
-			require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "%d: %v", i, tc)
-		} else {
-			require.Equal(t, 0, ociMock.MockModify.Callcount, "%d: %v", i, tc)
-			require.Nil(t, spec.Hooks, "%d: %v", i, tc)
-		}
-
-		writeExpected := tc.shouldModify && tc.modifyError == nil
-		if writeExpected {
-			require.Equal(t, 1, ociMock.MockFlush.Callcount, "%d: %v", i, tc)
-		} else {
-			require.Equal(t, 0, ociMock.MockFlush.Callcount, "%d: %v", i, tc)
-		}
-	}
-}
-
-type MockShim struct {
-	called      bool
-	args        []string
-	returnError error
-}
-
-func (m *MockShim) Exec(args []string) error {
-	m.called = true
-	m.args = args
-	return m.returnError
-}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -1,5 +1,5 @@
 /*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,159 +18,53 @@ package main
 
 import (
 	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime/modifier"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+	"github.com/sirupsen/logrus"
 )
 
 const (
-	ociSpecFileName          = "config.json"
 	dockerRuncExecutableName = "docker-runc"
 	runcExecutableName       = "runc"
 )
 
-// newRuntime is a factory method that constructs a runtime based on the selected configuration.
-func newRuntime(argv []string) (oci.Runtime, error) {
-	ociSpec, err := newOCISpec(argv)
+// newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger
+func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) {
+	ociSpec, err := oci.NewSpec(logger, argv)
 	if err != nil {
 		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
 	}
 
-	runc, err := newRuncRuntime()
+	lowLevelRuntimeCandidates := []string{dockerRuncExecutableName, runcExecutableName}
+	lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, lowLevelRuntimeCandidates)
 	if err != nil {
-		return nil, fmt.Errorf("error constructing runc runtime: %v", err)
+		return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
 	}
 
-	r, err := newNvidiaContainerRuntimeWithLogger(logger.Logger, runc, ociSpec)
+	specModifier, err := newSpecModifier(logger, cfg, ociSpec)
 	if err != nil {
-		return nil, fmt.Errorf("error constructing NVIDIA Container Runtime: %v", err)
+		return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
 	}
 
+	// Create the wrapping runtime with the specified modifier
+	r := runtime.NewModifyingRuntimeWrapper(
+		logger,
+		lowLevelRuntime,
+		ociSpec,
+		specModifier,
+	)
+
 	return r, nil
 }
 
-// newOCISpec constructs an OCI spec for the provided arguments
-func newOCISpec(argv []string) (oci.Spec, error) {
-	bundlePath, err := getBundlePath(argv)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing command line arguments: %v", err)
+// newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
+func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {
+	if !cfg.NVIDIAContainerRuntimeConfig.Experimental {
+		return modifier.NewStableRuntimeModifier(logger), nil
 	}
 
-	ociSpecPath, err := getOCISpecFilePath(bundlePath)
-	if err != nil {
-		return nil, fmt.Errorf("error getting OCI specification file path: %v", err)
-	}
-	ociSpec := oci.NewSpecFromFile(ociSpecPath)
-
-	return ociSpec, nil
-}
-
-// newRuncRuntime locates the runc binary and wraps it in a SyscallExecRuntime
-func newRuncRuntime() (oci.Runtime, error) {
-	runtimePath, err := findRunc()
-	if err != nil {
-		return nil, fmt.Errorf("error locating runtime: %v", err)
-	}
-
-	runc, err := oci.NewSyscallExecRuntimeWithLogger(logger.Logger, runtimePath)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing runtime: %v", err)
-	}
-
-	return runc, nil
-}
-
-// getBundlePath checks the specified slice of strings (argv) for a 'bundle' flag as allowed by runc.
-// The following are supported:
-// --bundle{{SEP}}BUNDLE_PATH
-// -bundle{{SEP}}BUNDLE_PATH
-// -b{{SEP}}BUNDLE_PATH
-// where {{SEP}} is either ' ' or '='
-func getBundlePath(argv []string) (string, error) {
-	var bundlePath string
-
-	for i := 0; i < len(argv); i++ {
-		param := argv[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		if !isBundleFlag(parts[0]) {
-			continue
-		}
-
-		// The flag has the format --bundle=/path
-		if len(parts) == 2 {
-			bundlePath = parts[1]
-			continue
-		}
-
-		// The flag has the format --bundle /path
-		if i+1 < len(argv) {
-			bundlePath = argv[i+1]
-			i++
-			continue
-		}
-
-		// --bundle / -b was the last element of argv
-		return "", fmt.Errorf("bundle option requires an argument")
-	}
-
-	return bundlePath, nil
-}
-
-// findRunc locates runc in the path, returning the full path to the
-// binary or an error.
-func findRunc() (string, error) {
-	runtimeCandidates := []string{
-		dockerRuncExecutableName,
-		runcExecutableName,
-	}
-
-	return findRuntime(runtimeCandidates)
-}
-
-func findRuntime(runtimeCandidates []string) (string, error) {
-	for _, candidate := range runtimeCandidates {
-		logger.Infof("Looking for runtime binary '%v'", candidate)
-		runcPath, err := exec.LookPath(candidate)
-		if err == nil {
-			logger.Infof("Found runtime binary '%v'", runcPath)
-			return runcPath, nil
-		}
-		logger.Warnf("Runtime binary '%v' not found: %v", candidate, err)
-	}
-
-	return "", fmt.Errorf("no runtime binary found from candidate list: %v", runtimeCandidates)
-}
-
-func isBundleFlag(arg string) bool {
-	if !strings.HasPrefix(arg, "-") {
-		return false
-	}
-
-	trimmed := strings.TrimLeft(arg, "-")
-	return trimmed == "b" || trimmed == "bundle"
-}
-
-// getOCISpecFilePath returns the expected path to the OCI specification file for the given
-// bundle directory or the current working directory if not specified.
-func getOCISpecFilePath(bundleDir string) (string, error) {
-	if bundleDir == "" {
-		logger.Infof("Bundle directory path is empty, using working directory.")
-		workingDirectory, err := os.Getwd()
-		if err != nil {
-			return "", fmt.Errorf("error getting working directory: %v", err)
-		}
-		bundleDir = workingDirectory
-	}
-
-	logger.Infof("Using bundle directory: %v", bundleDir)
-
-	OCISpecFilePath := filepath.Join(bundleDir, ociSpecFileName)
-
-	logger.Infof("Using OCI specification file path: %v", OCISpecFilePath)
-
-	return OCISpecFilePath, nil
+	return modifier.NewExperimentalModifier(logger, cfg, ociSpec)
 }

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -17,176 +17,66 @@
 package main
 
 import (
+	"encoding/json"
+	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
-func TestConstructor(t *testing.T) {
-	shim, err := newRuntime([]string{})
-
-	require.NoError(t, err)
-	require.NotNil(t, shim)
-}
-
-func TestGetBundlePath(t *testing.T) {
-	type expected struct {
-		bundle  string
-		isError bool
-	}
-	testCases := []struct {
-		argv     []string
-		expected expected
-	}{
-		{
-			argv: []string{},
-		},
-		{
-			argv: []string{"create"},
-		},
-		{
-			argv: []string{"--bundle"},
-			expected: expected{
-				isError: true,
-			},
-		},
-		{
-			argv: []string{"-b"},
-			expected: expected{
-				isError: true,
-			},
-		},
-		{
-			argv: []string{"--bundle", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"--not-bundle", "/foo/bar"},
-		},
-		{
-			argv: []string{"--"},
-		},
-		{
-			argv: []string{"-bundle", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"--bundle=/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"-b=/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"-b=/foo/=bar"},
-			expected: expected{
-				bundle: "/foo/=bar",
-			},
-		},
-		{
-			argv: []string{"-b", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"create", "-b", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"-b", "create", "create"},
-			expected: expected{
-				bundle: "create",
-			},
-		},
-		{
-			argv: []string{"-b=create", "create"},
-			expected: expected{
-				bundle: "create",
-			},
-		},
-		{
-			argv: []string{"-b", "create"},
-			expected: expected{
-				bundle: "create",
-			},
-		},
-	}
-
-	for i, tc := range testCases {
-		bundle, err := getBundlePath(tc.argv)
-
-		if tc.expected.isError {
-			require.Errorf(t, err, "%d: %v", i, tc)
-		} else {
-			require.NoErrorf(t, err, "%d: %v", i, tc)
-		}
-
-		require.Equalf(t, tc.expected.bundle, bundle, "%d: %v", i, tc)
-	}
-}
-
-func TestFindRunc(t *testing.T) {
-	testLogger, _ := testlog.NewNullLogger()
-	logger.Logger = testLogger
-
-	runcPath, err := findRunc()
-	require.NoError(t, err)
-	require.Equal(t, filepath.Join(cfg.binPath, runcExecutableName), runcPath)
-}
-
-func TestFindRuntime(t *testing.T) {
-	testLogger, _ := testlog.NewNullLogger()
-	logger.Logger = testLogger
+func TestFactoryMethod(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
 
 	testCases := []struct {
-		candidates   []string
-		expectedPath string
+		description   string
+		cfg           *config.Config
+		spec          *specs.Spec
+		expectedError bool
 	}{
 		{
-			candidates: []string{},
+			description: "empty config no error",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{},
+			},
 		},
 		{
-			candidates: []string{"not-runc"},
-		},
-		{
-			candidates: []string{"not-runc", "also-not-runc"},
-		},
-		{
-			candidates:   []string{runcExecutableName},
-			expectedPath: filepath.Join(cfg.binPath, runcExecutableName),
-		},
-		{
-			candidates:   []string{runcExecutableName, "not-runc"},
-			expectedPath: filepath.Join(cfg.binPath, runcExecutableName),
-		},
-		{
-			candidates:   []string{"not-runc", runcExecutableName},
-			expectedPath: filepath.Join(cfg.binPath, runcExecutableName),
+			description: "experimental flag supported",
+			cfg: &config.Config{
+				NVIDIAContainerRuntimeConfig: config.RuntimeConfig{
+					Experimental: true,
+					DiscoverMode: "legacy",
+				},
+			},
+			spec: &specs.Spec{
+				Process: &specs.Process{
+					Env: []string{
+						"NVIDIA_VISIBLE_DEVICES=all",
+					},
+				},
+			},
 		},
 	}
 
-	for i, tc := range testCases {
-		runcPath, err := findRuntime(tc.candidates)
-		if tc.expectedPath == "" {
-			require.Error(t, err, "%d: %v", i, tc)
-		} else {
-			require.NoError(t, err, "%d: %v", i, tc)
-		}
-		require.Equal(t, tc.expectedPath, runcPath, "%d: %v", i, tc)
-	}
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			bundleDir := t.TempDir()
 
+			specFile, err := os.Create(filepath.Join(bundleDir, "config.json"))
+			require.NoError(t, err)
+			require.NoError(t, json.NewEncoder(specFile).Encode(tc.spec))
+
+			argv := []string{"--bundle", bundleDir}
+
+			_, err = newNVIDIAContainerRuntime(logger, tc.cfg, argv)
+			if tc.expectedError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
 }

cmd/nvidia-container-toolkit/capabilities.go

@@ -2,6 +2,15 @@ package main
 
 import (
 	"log"
+	"strings"
+)
+
+const (
+	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
+	defaultDriverCapabilities = DriverCapabilities("utility,compute")
+
+	none = DriverCapabilities("")
+	all  = DriverCapabilities("all")
 )
 
 func capabilityToCLI(cap string) string {
@@ -25,3 +34,50 @@ func capabilityToCLI(cap string) string {
 	}
 	return ""
 }
+
+// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
+// variable. Operations include default values, filtering, and handling meta values such as "all"
+type DriverCapabilities string
+
+// Intersection returns intersection between two sets of capabilities.
+func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
+	if capabilities == all {
+		return d
+	}
+	if d == all {
+		return capabilities
+	}
+
+	lookup := make(map[string]bool)
+	for _, c := range d.list() {
+		lookup[c] = true
+	}
+	var found []string
+	for _, c := range capabilities.list() {
+		if lookup[c] {
+			found = append(found, c)
+		}
+	}
+
+	intersection := DriverCapabilities(strings.Join(found, ","))
+	return intersection
+}
+
+// String returns the string representation of the driver capabilities
+func (d DriverCapabilities) String() string {
+	return string(d)
+}
+
+// list returns the driver capabilities as a list
+func (d DriverCapabilities) list() []string {
+	var caps []string
+	for _, c := range strings.Split(string(d), ",") {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) == 0 {
+			continue
+		}
+		caps = append(caps, trimmed)
+	}
+
+	return caps
+}

cmd/nvidia-container-toolkit/capabilities_test.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDriverCapabilitiesIntersection(t *testing.T) {
+	testCases := []struct {
+		capabilities          DriverCapabilities
+		supportedCapabilities DriverCapabilities
+		expectedIntersection  DriverCapabilities
+	}{
+		{
+			capabilities:          none,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          allDriverCapabilities,
+			supportedCapabilities: all,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: all,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: DriverCapabilities("cap1"),
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("cap0,cap1"),
+			supportedCapabilities: DriverCapabilities("cap1,cap0"),
+			expectedIntersection:  DriverCapabilities("cap0,cap1"),
+		},
+		{
+			capabilities:          defaultDriverCapabilities,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  defaultDriverCapabilities,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+		{
+			capabilities:          DriverCapabilities("cap1"),
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
+			require.EqualValues(t, tc.expectedIntersection, intersection)
+		})
+	}
+}
+
+func TestDriverCapabilitiesList(t *testing.T) {
+	testCases := []struct {
+		capabilities DriverCapabilities
+		expected     []string
+	}{
+		{
+			capabilities: DriverCapabilities(""),
+		},
+		{
+			capabilities: DriverCapabilities("  "),
+		},
+		{
+			capabilities: DriverCapabilities(","),
+		},
+		{
+			capabilities: DriverCapabilities(",cap"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap,"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap0,,cap1"),
+			expected:     []string{"cap0", "cap1"},
+		},
+		{
+			capabilities: DriverCapabilities("cap1,cap0,cap3"),
+			expected:     []string{"cap1", "cap0", "cap3"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			require.EqualValues(t, tc.expected, tc.capabilities.list())
+		})
+	}
+}

cmd/nvidia-container-toolkit/container_config.go

@@ -26,11 +26,6 @@ const (
 	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
-const (
-	allDriverCapabilities     = "compute,compat32,graphics,utility,video,display,ngx"
-	defaultDriverCapabilities = "utility,compute"
-)
-
 const (
 	capSysAdmin = "CAP_SYS_ADMIN"
 )
@@ -316,33 +311,27 @@ func getMigMonitorDevices(env map[string]string) *string {
 	return nil
 }
 
-func getDriverCapabilities(env map[string]string, legacyImage bool) *string {
-	// Grab a reference to the capabilities from the envvar
-	// if it actually exists in the environment.
-	var capabilities *string
-	if caps, ok := env[envNVDriverCapabilities]; ok {
-		capabilities = &caps
+func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
+	// We use the default driver capabilities by default. This is filtered to only include the
+	// supported capabilities
+	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
+
+	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
+
+	if !capsEnvSpecified && legacyImage {
+		// Environment variable unset with legacy image: set all capabilities.
+		return supportedDriverCapabilities
 	}
 
-	// Environment variable unset with legacy image: set all capabilities.
-	if capabilities == nil && legacyImage {
-		allCaps := allDriverCapabilities
-		return &allCaps
+	if capsEnvSpecified && len(capsEnv) > 0 {
+		// If the envvironment variable is specified and is non-empty, use the capabilities value
+		envCapabilities := DriverCapabilities(capsEnv)
+		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
+		if envCapabilities != all && capabilities != envCapabilities {
+			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
+		}
 	}
 
-	// Environment variable unset or set but empty: set default capabilities.
-	if capabilities == nil || len(*capabilities) == 0 {
-		defaultCaps := defaultDriverCapabilities
-		return &defaultCaps
-	}
-
-	// Environment variable set to "all": set all capabilities.
-	if *capabilities == "all" {
-		allCaps := allDriverCapabilities
-		return &allCaps
-	}
-
-	// Any other value
 	return capabilities
 }
 
@@ -389,10 +378,7 @@ func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mou
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	var driverCapabilities string
-	if c := getDriverCapabilities(env, legacyImage); c != nil {
-		driverCapabilities = *c
-	}
+	driverCapabilities := getDriverCapabilities(env, hookConfig.SupportedDriverCapabilities, legacyImage).String()
 
 	requirements := getRequirements(env, legacyImage)
 

cmd/nvidia-container-toolkit/container_config_test.go

@@ -12,6 +12,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 		description    string
 		env            map[string]string
 		privileged     bool
+		hookConfig     *HookConfig
 		expectedConfig *nvidiaConfig
 		expectedPanic  bool
 	}{
@@ -35,7 +36,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -49,7 +50,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -81,7 +82,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -95,7 +96,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -110,7 +111,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -125,7 +126,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -135,12 +136,12 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envCUDAVersion:          "9.0",
 				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "cap0,cap1",
+				envNVDriverCapabilities: "video,display",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -150,14 +151,14 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envCUDAVersion:              "9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     false,
 			},
@@ -167,7 +168,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envCUDAVersion:              "9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 				envNVDisableRequire:         "true",
@@ -175,7 +176,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     true,
 			},
@@ -206,7 +207,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -238,7 +239,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -252,7 +253,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -267,7 +268,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -282,7 +283,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -292,12 +293,12 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envNVRequireCUDA:        "cuda>=9.0",
 				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "cap0,cap1",
+				envNVDriverCapabilities: "video,display",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -307,14 +308,14 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envNVRequireCUDA:            "cuda>=9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     false,
 			},
@@ -324,7 +325,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envNVRequireCUDA:            "cuda>=9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 				envNVDisableRequire:         "true",
@@ -332,7 +333,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     true,
 			},
@@ -346,7 +347,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{},
 				DisableRequire:     false,
 			},
@@ -362,7 +363,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigConfigDevices:   "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -388,7 +389,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigMonitorDevices:  "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -403,14 +404,62 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged:    false,
 			expectedPanic: true,
 		},
+		{
+			description: "Hook config set as driver-capabilities-all",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				envNVDriverCapabilities: "all",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: "video,display",
+			},
+		},
+		{
+			description: "Hook config set, envvar sets driver-capabilities",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				envNVDriverCapabilities: "video,display",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display,compute,utility",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: "video,display",
+			},
+		},
+		{
+			description: "Hook config set, envvar unset sets default driver-capabilities",
+			env: map[string]string{
+				envNVVisibleDevices: "all",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+			},
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			// Wrap the call to getNvidiaConfig() in a closure.
 			var config *nvidiaConfig
 			getConfig := func() {
-				hookConfig := getDefaultHookConfig()
-				config = getNvidiaConfig(&hookConfig, tc.env, nil, tc.privileged)
+				hookConfig := tc.hookConfig
+				if hookConfig == nil {
+					defaultConfig := getDefaultHookConfig()
+					hookConfig = &defaultConfig
+				}
+				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
 			}
 
 			// For any tests that are expected to panic, make sure they do.
@@ -822,3 +871,119 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 		})
 	}
 }
+
+func TestGetDriverCapabilities(t *testing.T) {
+
+	supportedCapabilities := "compute,utility,display,video"
+
+	testCases := []struct {
+		description           string
+		env                   map[string]string
+		legacyImage           bool
+		supportedCapabilities string
+		expectedPanic         bool
+		expectedCapabilities  string
+	}{
+		{
+			description: "Env is set for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "display,video",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description: "Env is all for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "all",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description:           "Env unset for legacy image is 'all'",
+			env:                   map[string]string{},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is set for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "display,video",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description:           "Env unset for modern image is default",
+			env:                   map[string]string{},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description: "Env is all for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "all",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description: "Invalid capabilities panic",
+			env: map[string]string{
+				envNVDriverCapabilities: "compute,utility",
+			},
+			supportedCapabilities: "not-compute,not-utility",
+			expectedPanic:         true,
+		},
+		{
+			description:           "Default is restricted for modern image",
+			legacyImage:           false,
+			supportedCapabilities: "compute",
+			expectedCapabilities:  "compute",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			var capabilites DriverCapabilities
+
+			getDriverCapabilities := func() {
+				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
+				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getDriverCapabilities)
+				return
+			}
+
+			getDriverCapabilities()
+			require.EqualValues(t, tc.expectedCapabilities, capabilites)
+		})
+	}
+}

cmd/nvidia-container-toolkit/hook_config.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 
 	"github.com/BurntSushi/toml"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 )
 
 const (
@@ -35,20 +36,23 @@ type CLIConfig struct {
 
 // HookConfig : options for the nvidia-container-toolkit.
 type HookConfig struct {
-	DisableRequire                 bool    `toml:"disable-require"`
-	SwarmResource                  *string `toml:"swarm-resource"`
-	AcceptEnvvarUnprivileged       bool    `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-	AcceptDeviceListAsVolumeMounts bool    `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	DisableRequire                 bool               `toml:"disable-require"`
+	SwarmResource                  *string            `toml:"swarm-resource"`
+	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
-	NvidiaContainerCLI CLIConfig `toml:"nvidia-container-cli"`
+	NvidiaContainerCLI     CLIConfig            `toml:"nvidia-container-cli"`
+	NVIDIAContainerRuntime config.RuntimeConfig `toml:"nvidia-container-runtime"`
 }
 
-func getDefaultHookConfig() (config HookConfig) {
+func getDefaultHookConfig() HookConfig {
 	return HookConfig{
 		DisableRequire:                 false,
 		SwarmResource:                  nil,
 		AcceptEnvvarUnprivileged:       true,
 		AcceptDeviceListAsVolumeMounts: false,
+		SupportedDriverCapabilities:    allDriverCapabilities,
 		NvidiaContainerCLI: CLIConfig{
 			Root:        nil,
 			Path:        nil,
@@ -61,6 +65,7 @@ func getDefaultHookConfig() (config HookConfig) {
 			User:        nil,
 			Ldconfig:    nil,
 		},
+		NVIDIAContainerRuntime: *config.GetDefaultRuntimeConfig(),
 	}
 }
 
@@ -85,6 +90,15 @@ func getHookConfig() (config HookConfig) {
 		}
 	}
 
+	if config.SupportedDriverCapabilities == all {
+		config.SupportedDriverCapabilities = allDriverCapabilities
+	}
+	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
+	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
+	}
+
 	return config
 }
 

cmd/nvidia-container-toolkit/hook_config_test.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHookConfig(t *testing.T) {
+	testCases := []struct {
+		lines                      []string
+		expectedPanic              bool
+		expectedDriverCapabilities DriverCapabilities
+	}{
+		{
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"all\"",
+			},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"compute,utility,not-compute\"",
+			},
+			expectedPanic: true,
+		},
+		{
+			lines:                      []string{},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"\"",
+			},
+			expectedDriverCapabilities: none,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"utility,compute\"",
+			},
+			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			var filename string
+			defer func() {
+				if len(filename) > 0 {
+					os.Remove(filename)
+				}
+				configflag = nil
+			}()
+
+			if tc.lines != nil {
+				configFile, err := os.CreateTemp("", "*.toml")
+				require.NoError(t, err)
+				defer configFile.Close()
+
+				filename = configFile.Name()
+				configflag = &filename
+
+				for _, line := range tc.lines {
+					_, err := configFile.WriteString(fmt.Sprintf("%s\n", line))
+					require.NoError(t, err)
+				}
+			}
+
+			var config HookConfig
+			getHookConfig := func() {
+				config = getHookConfig()
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getHookConfig)
+				return
+			}
+
+			getHookConfig()
+
+			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+		})
+	}
+}

cmd/nvidia-container-toolkit/main.go

@@ -17,6 +17,7 @@ import (
 
 var (
 	debugflag  = flag.Bool("debug", false, "enable debug output")
+	forceflag  = flag.Bool("force", false, "force execution of prestart hook in experimental mode")
 	configflag = flag.String("config", "", "configuration file")
 
 	defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
@@ -85,6 +86,10 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
+	if hook.NVIDIAContainerRuntime.Experimental && !*forceflag {
+		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.")
+	}
+
 	container := getContainerConfig(hook)
 	nvidia := container.Nvidia
 	if nvidia == nil {

cmd/nvidia-ctk/README.md

@@ -0,0 +1,3 @@
+# NVIDIA Container Toolkit CLI
+
+The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -0,0 +1,209 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package symlinks
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	hostRoot      string
+	filenames     cli.StringSlice
+	containerSpec string
+}
+
+// NewCommand constructs a hook command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the '' command
+	c := cli.Command{
+		Name:  "create-symlinks",
+		Usage: "A hook to create symlinks in the container. This can be used to proces CSV mount specs",
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "The root on the host filesystem to use to resolve symlinks",
+			Destination: &cfg.hostRoot,
+		},
+		&cli.StringSliceFlag{
+			Name:        "csv-filenames",
+			Aliases:     []string{"f"},
+			Usage:       "Specify the (CSV) filenames to process",
+			Destination: &cfg.filenames,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	spec, err := s.LoadSpec()
+	if err != nil {
+		return fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	var containerRoot string
+	if spec.Root != nil {
+		containerRoot = spec.Root.Path
+	}
+
+	csvFiles := cfg.filenames.Value()
+
+	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
+
+	var candidates []string
+	for _, file := range csvFiles {
+		mountSpecs, err := csv.NewCSVFileParser(m.logger, file).Parse()
+		if err != nil {
+			m.logger.Debugf("Skipping CSV file %v: %v", file, err)
+			continue
+		}
+
+		for _, ms := range mountSpecs {
+			if ms.Type != csv.MountSpecSym {
+				continue
+			}
+			targets, err := chainLocator.Locate(ms.Path)
+			if err != nil {
+				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
+			}
+			candidates = append(candidates, targets...)
+		}
+	}
+
+	created := make(map[string]bool)
+	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
+	for _, candidate := range candidates {
+		targets, err := m.Locate(candidate)
+		if err != nil {
+			m.logger.Debugf("Skipping invalid link: %v", err)
+			continue
+		} else if len(targets) != 1 {
+			m.logger.Debugf("Unexepected number of targets: %v", targets)
+			continue
+		} else if targets[0] == candidate {
+			m.logger.Debugf("%v is not a symlink", candidate)
+			continue
+		}
+		target, err := changeRoot(cfg.hostRoot, "/", targets[0])
+		if err != nil {
+			m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, cfg.hostRoot, err)
+			continue
+		}
+
+		linkPath, err := changeRoot(cfg.hostRoot, containerRoot, candidate)
+		if err != nil {
+			m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", candidate, cfg.hostRoot, err)
+			continue
+		}
+
+		if created[linkPath] {
+			m.logger.Debugf("Link %v already created", linkPath)
+			continue
+		}
+		m.logger.Infof("Symlinking %v to %v", linkPath, target)
+		err = os.MkdirAll(filepath.Dir(linkPath), 0755)
+		if err != nil {
+			m.logger.Warnf("Faild to create directory: %v", err)
+			continue
+		}
+		err = os.Symlink(target, linkPath)
+		if err != nil {
+			m.logger.Warnf("Failed to create symlink: %v", err)
+			continue
+		}
+		created[linkPath] = true
+	}
+
+	return nil
+
+}
+
+func changeRoot(current string, new string, path string) (string, error) {
+	if !filepath.IsAbs(path) {
+		return path, nil
+	}
+
+	relative := path
+	if current != "" {
+		r, err := filepath.Rel(current, path)
+		if err != nil {
+			return "", err
+		}
+		relative = r
+	}
+
+	return filepath.Join(new, relative), nil
+}
+
+// Locate returns the link target of the specified filename or an empty slice if the
+// specified filename is not a symlink.
+func (m command) Locate(filename string) ([]string, error) {
+	info, err := os.Lstat(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get file info: %v", info)
+	}
+	if info.Mode()&os.ModeSymlink == 0 {
+		m.logger.Debugf("%v is not a symlink", filename)
+		return nil, nil
+	}
+
+	target, err := os.Readlink(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error checking symlink: %v", err)
+	}
+
+	m.logger.Debugf("Resolved link: '%v' => '%v'", filename, target)
+
+	return []string{target}, nil
+}

cmd/nvidia-ctk/hook/hook.go

@@ -0,0 +1,52 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package hook
+
+import (
+	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type hookCommand struct {
+	logger *logrus.Logger
+}
+
+// NewCommand constructs a hook command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := hookCommand{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build
+func (m hookCommand) build() *cli.Command {
+	// Create the 'hook' command
+	hook := cli.Command{
+		Name:  "hook",
+		Usage: "A collection of hooks that may be injected into an OCI spec",
+	}
+
+	hook.Subcommands = []*cli.Command{
+		ldcache.NewCommand(m.logger),
+		symlinks.NewCommand(m.logger),
+	}
+
+	return &hook
+}

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package ldcache
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type command struct {
+	logger *logrus.Logger
+}
+
+type config struct {
+	folders       cli.StringSlice
+	containerSpec string
+}
+
+// NewCommand constructs an update-ldcache command with the specified logger
+func NewCommand(logger *logrus.Logger) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the update-ldcache command
+func (m command) build() *cli.Command {
+	cfg := config{}
+
+	// Create the 'update-ldcache' command
+	c := cli.Command{
+		Name:  "update-ldcache",
+		Usage: "Update ldcache in a container by running ldconfig",
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "folders",
+			Usage:       "Specifiy the additional folders to add to /etc/ld.so.conf before updating the ld cache",
+			Destination: &cfg.folders,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) run(c *cli.Context, cfg *config) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	spec, err := s.LoadSpec()
+	if err != nil {
+		return fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	var containerRoot string
+	if spec.Root != nil {
+		containerRoot = spec.Root.Path
+	}
+
+	err = m.createConfig(containerRoot, cfg.folders.Value())
+	if err != nil {
+		return fmt.Errorf("failed to update ld.so.conf: %v", err)
+	}
+
+	args := []string{"/sbin/ldconfig"}
+	if containerRoot != "" {
+		args = append(args, "-r", containerRoot)
+	}
+
+	return syscall.Exec(args[0], args, nil)
+}
+
+// createConfig creates (or updates) /etc/ld.so.conf.d/nvcr-<RANDOM_STRING>.conf in the container
+// to include the required paths.
+func (m command) createConfig(root string, folders []string) error {
+	if len(folders) == 0 {
+		m.logger.Debugf("No folders to add to /etc/ld.so.conf")
+		return nil
+	}
+
+	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
+	if err != nil {
+		return fmt.Errorf("failed to create config file: %v", err)
+	}
+	defer configFile.Close()
+
+	m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name())
+
+	configured := make(map[string]bool)
+	for _, folder := range folders {
+		if configured[folder] {
+			continue
+		}
+		_, err = configFile.WriteString(fmt.Sprintf("%s\n", folder))
+		if err != nil {
+			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
+		}
+		configured[folder] = true
+	}
+
+	return nil
+}

cmd/nvidia-ctk/main.go

@@ -0,0 +1,81 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+var version string
+
+var logger = log.New()
+
+// config defines the options that can be set for the CLI through config files,
+// environment variables, or command line flags
+type config struct {
+	// Debug indicates whether the CLI is started in "debug" mode
+	Debug bool
+}
+
+func main() {
+	// Create a config struct to hold the parsed environment variables or command line flags
+	config := config{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.UseShortOptionHandling = true
+	c.EnableBashCompletion = true
+	c.Usage = "Tools to configure the NVIDIA Container Toolkit"
+	c.Version = version
+
+	// Setup the flags for this command
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "debug",
+			Aliases:     []string{"d"},
+			Usage:       "Enable debug-level logging",
+			Destination: &config.Debug,
+			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
+		},
+	}
+
+	// Set log-level for all subcommands
+	c.Before = func(c *cli.Context) error {
+		logLevel := log.InfoLevel
+		if config.Debug {
+			logLevel = log.DebugLevel
+		}
+		logger.SetLevel(logLevel)
+		return nil
+	}
+
+	// Define the subcommands
+	c.Commands = []*cli.Command{
+		hook.NewCommand(logger),
+	}
+
+	// Run the CLI
+	err := c.Run(os.Args)
+	if err != nil {
+		log.Errorf("%v", err)
+		log.Exit(1)
+	}
+}

config/config.toml.amazonlinux

@@ -16,3 +16,4 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+#experimental = false

config/config.toml.centos

@@ -16,3 +16,4 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+#experimental = false

config/config.toml.debian

@@ -16,3 +16,4 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+#experimental = false

config/config.toml.opensuse-leap

@@ -16,3 +16,4 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+#experimental = false

config/config.toml.ubuntu

@@ -16,3 +16,4 @@ ldconfig = "@/sbin/ldconfig.real"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
+#experimental = false

docker/Dockerfile.amazonlinux

@@ -42,7 +42,9 @@ COPY . .
 
 RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.amzn $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 # This might not be useful on Amazon Linux, but it's simpler to keep the RHEL

docker/Dockerfile.centos

@@ -3,6 +3,7 @@ FROM ${BASEIMAGE}
 
 RUN yum install -y \
         ca-certificates \
+        gcc \
         wget \
         git \
         make \
@@ -42,7 +43,9 @@ COPY . .
 
 RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.centos $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook

docker/Dockerfile.debian

@@ -50,7 +50,9 @@ COPY . .
 
 RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.debian $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Debian Jessie still had ldconfig.real
 RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
@@ -61,6 +63,8 @@ WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
 RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \

docker/Dockerfile.opensuse-leap

@@ -47,7 +47,9 @@ COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
 # Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
 COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 
-COPY config/config.toml.opensuse-leap $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .

docker/Dockerfile.ubuntu

@@ -48,12 +48,16 @@ COPY . .
 
 RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.ubuntu $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
 RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \

docker/docker.mk

@@ -14,7 +14,7 @@
 
 # Supported OSs by architecture
 AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
-X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux1 amazonlinux2 opensuse-leap15.1
+X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
 AARCH64_TARGETS := centos8 rhel8 amazonlinux2
@@ -98,6 +98,7 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 # private centos target
 --centos%: OS := centos
 --centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+--centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
@@ -113,20 +114,27 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 --rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
+--rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
+
+# We allow the CONFIG_TOML_SUFFIX to be overridden.
+CONFIG_TOML_SUFFIX ?= $(OS)
 
 docker-build-%:
 	@echo "Building for $(TARGET_PLATFORM)"
 	docker pull --platform=linux/$(ARCH) $(BASEIMAGE)
 	DOCKER_BUILDKIT=1 \
 	$(DOCKER) build \
+	    --platform=linux/$(ARCH) \
 	    --progress=plain \
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 	    --build-arg PKG_VERS="$(LIB_VERSION)" \
 	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \
+	    --platform=linux/$(ARCH) \
 	    -e DISTRIB \
 	    -e SECTION \
 	    -v $(ARTIFACTS_DIR):/dist \

go.mod

@@ -3,11 +3,15 @@ module github.com/NVIDIA/nvidia-container-toolkit
 go 1.14
 
 require (
-	github.com/BurntSushi/toml v0.3.1
-	github.com/opencontainers/runtime-spec v1.0.2
-	github.com/pelletier/go-toml v1.9.3
+	github.com/BurntSushi/toml v1.0.0
+	github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b
+	github.com/containers/podman/v4 v4.0.1
+	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
+	github.com/pelletier/go-toml v1.9.4
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
-	golang.org/x/mod v0.3.0
+	github.com/urfave/cli/v2 v2.3.0
+	golang.org/x/mod v0.5.0
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 )

internal/config/cli.go

@@ -0,0 +1,48 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"github.com/pelletier/go-toml"
+)
+
+// ContainerCLIConfig stores the options for the nvidia-container-cli
+type ContainerCLIConfig struct {
+	Root string
+}
+
+// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
+	cfg := getDefaultContainerCLIConfig()
+
+	if toml == nil {
+		return cfg
+	}
+
+	cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
+
+	return cfg
+}
+
+// getDefaultContainerCLIConfig defines the default values for the config
+func getDefaultContainerCLIConfig() *ContainerCLIConfig {
+	c := ContainerCLIConfig{
+		Root: "",
+	}
+
+	return &c
+}

internal/config/config.go

@@ -0,0 +1,110 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/pelletier/go-toml"
+)
+
+const (
+	configOverride = "XDG_CONFIG_HOME"
+	configFilePath = "nvidia-container-runtime/config.toml"
+)
+
+var (
+	// DefaultExecutableDir specifies the default path to use for executables if they cannot be located in the path.
+	DefaultExecutableDir = "/usr/bin"
+
+	// NVIDIAContainerRuntimeHookExecutable is the executable name for the NVIDIA Container Runtime Hook
+	NVIDIAContainerRuntimeHookExecutable = "nvidia-container-runtime-hook"
+	// NVIDIAContainerToolkitExecutable is the executable name for the NVIDIA Container Toolkit (an alias for the NVIDIA Container Runtime Hook)
+	NVIDIAContainerToolkitExecutable = "nvidia-container-toolkit"
+
+	configDir = "/etc/"
+)
+
+// Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
+// Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
+type Config struct {
+	NVIDIAContainerCLIConfig     ContainerCLIConfig `toml:"nvidia-container-cli"`
+	NVIDIACTKConfig              CTKConfig          `toml:"nvidia-ctk"`
+	NVIDIAContainerRuntimeConfig RuntimeConfig      `toml:"nvidia-container-runtime"`
+}
+
+// GetConfig sets up the config struct. Values are read from a toml file
+// or set via the environment.
+func GetConfig() (*Config, error) {
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		configDir = XDGConfigDir
+	}
+
+	configFilePath := path.Join(configDir, configFilePath)
+
+	tomlFile, err := os.Open(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open config file %v: %v", configFilePath, err)
+	}
+	defer tomlFile.Close()
+
+	cfg, err := loadConfigFrom(tomlFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config values: %v", err)
+	}
+
+	return cfg, nil
+}
+
+// loadRuntimeConfigFrom reads the config from the specified Reader
+func loadConfigFrom(reader io.Reader) (*Config, error) {
+	toml, err := toml.LoadReader(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	return getConfigFrom(toml), nil
+}
+
+// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getConfigFrom(toml *toml.Tree) *Config {
+	cfg := getDefaultConfig()
+
+	if toml == nil {
+		return cfg
+	}
+
+	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
+	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
+	cfg.NVIDIAContainerRuntimeConfig = *getRuntimeConfigFrom(toml)
+
+	return cfg
+}
+
+// getDefaultConfig defines the default values for the config
+func getDefaultConfig() *Config {
+	c := Config{
+		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
+		NVIDIACTKConfig:              *getDefaultCTKConfig(),
+		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
+	}
+
+	return &c
+}

internal/config/config_test.go

@@ -0,0 +1,143 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetConfigWithCustomConfig(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// By default debug is disabled
+	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
+	testDir := filepath.Join(wd, "test")
+	filename := filepath.Join(testDir, configFilePath)
+
+	os.Setenv(configOverride, testDir)
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
+
+	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+
+	cfg, err := GetConfig()
+	require.NoError(t, err)
+	require.Equal(t, cfg.NVIDIAContainerRuntimeConfig.DebugFilePath, "/nvidia-container-toolkit.log")
+}
+
+func TestGetConfig(t *testing.T) {
+	testCases := []struct {
+		description    string
+		contents       []string
+		expectedError  error
+		expectedConfig *Config
+	}{
+		{
+			description: "empty config is default",
+			expectedConfig: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root: "",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/dev/null",
+					Experimental:  false,
+					DiscoverMode:  "auto",
+					LogLevel:      "info",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "nvidia-ctk",
+				},
+			},
+		},
+		{
+			description: "config options set inline",
+			contents: []string{
+				"nvidia-container-cli.root = \"/bar/baz\"",
+				"nvidia-container-runtime.debug = \"/foo/bar\"",
+				"nvidia-container-runtime.experimental = true",
+				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
+				"nvidia-container-runtime.log-level = \"debug\"",
+				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
+			},
+			expectedConfig: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root: "/bar/baz",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/foo/bar",
+					Experimental:  true,
+					DiscoverMode:  "not-legacy",
+					LogLevel:      "debug",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "/foo/bar/nvidia-ctk",
+				},
+			},
+		},
+		{
+			description: "config options set in section",
+			contents: []string{
+				"[nvidia-container-cli]",
+				"root = \"/bar/baz\"",
+				"[nvidia-container-runtime]",
+				"debug = \"/foo/bar\"",
+				"experimental = true",
+				"discover-mode = \"not-legacy\"",
+				"log-level = \"debug\"",
+				"[nvidia-ctk]",
+				"path = \"/foo/bar/nvidia-ctk\"",
+			},
+			expectedConfig: &Config{
+				NVIDIAContainerCLIConfig: ContainerCLIConfig{
+					Root: "/bar/baz",
+				},
+				NVIDIAContainerRuntimeConfig: RuntimeConfig{
+					DebugFilePath: "/foo/bar",
+					Experimental:  true,
+					DiscoverMode:  "not-legacy",
+					LogLevel:      "debug",
+				},
+				NVIDIACTKConfig: CTKConfig{
+					Path: "/foo/bar/nvidia-ctk",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
+
+			cfg, err := loadConfigFrom(reader)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.EqualValues(t, tc.expectedConfig, cfg)
+		})
+	}
+}

internal/config/runtime.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import (
+	"github.com/pelletier/go-toml"
+	"github.com/sirupsen/logrus"
+)
+
+// RuntimeConfig stores the config options for the NVIDIA Container Runtime
+type RuntimeConfig struct {
+	DebugFilePath string
+	Experimental  bool
+	DiscoverMode  string
+	// LogLevel defines the logging level for the application
+	LogLevel string
+}
+
+// getRuntimeConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getRuntimeConfigFrom(toml *toml.Tree) *RuntimeConfig {
+	cfg := GetDefaultRuntimeConfig()
+
+	if toml == nil {
+		return cfg
+	}
+
+	cfg.DebugFilePath = toml.GetDefault("nvidia-container-runtime.debug", cfg.DebugFilePath).(string)
+	cfg.Experimental = toml.GetDefault("nvidia-container-runtime.experimental", cfg.Experimental).(bool)
+	cfg.DiscoverMode = toml.GetDefault("nvidia-container-runtime.discover-mode", cfg.DiscoverMode).(string)
+	cfg.LogLevel = toml.GetDefault("nvidia-container-runtime.log-level", cfg.LogLevel).(string)
+
+	return cfg
+}
+
+// GetDefaultRuntimeConfig defines the default values for the config
+func GetDefaultRuntimeConfig() *RuntimeConfig {
+	c := RuntimeConfig{
+		DebugFilePath: "/dev/null",
+		Experimental:  false,
+		DiscoverMode:  "auto",
+		LogLevel:      logrus.InfoLevel.String(),
+	}
+
+	return &c
+}

internal/config/toolkit-cli.go

@@ -0,0 +1,46 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package config
+
+import "github.com/pelletier/go-toml"
+
+// CTKConfig stores the config options for the NVIDIA Container Toolkit CLI (nvidia-ctk)
+type CTKConfig struct {
+	Path string `toml:"path"`
+}
+
+// getCTKConfigFrom reads the nvidia container runtime config from the specified toml Tree.
+func getCTKConfigFrom(toml *toml.Tree) *CTKConfig {
+	cfg := getDefaultCTKConfig()
+
+	if toml == nil {
+		return cfg
+	}
+
+	cfg.Path = toml.GetDefault("nvidia-ctk.path", cfg.Path).(string)
+
+	return cfg
+}
+
+// getDefaultCTKConfig defines the default values for the config
+func getDefaultCTKConfig() *CTKConfig {
+	c := CTKConfig{
+		Path: "nvidia-ctk",
+	}
+
+	return &c
+}

internal/discover/csv.go

@@ -0,0 +1,139 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied.
+// The constructed discoverer is comprised of a list, with each element in the list being associated with a
+// single CSV files.
+func NewFromCSVFiles(logger *logrus.Logger, files []string, root string) (Discover, error) {
+	if len(files) == 0 {
+		logger.Warnf("No CSV files specified")
+		return None{}, nil
+	}
+
+	symlinkLocator := lookup.NewSymlinkLocator(logger, root)
+	locators := map[csv.MountSpecType]lookup.Locator{
+		csv.MountSpecDev: lookup.NewCharDeviceLocator(logger, root),
+		csv.MountSpecDir: lookup.NewDirectoryLocator(logger, root),
+		// Libraries and symlinks are handled in the same way
+		csv.MountSpecLib: symlinkLocator,
+		csv.MountSpecSym: symlinkLocator,
+	}
+
+	var discoverers []Discover
+	for _, filename := range files {
+		d, err := NewFromCSVFile(logger, locators, filename)
+		if err != nil {
+			logger.Warnf("Skipping CSV file %v: %v", filename, err)
+			continue
+		}
+		discoverers = append(discoverers, d)
+	}
+
+	return &list{discoverers: discoverers}, nil
+}
+
+// NewFromCSVFile creates a discoverer for the specified CSV file. A logger is also supplied.
+// The constructed discoverer is comprised of a list, with each element in the list being associated with a particular
+// MountSpecType.
+func NewFromCSVFile(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, filename string) (Discover, error) {
+	// Create a discoverer for each file-kind combination
+	targets, err := csv.NewCSVFileParser(logger, filename).Parse()
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CSV file: %v", err)
+	}
+	if len(targets) == 0 {
+		return nil, fmt.Errorf("CSV file is empty")
+	}
+
+	return newFromMountSpecs(logger, locators, targets)
+}
+
+// newFromMountSpecs creates a discoverer for the CSV file. A logger is also supplied.
+// A list of csvDiscoverers is returned, with each being associated with a single MountSpecType.
+func newFromMountSpecs(logger *logrus.Logger, locators map[csv.MountSpecType]lookup.Locator, targets []*csv.MountSpec) (Discover, error) {
+	if len(targets) == 0 {
+		return &None{}, nil
+	}
+
+	var discoverers []Discover
+	var mountSpecTypes []csv.MountSpecType
+	candidatesByType := make(map[csv.MountSpecType][]string)
+	for _, t := range targets {
+		if _, exists := candidatesByType[t.Type]; !exists {
+			mountSpecTypes = append(mountSpecTypes, t.Type)
+		}
+		candidatesByType[t.Type] = append(candidatesByType[t.Type], t.Path)
+	}
+
+	for _, t := range mountSpecTypes {
+		locator, exists := locators[t]
+		if !exists {
+			return nil, fmt.Errorf("no locator defined for '%v'", t)
+		}
+
+		m := &mounts{
+			logger:   logger,
+			lookup:   locator,
+			required: candidatesByType[t],
+		}
+
+		switch t {
+		case csv.MountSpecDev:
+			// For device mount specs, we insert a charDevices into the list of discoverers.
+			discoverers = append(discoverers, (*charDevices)(m))
+		default:
+			discoverers = append(discoverers, m)
+		}
+	}
+
+	return &list{discoverers: discoverers}, nil
+}
+
+// Mounts returns the discovered mounts for the charDevices. Since this explicitly specifies a
+// device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices. Here the device nodes are first
+// discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	devicesAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range devicesAsMounts {
+		devices = append(devices, Device(mount))
+	}
+
+	return devices, nil
+}

internal/discover/csv/csv.go

@@ -0,0 +1,131 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// DefaultMountSpecPath is default location of CSV files that define the modifications required to the OCI spec
+	DefaultMountSpecPath = "/etc/nvidia-container-runtime/host-files-for-container.d"
+)
+
+// GetFileList returns the (non-recursive) list of CSV files in the specified
+// folder
+func GetFileList(root string) ([]string, error) {
+	contents, err := os.ReadDir(root)
+	if err != nil && errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to read the contents of %v: %v", root, err)
+	}
+
+	var csvFilePaths []string
+	for _, c := range contents {
+		if c.IsDir() {
+			continue
+		}
+		if c.Name() == ".csv" {
+			continue
+		}
+		ext := strings.ToLower(filepath.Ext(c.Name()))
+		if ext != ".csv" {
+			continue
+		}
+
+		csvFilePaths = append(csvFilePaths, filepath.Join(root, c.Name()))
+	}
+
+	return csvFilePaths, nil
+}
+
+// BaseFilesOnly filters out non-base CSV files from the list of CSV files.
+func BaseFilesOnly(filenames []string) []string {
+	filter := map[string]bool{
+		"l4t.csv":     true,
+		"drivers.csv": true,
+		"devices.csv": true,
+	}
+
+	var selected []string
+	for _, file := range filenames {
+		base := filepath.Base(file)
+		if filter[base] {
+			selected = append(selected, file)
+		}
+	}
+
+	return selected
+}
+
+// Parser specifies an interface for parsing MountSpecs
+type Parser interface {
+	Parse() ([]*MountSpec, error)
+}
+
+type csv struct {
+	logger   *logrus.Logger
+	filename string
+}
+
+// NewCSVFileParser creates a new parser for reading MountSpecs from the specified CSV file
+func NewCSVFileParser(logger *logrus.Logger, filename string) Parser {
+	p := csv{
+		logger:   logger,
+		filename: filename,
+	}
+
+	return &p
+}
+
+// Parse parses the csv file and returns a list of MountSpecs in the file
+func (p csv) Parse() ([]*MountSpec, error) {
+	reader, err := os.Open(p.filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %v for reading: %v", p.filename, err)
+	}
+	defer reader.Close()
+
+	return p.parseFromReader(reader), nil
+}
+
+// parseFromReader parses the specified file and returns a list of required jetson mounts
+func (p csv) parseFromReader(reader io.Reader) []*MountSpec {
+	var targets []*MountSpec
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+		target, err := NewMountSpecFromLine(line)
+		if err != nil {
+			p.logger.Debugf("Skipping invalid mount spec '%v': %v", line, err)
+			continue
+		}
+		targets = append(targets, target)
+	}
+
+	return targets
+}

internal/discover/csv/csv_test.go

@@ -0,0 +1,83 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetFileList(t *testing.T) {
+	moduleRoot, _ := test.GetModuleRoot()
+
+	testCases := []struct {
+		description   string
+		root          string
+		files         []string
+		expectedError error
+	}{
+		{
+			description: "returns list of CSV files",
+			root:        "test/input/csv_samples/",
+			files: []string{
+				"jetson.csv",
+				"simple_wrong.csv",
+				"simple.csv",
+				"spaced.csv",
+			},
+		},
+		{
+			description: "handles empty folder",
+			root:        "test/input/csv_samples/empty",
+		},
+		{
+			description: "handles non-existent folder",
+			root:        "test/input/csv_samples/NONEXISTENT",
+		},
+		{
+			description: "handles non-existent folder root",
+			root:        "/NONEXISTENT/test/input/csv_samples/",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			root := filepath.Join(moduleRoot, tc.root)
+			files, err := GetFileList(root)
+
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				require.Empty(t, files)
+				return
+			}
+
+			require.NoError(t, err)
+
+			var foundFiles []string
+			for _, f := range files {
+				require.Equal(t, root, filepath.Dir(f))
+				require.Equal(t, ".csv", filepath.Ext(f))
+				foundFiles = append(foundFiles, filepath.Base(f))
+			}
+
+			require.ElementsMatch(t, tc.files, foundFiles)
+		})
+	}
+}

internal/discover/csv/mount_spec.go

@@ -0,0 +1,74 @@
+/**
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"fmt"
+	"strings"
+)
+
+// MountSpecType defines the mount types allowed in a CSV file
+type MountSpecType string
+
+const (
+	// MountSpecDev is used for character devices
+	MountSpecDev = MountSpecType("dev")
+	// MountSpecDir is used for directories
+	MountSpecDir = MountSpecType("dir")
+	// MountSpecLib is used for libraries or regular files
+	MountSpecLib = MountSpecType("lib")
+	// MountSpecSym is used for symlinks.
+	MountSpecSym = MountSpecType("sym")
+)
+
+// MountSpec represents a Jetson mount consisting of a type and a path.
+type MountSpec struct {
+	Type MountSpecType
+	Path string
+}
+
+// NewMountSpecFromLine parses the specified line and returns the MountSpec or an error if the line is malformed
+func NewMountSpecFromLine(line string) (*MountSpec, error) {
+	parts := strings.SplitN(strings.TrimSpace(line), ",", 2)
+	if len(parts) < 2 {
+		return nil, fmt.Errorf("failed to parse line: %v", line)
+	}
+	mountType := strings.TrimSpace(parts[0])
+	path := strings.TrimSpace(parts[1])
+
+	return NewMountSpec(mountType, path)
+}
+
+// NewMountSpec creates a MountSpec with the specified type and path. An error is returned if the type is invalid.
+func NewMountSpec(mountType string, path string) (*MountSpec, error) {
+	mt := MountSpecType(mountType)
+	switch mt {
+	case MountSpecDev, MountSpecLib, MountSpecSym, MountSpecDir:
+	default:
+		return nil, fmt.Errorf("unexpected mount type: %v", mt)
+	}
+	if path == "" {
+		return nil, fmt.Errorf("invalid path: %v", path)
+	}
+
+	mount := MountSpec{
+		Type: mt,
+		Path: path,
+	}
+
+	return &mount, nil
+}

internal/discover/csv/mount_spec_test.go

@@ -0,0 +1,82 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package csv
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewMountSpecFromLine(t *testing.T) {
+	parseError := fmt.Errorf("failed to parse line")
+	unexpectedError := fmt.Errorf("unexpected mount type")
+
+	testCases := []struct {
+		line          string
+		expectedError error
+		expectedValue MountSpec
+	}{
+		{
+			line:          "",
+			expectedError: parseError,
+		},
+		{
+			line:          "\t",
+			expectedError: parseError,
+		},
+		{
+			line:          ",",
+			expectedError: parseError,
+		},
+		{
+			line:          "dev,",
+			expectedError: parseError,
+		},
+		{
+			line: "dev ,/a/path",
+			expectedValue: MountSpec{
+				Path: "/a/path",
+				Type: "dev",
+			},
+		},
+		{
+			line: "dev ,/a/path,with,commas",
+			expectedValue: MountSpec{
+				Path: "/a/path,with,commas",
+				Type: "dev",
+			},
+		},
+		{
+			line:          "not-dev ,/a/path",
+			expectedError: unexpectedError,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			target, err := NewMountSpecFromLine(tc.line)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.EqualValues(t, &tc.expectedValue, target)
+		})
+	}
+}

internal/discover/csv_test.go

@@ -0,0 +1,160 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCharDevices(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description          string
+		input                *charDevices
+		expectedMounts       []Mount
+		expectedMountsError  error
+		expectedDevicesError error
+		expectedDevices      []Device
+	}{
+		{
+			description: "dev mounts are empty",
+			input: (*charDevices)(
+				&mounts{
+					lookup: &lookup.LocatorMock{
+						LocateFunc: func(string) ([]string, error) {
+							return []string{"located"}, nil
+						},
+					},
+					required: []string{"required"},
+				},
+			),
+			expectedDevices: []Device{{Path: "located"}},
+		},
+		{
+			description:          "dev devices returns error for nil lookup",
+			input:                &charDevices{},
+			expectedDevicesError: fmt.Errorf("no lookup defined"),
+		},
+	}
+
+	for _, tc := range testCases {
+		logHook.Reset()
+
+		t.Run(tc.description, func(t *testing.T) {
+			tc.input.logger = logger
+
+			mounts, err := tc.input.Mounts()
+			if tc.expectedMountsError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedMounts, mounts)
+
+			devices, err := tc.input.Devices()
+			if tc.expectedDevicesError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedDevices, devices)
+		})
+	}
+}
+
+func TestNewFromMountSpec(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	locators := map[csv.MountSpecType]lookup.Locator{
+		"dev": &lookup.LocatorMock{},
+		"lib": &lookup.LocatorMock{},
+	}
+
+	testCases := []struct {
+		description        string
+		targets            []*csv.MountSpec
+		expectedError      error
+		expectedDiscoverer Discover
+	}{
+		{
+			description:        "empty targets returns None discoverer list",
+			expectedDiscoverer: &None{},
+		},
+		{
+			description: "unexpected locator returns error",
+			targets: []*csv.MountSpec{
+				{
+					Type: "foo",
+					Path: "bar",
+				},
+			},
+			expectedError: fmt.Errorf("no locator defined for foo"),
+		},
+		{
+			description: "creates discoverers based on type",
+			targets: []*csv.MountSpec{
+				{
+					Type: "dev",
+					Path: "dev0",
+				},
+				{
+					Type: "lib",
+					Path: "lib0",
+				},
+				{
+					Type: "dev",
+					Path: "dev1",
+				},
+			},
+			expectedDiscoverer: &list{
+				discoverers: []Discover{
+					(*charDevices)(
+						&mounts{
+							logger:   logger,
+							lookup:   locators["dev"],
+							required: []string{"dev0", "dev1"},
+						},
+					),
+					&mounts{
+						logger:   logger,
+						lookup:   locators["lib"],
+						required: []string{"lib0"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			discoverer, err := newFromMountSpecs(logger, locators, tc.targets)
+			if tc.expectedError != nil {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expectedDiscoverer, discoverer)
+		})
+	}
+}

internal/discover/discover.go

@@ -0,0 +1,48 @@
+/*
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+// Config represents the configuration options for discovery
+type Config struct {
+	Root                                    string
+	NVIDIAContainerToolkitCLIExecutablePath string
+}
+
+// Device represents a discovered character device.
+type Device struct {
+	Path string
+}
+
+// Mount represents a discovered mount.
+type Mount struct {
+	Path string
+}
+
+// Hook represents a discovered hook.
+type Hook struct {
+	Lifecycle string
+	Path      string
+	Args      []string
+}
+
+//go:generate moq -stub -out discover_mock.go . Discover
+// Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+type Discover interface {
+	Devices() ([]Device, error)
+	Mounts() ([]Mount, error)
+	Hooks() ([]Hook, error)
+}

internal/discover/discover_mock.go

@@ -0,0 +1,150 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package discover
+
+import (
+	"sync"
+)
+
+// Ensure, that DiscoverMock does implement Discover.
+// If this is not the case, regenerate this file with moq.
+var _ Discover = &DiscoverMock{}
+
+// DiscoverMock is a mock implementation of Discover.
+//
+// 	func TestSomethingThatUsesDiscover(t *testing.T) {
+//
+// 		// make and configure a mocked Discover
+// 		mockedDiscover := &DiscoverMock{
+// 			DevicesFunc: func() ([]Device, error) {
+// 				panic("mock out the Devices method")
+// 			},
+// 			HooksFunc: func() ([]Hook, error) {
+// 				panic("mock out the Hooks method")
+// 			},
+// 			MountsFunc: func() ([]Mount, error) {
+// 				panic("mock out the Mounts method")
+// 			},
+// 		}
+//
+// 		// use mockedDiscover in code that requires Discover
+// 		// and then make assertions.
+//
+// 	}
+type DiscoverMock struct {
+	// DevicesFunc mocks the Devices method.
+	DevicesFunc func() ([]Device, error)
+
+	// HooksFunc mocks the Hooks method.
+	HooksFunc func() ([]Hook, error)
+
+	// MountsFunc mocks the Mounts method.
+	MountsFunc func() ([]Mount, error)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Devices holds details about calls to the Devices method.
+		Devices []struct {
+		}
+		// Hooks holds details about calls to the Hooks method.
+		Hooks []struct {
+		}
+		// Mounts holds details about calls to the Mounts method.
+		Mounts []struct {
+		}
+	}
+	lockDevices sync.RWMutex
+	lockHooks   sync.RWMutex
+	lockMounts  sync.RWMutex
+}
+
+// Devices calls DevicesFunc.
+func (mock *DiscoverMock) Devices() ([]Device, error) {
+	callInfo := struct {
+	}{}
+	mock.lockDevices.Lock()
+	mock.calls.Devices = append(mock.calls.Devices, callInfo)
+	mock.lockDevices.Unlock()
+	if mock.DevicesFunc == nil {
+		var (
+			devicesOut []Device
+			errOut     error
+		)
+		return devicesOut, errOut
+	}
+	return mock.DevicesFunc()
+}
+
+// DevicesCalls gets all the calls that were made to Devices.
+// Check the length with:
+//     len(mockedDiscover.DevicesCalls())
+func (mock *DiscoverMock) DevicesCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockDevices.RLock()
+	calls = mock.calls.Devices
+	mock.lockDevices.RUnlock()
+	return calls
+}
+
+// Hooks calls HooksFunc.
+func (mock *DiscoverMock) Hooks() ([]Hook, error) {
+	callInfo := struct {
+	}{}
+	mock.lockHooks.Lock()
+	mock.calls.Hooks = append(mock.calls.Hooks, callInfo)
+	mock.lockHooks.Unlock()
+	if mock.HooksFunc == nil {
+		var (
+			hooksOut []Hook
+			errOut   error
+		)
+		return hooksOut, errOut
+	}
+	return mock.HooksFunc()
+}
+
+// HooksCalls gets all the calls that were made to Hooks.
+// Check the length with:
+//     len(mockedDiscover.HooksCalls())
+func (mock *DiscoverMock) HooksCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockHooks.RLock()
+	calls = mock.calls.Hooks
+	mock.lockHooks.RUnlock()
+	return calls
+}
+
+// Mounts calls MountsFunc.
+func (mock *DiscoverMock) Mounts() ([]Mount, error) {
+	callInfo := struct {
+	}{}
+	mock.lockMounts.Lock()
+	mock.calls.Mounts = append(mock.calls.Mounts, callInfo)
+	mock.lockMounts.Unlock()
+	if mock.MountsFunc == nil {
+		var (
+			mountsOut []Mount
+			errOut    error
+		)
+		return mountsOut, errOut
+	}
+	return mock.MountsFunc()
+}
+
+// MountsCalls gets all the calls that were made to Mounts.
+// Check the length with:
+//     len(mockedDiscover.MountsCalls())
+func (mock *DiscoverMock) MountsCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockMounts.RLock()
+	calls = mock.calls.Mounts
+	mock.lockMounts.RUnlock()
+	return calls
+}

internal/discover/ldconfig.go

@@ -0,0 +1,126 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+)
+
+// NewLDCacheUpdateHook creates a discoverer that updates the ldcache for the specified mounts. A logger can also be specified
+func NewLDCacheUpdateHook(logger *logrus.Logger, mounts Discover, cfg *Config) (Discover, error) {
+	d := ldconfig{
+		logger:                  logger,
+		mountsFrom:              mounts,
+		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
+		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+	}
+
+	return &d, nil
+}
+
+const (
+	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+)
+
+type ldconfig struct {
+	None
+	logger                  *logrus.Logger
+	mountsFrom              Discover
+	lookup                  lookup.Locator
+	nvidiaCTKExecutablePath string
+}
+
+// Hooks checks the required mounts for libraries and returns a hook to update the LDcache for the discovered paths.
+func (d ldconfig) Hooks() ([]Hook, error) {
+	mounts, err := d.mountsFrom.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
+	}
+
+	libDirs := getLibDirs(mounts)
+
+	hookPath := nvidiaCTKDefaultFilePath
+	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
+	if err != nil {
+		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
+	} else if len(targets) == 0 {
+		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
+	} else {
+		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
+		hookPath = targets[0]
+	}
+	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
+
+	args := []string{hookPath, "hook", "update-ldcache"}
+	for _, f := range libDirs {
+		args = append(args, "--folders", f)
+	}
+	h := Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      hookPath,
+		Args:      args,
+	}
+
+	return []Hook{h}, nil
+}
+
+// getLibDirs extracts the library dirs from the specified mounts
+func getLibDirs(mounts []Mount) []string {
+	var paths []string
+	checked := make(map[string]bool)
+
+	for _, m := range mounts {
+		dir := filepath.Dir(m.Path)
+		if dir == "" {
+			continue
+		}
+
+		_, exists := checked[dir]
+		if exists {
+			continue
+		}
+		checked[dir] = isLibName(filepath.Base(m.Path))
+
+		if checked[dir] {
+			paths = append(paths, dir)
+		}
+	}
+
+	sort.Strings(paths)
+
+	return paths
+}
+
+// isLibName checks if the specified filename is a library (i.e. ends in `.so*`)
+func isLibName(filename string) bool {
+	parts := strings.Split(filename, ".")
+
+	for _, p := range parts {
+		if p == "so" {
+			return true
+		}
+	}
+
+	return false
+}

internal/discover/legacy.go

@@ -0,0 +1,70 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+)
+
+// NewLegacyDiscoverer creates a discoverer for the experimental runtime
+func NewLegacyDiscoverer(logger *logrus.Logger, cfg *Config) (Discover, error) {
+	d := legacy{
+		logger: logger,
+		lookup: lookup.NewExecutableLocator(logger, cfg.Root),
+	}
+
+	return &d, nil
+}
+
+type legacy struct {
+	None
+	logger *logrus.Logger
+	lookup lookup.Locator
+}
+
+var _ Discover = (*legacy)(nil)
+
+// Hooks returns the "legacy" NVIDIA Container Runtime hook. This hook calls out
+// to the nvidia-container-cli to make modifications to the container as defined
+// in libnvidia-container.
+func (d legacy) Hooks() ([]Hook, error) {
+	hookPath := filepath.Join(config.DefaultExecutableDir, config.NVIDIAContainerRuntimeHookExecutable)
+	targets, err := d.lookup.Locate(config.NVIDIAContainerRuntimeHookExecutable)
+	if err != nil {
+		d.logger.Warnf("Failed to locate %v: %v", config.NVIDIAContainerRuntimeHookExecutable, err)
+	} else if len(targets) == 0 {
+		d.logger.Warnf("%v not found", config.NVIDIAContainerRuntimeHookExecutable)
+	} else {
+		d.logger.Debugf("Found %v candidates: %v", config.NVIDIAContainerRuntimeHookExecutable, targets)
+		hookPath = targets[0]
+	}
+	d.logger.Debugf("Using NVIDIA Container Runtime Hook path %v", hookPath)
+
+	args := []string{hookPath, "--force", "prestart"}
+	h := Hook{
+		Lifecycle: cdi.PrestartHook,
+		Path:      hookPath,
+		Args:      args,
+	}
+
+	return []Hook{h}, nil
+}

internal/discover/list.go

@@ -0,0 +1,82 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+import "fmt"
+
+// list is a discoverer that contains a list of Discoverers. The output of the
+// Mounts functions is the concatenation of the output for each of the
+// elements in the list.
+type list struct {
+	discoverers []Discover
+}
+
+var _ Discover = (*list)(nil)
+
+// NewList creates a discoverer that is the composite of a list of discoveres.
+func NewList(d ...Discover) Discover {
+	l := list{
+		discoverers: d,
+	}
+
+	return &l
+}
+
+// Devices returns all devices from the included discoverers
+func (d list) Devices() ([]Device, error) {
+	var allDevices []Device
+
+	for i, di := range d.discoverers {
+		devices, err := di.Devices()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering devices for discoverer %v: %v", i, err)
+		}
+		allDevices = append(allDevices, devices...)
+	}
+
+	return allDevices, nil
+}
+
+// Mounts returns all mounts from the included discoverers
+func (d list) Mounts() ([]Mount, error) {
+	var allMounts []Mount
+
+	for i, di := range d.discoverers {
+		mounts, err := di.Mounts()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering mounts for discoverer %v: %v", i, err)
+		}
+		allMounts = append(allMounts, mounts...)
+	}
+
+	return allMounts, nil
+}
+
+// Hooks returns all Hooks from the included discoverers
+func (d list) Hooks() ([]Hook, error) {
+	var allHooks []Hook
+
+	for i, di := range d.discoverers {
+		hooks, err := di.Hooks()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering hooks for discoverer %v: %v", i, err)
+		}
+		allHooks = append(allHooks, hooks...)
+	}
+
+	return allHooks, nil
+}

internal/discover/mounts.go

@@ -0,0 +1,85 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+// mounts is a generic discoverer for Mounts. It is customized by specifying the
+// required entities as a list and a Locator that is used to find the target mounts
+// based on the entry in the list.
+type mounts struct {
+	None
+	logger   *logrus.Logger
+	lookup   lookup.Locator
+	required []string
+	sync.Mutex
+	cache []Mount
+}
+
+var _ Discover = (*mounts)(nil)
+
+func (d *mounts) Mounts() ([]Mount, error) {
+	if d.lookup == nil {
+		return nil, fmt.Errorf("no lookup defined")
+	}
+
+	if d.cache != nil {
+		d.logger.Debugf("returning cached mounts")
+		return d.cache, nil
+	}
+
+	d.Lock()
+	defer d.Unlock()
+
+	paths := make(map[string]bool)
+
+	for _, candidate := range d.required {
+		d.logger.Debugf("Locating %v", candidate)
+		located, err := d.lookup.Locate(candidate)
+		if err != nil {
+			d.logger.Warnf("Could not locate %v: %v", candidate, err)
+			continue
+		}
+		if len(located) == 0 {
+			d.logger.Warnf("Missing %v", candidate)
+			continue
+		}
+		d.logger.Debugf("Located %v as %v", candidate, located)
+		for _, p := range located {
+			paths[p] = true
+		}
+	}
+
+	var mounts []Mount
+	for path := range paths {
+		d.logger.Infof("Selecting %v", path)
+		mount := Mount{
+			Path: path,
+		}
+		mounts = append(mounts, mount)
+	}
+
+	d.cache = mounts
+
+	return mounts, nil
+}

internal/discover/mounts_test.go

@@ -0,0 +1,165 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/stretchr/testify/require"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestMountsReturnsEmptyDevices(t *testing.T) {
+	d := mounts{}
+	devices, err := d.Devices()
+
+	require.NoError(t, err)
+	require.Empty(t, devices)
+}
+
+func TestMounts(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description    string
+		expectedError  error
+		expectedMounts []Mount
+		input          *mounts
+	}{
+		{
+			description:   "nill lookup returns error",
+			expectedError: fmt.Errorf("no lookup defined"),
+			input:         &mounts{},
+		},
+		{
+			description:   "empty required returns no mounts",
+			expectedError: nil,
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(string) ([]string, error) {
+						return []string{"located"}, nil
+					},
+				},
+			},
+		},
+		{
+			description:   "required returns located",
+			expectedError: nil,
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(string) ([]string, error) {
+						return []string{"located"}, nil
+					},
+				},
+				required: []string{"required"},
+			},
+			expectedMounts: []Mount{{Path: "located"}},
+		},
+		{
+			description:   "mounts removes located duplicates",
+			expectedError: nil,
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(string) ([]string, error) {
+						return []string{"located"}, nil
+					},
+				},
+				required: []string{"required0", "required1"},
+			},
+			expectedMounts: []Mount{{Path: "located"}},
+		},
+		{
+			description: "mounts skips located errors",
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(s string) ([]string, error) {
+						if s == "error" {
+							return nil, fmt.Errorf(s)
+						}
+						return []string{s}, nil
+					},
+				},
+				required: []string{"required0", "error", "required1"},
+			},
+			expectedMounts: []Mount{{Path: "required0"}, {Path: "required1"}},
+		},
+		{
+			description: "mounts skips unlocated",
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(s string) ([]string, error) {
+						if s == "empty" {
+							return nil, nil
+						}
+						return []string{s}, nil
+					},
+				},
+				required: []string{"required0", "empty", "required1"},
+			},
+			expectedMounts: []Mount{{Path: "required0"}, {Path: "required1"}},
+		},
+		{
+			description: "mounts skips unlocated",
+			input: &mounts{
+				lookup: &lookup.LocatorMock{
+					LocateFunc: func(s string) ([]string, error) {
+						if s == "multiple" {
+							return []string{"multiple0", "multiple1"}, nil
+						}
+						return []string{s}, nil
+					},
+				},
+				required: []string{"required0", "multiple", "required1"},
+			},
+			expectedMounts: []Mount{
+				{Path: "required0"},
+				{Path: "multiple0"},
+				{Path: "multiple1"},
+				{Path: "required1"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		logHook.Reset()
+		t.Run(tc.description, func(t *testing.T) {
+			tc.input.logger = logger
+			mounts, err := tc.input.Mounts()
+
+			if tc.expectedError != nil {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.ElementsMatch(t, tc.expectedMounts, mounts)
+
+			// We check that the mock is called for each element of required
+			if tc.input.lookup != nil {
+				mock := tc.input.lookup.(*lookup.LocatorMock)
+				require.Len(t, mock.LocateCalls(), len(tc.input.required))
+				var args []string
+				for _, c := range mock.LocateCalls() {
+					args = append(args, c.S)
+				}
+				require.EqualValues(t, args, tc.input.required)
+			}
+		})
+	}
+}

internal/discover/none.go

@@ -0,0 +1,38 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+// None is a null discoverer that returns an empty list of devices and
+// mounts.
+type None struct{}
+
+var _ Discover = (*None)(nil)
+
+// Devices returns an empty list of devices
+func (e None) Devices() ([]Device, error) {
+	return []Device{}, nil
+}
+
+// Mounts returns an empty list of mounts
+func (e None) Mounts() ([]Mount, error) {
+	return []Mount{}, nil
+}
+
+// Hooks returns an empty list of hooks
+func (e None) Hooks() ([]Hook, error) {
+	return []Hook{}, nil
+}

internal/discover/none_test.go

@@ -0,0 +1,31 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package discover
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNone(t *testing.T) {
+	d := None{}
+
+	mounts, err := d.Mounts()
+	require.NoError(t, err)
+	require.Empty(t, mounts)
+}

internal/discover/symlinks.go

@@ -0,0 +1,71 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/sirupsen/logrus"
+)
+
+type symlinks struct {
+	None
+	logger                  *logrus.Logger
+	lookup                  lookup.Locator
+	nvidiaCTKExecutablePath string
+	csvFiles                []string
+}
+
+// NewCreateSymlinksHook creates a discoverer for a hook that creates required symlinks in the container
+func NewCreateSymlinksHook(logger *logrus.Logger, csvFiles []string, cfg *Config) (Discover, error) {
+	d := symlinks{
+		logger:                  logger,
+		lookup:                  lookup.NewExecutableLocator(logger, cfg.Root),
+		nvidiaCTKExecutablePath: cfg.NVIDIAContainerToolkitCLIExecutablePath,
+		csvFiles:                csvFiles,
+	}
+
+	return &d, nil
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d symlinks) Hooks() ([]Hook, error) {
+	hookPath := nvidiaCTKDefaultFilePath
+	targets, err := d.lookup.Locate(d.nvidiaCTKExecutablePath)
+	if err != nil {
+		d.logger.Warnf("Failed to locate %v: %v", d.nvidiaCTKExecutablePath, err)
+	} else if len(targets) == 0 {
+		d.logger.Warnf("%v not found", d.nvidiaCTKExecutablePath)
+	} else {
+		d.logger.Debugf("Found %v candidates: %v", d.nvidiaCTKExecutablePath, targets)
+		hookPath = targets[0]
+	}
+	d.logger.Debugf("Using NVIDIA Container Toolkit CLI path %v", hookPath)
+
+	args := []string{hookPath, "hook", "create-symlinks"}
+	for _, f := range d.csvFiles {
+		args = append(args, "--csv-filenames", f)
+	}
+
+	h := Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      hookPath,
+		Args:      args,
+	}
+
+	return []Hook{h}, nil
+}

internal/edits/device.go

@@ -0,0 +1,45 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type device discover.Device
+
+// toEdits converts a discovered device to CDI Container Edits.
+func (d device) toEdits() *cdi.ContainerEdits {
+	e := cdi.ContainerEdits{
+		ContainerEdits: &specs.ContainerEdits{
+			DeviceNodes: []*specs.DeviceNode{d.toSpec()},
+		},
+	}
+	return &e
+}
+
+// toSpec converts a discovered Device to a CDI Spec Device. Note
+// that missing info is filled in when edits are applied by querying the Device node.
+func (d device) toSpec() *specs.DeviceNode {
+	s := specs.DeviceNode{
+		Path: d.Path,
+	}
+
+	return &s
+}

internal/edits/edits.go

@@ -0,0 +1,93 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	ociSpecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+type edits struct {
+	cdi.ContainerEdits
+	logger *logrus.Logger
+}
+
+// NewSpecEdits creates a SpecModifier that defines the required OCI spec edits (as CDI ContainerEdits) from the specified
+// discoverer.
+func NewSpecEdits(logger *logrus.Logger, d discover.Discover) (oci.SpecModifier, error) {
+	devices, err := d.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices: %v", err)
+	}
+
+	mounts, err := d.Mounts()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover mounts: %v", err)
+	}
+
+	hooks, err := d.Hooks()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover hooks: %v", err)
+	}
+
+	c := cdi.ContainerEdits{}
+	for _, d := range devices {
+		c.Append(device(d).toEdits())
+	}
+
+	for _, m := range mounts {
+		c.Append(mount(m).toEdits())
+	}
+
+	for _, h := range hooks {
+		c.Append(hook(h).toEdits())
+	}
+
+	e := edits{
+		ContainerEdits: c,
+		logger:         logger,
+	}
+
+	return &e, nil
+}
+
+// Modify applies the defined edits to the incoming OCI spec
+func (e *edits) Modify(spec *ociSpecs.Spec) error {
+	if e == nil || e.ContainerEdits.ContainerEdits == nil {
+		return nil
+	}
+
+	e.logger.Info("Mounts:")
+	for _, mount := range e.Mounts {
+		e.logger.Infof("Mounting %v at %v", mount.HostPath, mount.ContainerPath)
+	}
+	e.logger.Infof("Devices:")
+	for _, device := range e.DeviceNodes {
+		e.logger.Infof("Injecting %v", device.Path)
+	}
+	e.logger.Infof("Hooks:")
+	for _, hook := range e.Hooks {
+		e.logger.Infof("Injecting %v", hook.Args)
+	}
+
+	return e.Apply(spec)
+}

internal/edits/hook.go

@@ -0,0 +1,47 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type hook discover.Hook
+
+// toEdits converts a discovered hook to CDI Container Edits.
+func (d hook) toEdits() *cdi.ContainerEdits {
+	e := cdi.ContainerEdits{
+		ContainerEdits: &specs.ContainerEdits{
+			Hooks: []*specs.Hook{d.toSpec()},
+		},
+	}
+	return &e
+}
+
+// toSpec converts a discovered Hook to a CDI Spec Hook. Note
+// that missing info is filled in when edits are applied by querying the Hook node.
+func (d hook) toSpec() *specs.Hook {
+	s := specs.Hook{
+		HookName: d.Lifecycle,
+		Path:     d.Path,
+		Args:     d.Args,
+	}
+
+	return &s
+}

internal/edits/mount.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package edits
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+type mount discover.Mount
+
+// toEdits converts a discovered mount to CDI Container Edits.
+func (d mount) toEdits() *cdi.ContainerEdits {
+	e := cdi.ContainerEdits{
+		ContainerEdits: &specs.ContainerEdits{
+			Mounts: []*specs.Mount{d.toSpec()},
+		},
+	}
+	return &e
+}
+
+// toSpec converts a discovered Mount to a CDI Spec Mount. Note
+// that missing info is filled in when edits are applied by querying the Mount node.
+func (d mount) toSpec() *specs.Mount {
+	s := specs.Mount{
+		HostPath: d.Path,
+		// TODO: We need to allow the container path to be customised
+		ContainerPath: d.Path,
+		Options: []string{
+			"ro",
+			"nosuid",
+			"nodev",
+			"bind",
+		},
+	}
+
+	return &s
+}

internal/lookup/device.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	devRoot = "/dev"
+)
+
+// NewCharDeviceLocator creates a Locator that can be used to find char devices at the specified root. A logger is
+// also specified.
+func NewCharDeviceLocator(logger *logrus.Logger, root string) Locator {
+	l := file{
+		logger:   logger,
+		prefixes: []string{root, filepath.Join(root, devRoot)},
+		filter:   assertCharDevice,
+	}
+
+	return &l
+}
+
+// assertCharDevice checks whether the specified path is a char device and returns an error if this is not the case.
+func assertCharDevice(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("error getting info: %v", err)
+	}
+	if info.Mode()|os.ModeCharDevice == 0 {
+		return fmt.Errorf("%v is not a char device", filename)
+	}
+	return nil
+}

internal/lookup/dir.go

@@ -0,0 +1,50 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package lookup
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// NewDirectoryLocator creates a Locator that can be used to find directories at the specified root. A logger
+// is also specified.
+func NewDirectoryLocator(logger *log.Logger, root string) Locator {
+	l := file{
+		logger:   logger,
+		prefixes: []string{root},
+		filter:   assertDirectory,
+	}
+
+	return &l
+}
+
+// assertDirectory checks wither the specified path is a directory.
+func assertDirectory(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("error getting info for %v: %v", filename, err)
+	}
+
+	if !info.IsDir() {
+		return fmt.Errorf("specified path '%v' is not a directory", filename)
+	}
+
+	return nil
+}

internal/lookup/executable.go

@@ -0,0 +1,93 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package lookup
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	envPath = "PATH"
+)
+
+var defaultPaths = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
+
+type executable struct {
+	file
+}
+
+// NewExecutableLocator creates a locator to fine executable files in the path. A logger can also be specified.
+func NewExecutableLocator(logger *log.Logger, root string) Locator {
+	pathEnv := os.Getenv(envPath)
+	paths := filepath.SplitList(pathEnv)
+
+	if root != "" {
+		paths = append(paths, defaultPaths...)
+	}
+
+	var prefixes []string
+	for _, dir := range paths {
+		prefixes = append(prefixes, filepath.Join(root, dir))
+	}
+	l := executable{
+		file: file{
+			logger:   logger,
+			prefixes: prefixes,
+			filter:   assertExecutable,
+		},
+	}
+	return &l
+}
+
+var _ Locator = (*executable)(nil)
+
+// Locate finds executable files in the path. If a relative or absolute path is specified, the prefix paths are not considered.
+func (p executable) Locate(filename string) ([]string, error) {
+	// For absolute paths we ensure that it is executable
+	if strings.Contains(filename, "/") {
+		err := assertExecutable(filename)
+		if err != nil {
+			return nil, fmt.Errorf("absolute path %v is not an executable file: %v", filename, err)
+		}
+		return []string{filename}, nil
+	}
+
+	return p.file.Locate(filename)
+}
+
+// assertExecutable checks whether the specified path is an execuable file.
+func assertExecutable(filename string) error {
+	err := assertFile(filename)
+	if err != nil {
+		return err
+	}
+	info, err := os.Stat(filename)
+	if err != nil {
+		return err
+	}
+
+	if info.Mode()&0111 == 0 {
+		return fmt.Errorf("specified file '%v' is not executable", filename)
+	}
+
+	return nil
+}

internal/lookup/file.go

@@ -0,0 +1,85 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package lookup
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// file can be used to locate file (or file-like elements) at a specified set of
+// prefixes. The validity of a file is determined by a filter function.
+type file struct {
+	logger   *log.Logger
+	prefixes []string
+	filter   func(string) error
+}
+
+// NewFileLocator creates a Locator that can be used to find files at the specified root. A logger
+// can also be specified.
+func NewFileLocator(logger *log.Logger, root string) Locator {
+	l := newFileLocator(logger, root)
+
+	return &l
+}
+
+func newFileLocator(logger *log.Logger, root string) file {
+	return file{
+		logger:   logger,
+		prefixes: []string{root},
+		filter:   assertFile,
+	}
+}
+
+var _ Locator = (*file)(nil)
+
+// Locate attempts to find the specified file. All prefixes are searched and any matching
+// candidates are returned. If no matches are found, an error is returned.
+func (p file) Locate(filename string) ([]string, error) {
+	var filenames []string
+	for _, prefix := range p.prefixes {
+		candidate := filepath.Join(prefix, filename)
+		p.logger.Debugf("Checking candidate '%v'", candidate)
+		err := p.filter(candidate)
+		if err != nil {
+			p.logger.Debugf("Candidate '%v' does not meet requirements: %v", candidate, err)
+			continue
+		}
+		filenames = append(filenames, candidate)
+	}
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("file %v not found", filename)
+	}
+	return filenames, nil
+}
+
+// assertFile checks whether the specified path is a regular file
+func assertFile(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("error getting info for %v: %v", filename, err)
+	}
+
+	if info.IsDir() {
+		return fmt.Errorf("specified path '%v' is a directory", filename)
+	}
+
+	return nil
+}

internal/lookup/locator.go

@@ -0,0 +1,24 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package lookup
+
+//go:generate moq -stub -out locator_mock.go . Locator
+
+// Locator defines the interface for locating files on a system.
+type Locator interface {
+	Locate(string) ([]string, error)
+}

internal/lookup/locator_mock.go

@@ -0,0 +1,77 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package lookup
+
+import (
+	"sync"
+)
+
+// Ensure, that LocatorMock does implement Locator.
+// If this is not the case, regenerate this file with moq.
+var _ Locator = &LocatorMock{}
+
+// LocatorMock is a mock implementation of Locator.
+//
+// 	func TestSomethingThatUsesLocator(t *testing.T) {
+//
+// 		// make and configure a mocked Locator
+// 		mockedLocator := &LocatorMock{
+// 			LocateFunc: func(s string) ([]string, error) {
+// 				panic("mock out the Locate method")
+// 			},
+// 		}
+//
+// 		// use mockedLocator in code that requires Locator
+// 		// and then make assertions.
+//
+// 	}
+type LocatorMock struct {
+	// LocateFunc mocks the Locate method.
+	LocateFunc func(s string) ([]string, error)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Locate holds details about calls to the Locate method.
+		Locate []struct {
+			// S is the s argument value.
+			S string
+		}
+	}
+	lockLocate sync.RWMutex
+}
+
+// Locate calls LocateFunc.
+func (mock *LocatorMock) Locate(s string) ([]string, error) {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockLocate.Lock()
+	mock.calls.Locate = append(mock.calls.Locate, callInfo)
+	mock.lockLocate.Unlock()
+	if mock.LocateFunc == nil {
+		var (
+			stringsOut []string
+			errOut     error
+		)
+		return stringsOut, errOut
+	}
+	return mock.LocateFunc(s)
+}
+
+// LocateCalls gets all the calls that were made to Locate.
+// Check the length with:
+//     len(mockedLocator.LocateCalls())
+func (mock *LocatorMock) LocateCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockLocate.RLock()
+	calls = mock.calls.Locate
+	mock.lockLocate.RUnlock()
+	return calls
+}

internal/lookup/symlinks.go

@@ -0,0 +1,123 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+type symlinkChain struct {
+	file
+}
+
+type symlink struct {
+	file
+}
+
+// NewSymlinkChainLocator creats a locator that can be used for locating files through symlinks.
+// A logger can also be specified.
+func NewSymlinkChainLocator(logger *logrus.Logger, root string) Locator {
+	l := symlinkChain{
+		file: newFileLocator(logger, root),
+	}
+
+	return &l
+}
+
+// NewSymlinkLocator creats a locator that can be used for locating files through symlinks.
+// A logger can also be specified.
+func NewSymlinkLocator(logger *logrus.Logger, root string) Locator {
+	l := symlink{
+		file: newFileLocator(logger, root),
+	}
+
+	return &l
+}
+
+// Locate finds the specified file at the specified root. If the file is a symlink, the link is followed and all candidates
+// to the final target are returned.
+func (p symlinkChain) Locate(filename string) ([]string, error) {
+	candidates, err := p.file.Locate(filename)
+	if err != nil {
+		return nil, err
+	}
+	if len(candidates) == 0 {
+		return candidates, nil
+	}
+
+	found := make(map[string]bool)
+	for len(candidates) > 0 {
+		candidate := candidates[0]
+		candidates = candidates[:len(candidates)-1]
+		if found[candidate] {
+			continue
+		}
+		found[candidate] = true
+
+		info, err := os.Lstat(candidate)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get file info: %v", info)
+		}
+		if info.Mode()&os.ModeSymlink == 0 {
+			continue
+		}
+		target, err := os.Readlink(candidate)
+		if err != nil {
+			return nil, fmt.Errorf("error checking symlink: %v", err)
+		}
+
+		if !filepath.IsAbs(target) {
+			target, err = filepath.Abs(filepath.Join(filepath.Dir(candidate), target))
+			if err != nil {
+				return nil, fmt.Errorf("failed to construct absolute path: %v", err)
+			}
+		}
+
+		p.logger.Debugf("Resolved link: '%v' => '%v'", candidate, target)
+		if !found[target] {
+			candidates = append(candidates, target)
+		}
+	}
+
+	var filenames []string
+	for f := range found {
+		filenames = append(filenames, f)
+	}
+	return filenames, nil
+}
+
+// Locate finds the specified file at the specified root. If the file is a symlink, the link is resolved and the target returned.
+func (p symlink) Locate(filename string) ([]string, error) {
+	candidates, err := p.file.Locate(filename)
+	if err != nil {
+		return nil, err
+	}
+	if len(candidates) != 1 {
+		return nil, fmt.Errorf("failed to uniquely resolve symlink %v: %v", filename, candidates)
+	}
+
+	target, err := filepath.EvalSymlinks(candidates[0])
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve link: %v", err)
+	}
+
+	return []string{target}, err
+}

internal/oci/args.go

@@ -0,0 +1,115 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	specFileName = "config.json"
+)
+
+// GetBundleDir returns the bundle directory or default depending on the
+// supplied command line arguments.
+func GetBundleDir(args []string) (string, error) {
+	bundleDir, err := GetBundleDirFromArgs(args)
+	if err != nil {
+		return "", fmt.Errorf("error getting bundle dir from args: %v", err)
+	}
+
+	return bundleDir, nil
+}
+
+// GetBundleDirFromArgs checks the specified slice of strings (argv) for a 'bundle' flag as allowed by runc.
+// The following are supported:
+// --bundle{{SEP}}BUNDLE_PATH
+// -bundle{{SEP}}BUNDLE_PATH
+// -b{{SEP}}BUNDLE_PATH
+// where {{SEP}} is either ' ' or '='
+func GetBundleDirFromArgs(args []string) (string, error) {
+	var bundleDir string
+
+	for i := 0; i < len(args); i++ {
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		if !IsBundleFlag(parts[0]) {
+			continue
+		}
+
+		// The flag has the format --bundle=/path
+		if len(parts) == 2 {
+			bundleDir = parts[1]
+			continue
+		}
+
+		// The flag has the format --bundle /path
+		if i+1 < len(args) {
+			bundleDir = args[i+1]
+			i++
+			continue
+		}
+
+		// --bundle / -b was the last element of args
+		return "", fmt.Errorf("bundle option requires an argument")
+	}
+
+	return bundleDir, nil
+}
+
+// GetSpecFilePath returns the expected path to the OCI specification file for the given
+// bundle directory.
+func GetSpecFilePath(bundleDir string) string {
+	specFilePath := filepath.Join(bundleDir, specFileName)
+	return specFilePath
+}
+
+// IsBundleFlag is a helper function that checks wither the specified argument represents
+// a bundle flag (--bundle or -b)
+func IsBundleFlag(arg string) bool {
+	if !strings.HasPrefix(arg, "-") {
+		return false
+	}
+
+	trimmed := strings.TrimLeft(arg, "-")
+	return trimmed == "b" || trimmed == "bundle"
+}
+
+// HasCreateSubcommand checks the supplied arguments for a 'create' subcommand
+func HasCreateSubcommand(args []string) bool {
+	var previousWasBundle bool
+	for _, a := range args {
+		// We check for '--bundle create' explicitly to ensure that we
+		// don't inadvertently trigger a modification if the bundle directory
+		// is specified as `create`
+		if !previousWasBundle && IsBundleFlag(a) {
+			previousWasBundle = true
+			continue
+		}
+
+		if !previousWasBundle && a == "create" {
+			return true
+		}
+
+		previousWasBundle = false
+	}
+
+	return false
+}

internal/oci/args_test.go

@@ -0,0 +1,184 @@
+package oci
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetBundleDir(t *testing.T) {
+	type expected struct {
+		bundle  string
+		isError bool
+	}
+	testCases := []struct {
+		argv     []string
+		expected expected
+	}{
+		{
+			argv: []string{},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"create"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"--bundle"},
+			expected: expected{
+				isError: true,
+			},
+		},
+		{
+			argv: []string{"-b"},
+			expected: expected{
+				isError: true,
+			},
+		},
+		{
+			argv: []string{"--bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"--not-bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"--"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"-bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"--bundle=/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b=/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b=/foo/=bar"},
+			expected: expected{
+				bundle: "/foo/=bar",
+			},
+		},
+		{
+			argv: []string{"-b", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"create", "-b", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b", "create", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+		{
+			argv: []string{"-b=create", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+		{
+			argv: []string{"-b", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		bundle, err := GetBundleDir(tc.argv)
+
+		if tc.expected.isError {
+			require.Errorf(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoErrorf(t, err, "%d: %v", i, tc)
+		}
+
+		require.Equalf(t, tc.expected.bundle, bundle, "%d: %v", i, tc)
+	}
+}
+
+func TestGetSpecFilePathAppendsFilename(t *testing.T) {
+	testCases := []struct {
+		bundleDir string
+		expected  string
+	}{
+		{
+			bundleDir: "",
+			expected:  "config.json",
+		},
+		{
+			bundleDir: "/not/empty/",
+			expected:  "/not/empty/config.json",
+		},
+		{
+			bundleDir: "not/absolute",
+			expected:  "not/absolute/config.json",
+		},
+	}
+
+	for i, tc := range testCases {
+		specPath := GetSpecFilePath(tc.bundleDir)
+
+		require.Equalf(t, tc.expected, specPath, "%d: %v", i, tc)
+	}
+}
+
+func TestHasCreateSubcommand(t *testing.T) {
+	testCases := []struct {
+		args         []string
+		shouldModify bool
+	}{
+		{
+			shouldModify: false,
+		},
+		{
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			args:         []string{"--bundle=create"},
+			shouldModify: false,
+		},
+		{
+			args:         []string{"--bundle", "create"},
+			shouldModify: false,
+		},
+		{
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		require.Equal(t, tc.shouldModify, HasCreateSubcommand(tc.args), "%d: %v", i, tc)
+	}
+}

internal/oci/runtime.go

@@ -16,6 +16,8 @@
 
 package oci
 
+//go:generate moq -stub -out runtime_mock.go . Runtime
+
 // Runtime is an interface for a runtime shim. The Exec method accepts a list
 // of command line arguments, and returns an error / nil.
 type Runtime interface {

internal/oci/runtime_exec.go

@@ -1,79 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package oci
-
-import (
-	"fmt"
-	"os"
-	"syscall"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// SyscallExecRuntime wraps the path that a binary and defines the semanitcs for how to exec into it.
-// This can be used to wrap an OCI-compliant low-level runtime binary, allowing it to be used through the
-// Runtime internface.
-type SyscallExecRuntime struct {
-	logger *log.Logger
-	path   string
-	// exec is used for testing. This defaults to syscall.Exec
-	exec func(argv0 string, argv []string, envv []string) error
-}
-
-var _ Runtime = (*SyscallExecRuntime)(nil)
-
-// NewSyscallExecRuntime creates a SyscallExecRuntime for the specified path with the standard logger
-func NewSyscallExecRuntime(path string) (Runtime, error) {
-	return NewSyscallExecRuntimeWithLogger(log.StandardLogger(), path)
-}
-
-// NewSyscallExecRuntimeWithLogger creates a SyscallExecRuntime for the specified logger and path
-func NewSyscallExecRuntimeWithLogger(logger *log.Logger, path string) (Runtime, error) {
-	info, err := os.Stat(path)
-	if err != nil {
-		return nil, fmt.Errorf("invalid path '%v': %v", path, err)
-	}
-	if info.IsDir() || info.Mode()&0111 == 0 {
-		return nil, fmt.Errorf("specified path '%v' is not an executable file", path)
-	}
-
-	shim := SyscallExecRuntime{
-		logger: logger,
-		path:   path,
-		exec:   syscall.Exec,
-	}
-
-	return &shim, nil
-}
-
-// Exec exces into the binary at the path from the SyscallExecRuntime struct, passing it the supplied arguments
-// after ensuring that the first argument is the path of the target binary.
-func (s SyscallExecRuntime) Exec(args []string) error {
-	runtimeArgs := []string{s.path}
-	if len(args) > 1 {
-		runtimeArgs = append(runtimeArgs, args[1:]...)
-	}
-
-	err := s.exec(s.path, runtimeArgs, os.Environ())
-	if err != nil {
-		return fmt.Errorf("could not exec '%v': %v", s.path, err)
-	}
-
-	// syscall.Exec is not expected to return. This is an error state regardless of whether
-	// err is nil or not.
-	return fmt.Errorf("unexpected return from exec '%v'", s.path)
-}

internal/oci/runtime_exec_test.go

@@ -1,100 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-package oci
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSyscallExecConstructor(t *testing.T) {
-	r, err := NewSyscallExecRuntime("////an/invalid/path")
-	require.Error(t, err)
-	require.Nil(t, r)
-
-	r, err = NewSyscallExecRuntime("/tmp")
-	require.Error(t, err)
-	require.Nil(t, r)
-
-	r, err = NewSyscallExecRuntime("/dev/null")
-	require.Error(t, err)
-	require.Nil(t, r)
-
-	r, err = NewSyscallExecRuntime("/bin/sh")
-	require.NoError(t, err)
-
-	f, ok := r.(*SyscallExecRuntime)
-	require.True(t, ok)
-
-	require.Equal(t, "/bin/sh", f.path)
-}
-
-func TestSyscallExecForwardsArgs(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-	f := SyscallExecRuntime{
-		logger: logger,
-		path:   "runtime",
-	}
-
-	testCases := []struct {
-		returnError error
-		args        []string
-		errorPrefix string
-	}{
-		{
-			returnError: nil,
-			errorPrefix: "unexpected return from exec",
-		},
-		{
-			returnError: fmt.Errorf("error from exec"),
-			errorPrefix: "could not exec",
-		},
-		{
-			returnError: nil,
-			args:        []string{"otherargv0"},
-			errorPrefix: "unexpected return from exec",
-		},
-		{
-			returnError: nil,
-			args:        []string{"otherargv0", "arg1", "arg2", "arg3"},
-			errorPrefix: "unexpected return from exec",
-		},
-	}
-
-	for i, tc := range testCases {
-		execMock := WithMockExec(f, tc.returnError)
-
-		err := execMock.Exec(tc.args)
-
-		require.Errorf(t, err, "%d: %v", i, tc)
-		require.Truef(t, strings.HasPrefix(err.Error(), tc.errorPrefix), "%d: %v", i, tc)
-		if tc.returnError != nil {
-			require.Truef(t, strings.HasSuffix(err.Error(), tc.returnError.Error()), "%d: %v", i, tc)
-		}
-
-		require.Equalf(t, f.path, execMock.argv0, "%d: %v", i, tc)
-		require.Equalf(t, f.path, execMock.argv[0], "%d: %v", i, tc)
-
-		require.LessOrEqualf(t, len(tc.args), len(execMock.argv), "%d: %v", i, tc)
-		if len(tc.args) > 1 {
-			require.Equalf(t, tc.args[1:], execMock.argv[1:], "%d: %v", i, tc)
-		}
-	}
-}

internal/oci/runtime_low_level.go

@@ -0,0 +1,56 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os/exec"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// NewLowLevelRuntime creates a Runtime that wraps a low-level runtime executable.
+// The executable specified is taken from the list of supplied candidates, with the first match
+// present in the PATH being selected. A logger is also specified.
+func NewLowLevelRuntime(logger *log.Logger, candidates []string) (Runtime, error) {
+	runtimePath, err := findRuntime(logger, candidates)
+	if err != nil {
+		return nil, fmt.Errorf("error locating runtime: %v", err)
+	}
+
+	return NewRuntimeForPath(logger, runtimePath)
+}
+
+// findRuntime checks elements in a list of supplied candidates for a matching executable in the PATH.
+// The absolute path to the first match is returned.
+func findRuntime(logger *log.Logger, candidates []string) (string, error) {
+	if len(candidates) == 0 {
+		return "", fmt.Errorf("at least one runtime candidate must be specified")
+	}
+
+	for _, candidate := range candidates {
+		logger.Infof("Looking for runtime binary '%v'", candidate)
+		runcPath, err := exec.LookPath(candidate)
+		if err == nil {
+			logger.Infof("Found runtime binary '%v'", runcPath)
+			return runcPath, nil
+		}
+		logger.Warnf("Runtime binary '%v' not found: %v", candidate, err)
+	}
+
+	return "", fmt.Errorf("no runtime binary found from candidate list: %v", candidates)
+}

internal/oci/runtime_mock.go

@@ -1,49 +1,76 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
 
 package oci
 
-// MockExecRuntime wraps a SyscallExecRuntime, intercepting the exec call for testing
-type MockExecRuntime struct {
-	SyscallExecRuntime
-	execMock
-}
+import (
+	"sync"
+)
 
-// WithMockExec wraps a specified SyscallExecRuntime with a mocked exec function for testing
-func WithMockExec(e SyscallExecRuntime, execResult error) *MockExecRuntime {
-	m := MockExecRuntime{
-		SyscallExecRuntime: e,
-		execMock:           execMock{result: execResult},
+// Ensure, that RuntimeMock does implement Runtime.
+// If this is not the case, regenerate this file with moq.
+var _ Runtime = &RuntimeMock{}
+
+// RuntimeMock is a mock implementation of Runtime.
+//
+// 	func TestSomethingThatUsesRuntime(t *testing.T) {
+//
+// 		// make and configure a mocked Runtime
+// 		mockedRuntime := &RuntimeMock{
+// 			ExecFunc: func(strings []string) error {
+// 				panic("mock out the Exec method")
+// 			},
+// 		}
+//
+// 		// use mockedRuntime in code that requires Runtime
+// 		// and then make assertions.
+//
+// 	}
+type RuntimeMock struct {
+	// ExecFunc mocks the Exec method.
+	ExecFunc func(strings []string) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Exec holds details about calls to the Exec method.
+		Exec []struct {
+			// Strings is the strings argument value.
+			Strings []string
+		}
 	}
-	// overrdie the exec function to the mocked exec function.
-	m.SyscallExecRuntime.exec = m.execMock.exec
-	return &m
+	lockExec sync.RWMutex
 }
 
-type execMock struct {
-	argv0  string
-	argv   []string
-	envv   []string
-	result error
+// Exec calls ExecFunc.
+func (mock *RuntimeMock) Exec(strings []string) error {
+	callInfo := struct {
+		Strings []string
+	}{
+		Strings: strings,
+	}
+	mock.lockExec.Lock()
+	mock.calls.Exec = append(mock.calls.Exec, callInfo)
+	mock.lockExec.Unlock()
+	if mock.ExecFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ExecFunc(strings)
 }
 
-func (m *execMock) exec(argv0 string, argv []string, envv []string) error {
-	m.argv0 = argv0
-	m.argv = argv
-	m.envv = envv
-
-	return m.result
+// ExecCalls gets all the calls that were made to Exec.
+// Check the length with:
+//     len(mockedRuntime.ExecCalls())
+func (mock *RuntimeMock) ExecCalls() []struct {
+	Strings []string
+} {
+	var calls []struct {
+		Strings []string
+	}
+	mock.lockExec.RLock()
+	calls = mock.calls.Exec
+	mock.lockExec.RUnlock()
+	return calls
 }

internal/oci/runtime_path.go

@@ -0,0 +1,65 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// pathRuntime wraps the path that a binary and defines the semanitcs for how to exec into it.
+// This can be used to wrap an OCI-compliant low-level runtime binary, allowing it to be used through the
+// Runtime internface.
+type pathRuntime struct {
+	logger      *log.Logger
+	path        string
+	execRuntime Runtime
+}
+
+var _ Runtime = (*pathRuntime)(nil)
+
+// NewRuntimeForPath creates a Runtime for the specified logger and path
+func NewRuntimeForPath(logger *log.Logger, path string) (Runtime, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return nil, fmt.Errorf("invalid path '%v': %v", path, err)
+	}
+	if info.IsDir() || info.Mode()&0111 == 0 {
+		return nil, fmt.Errorf("specified path '%v' is not an executable file", path)
+	}
+
+	shim := pathRuntime{
+		logger:      logger,
+		path:        path,
+		execRuntime: syscallExec{},
+	}
+
+	return &shim, nil
+}
+
+// Exec exces into the binary at the path from the pathRuntime struct, passing it the supplied arguments
+// after ensuring that the first argument is the path of the target binary.
+func (s pathRuntime) Exec(args []string) error {
+	runtimeArgs := []string{s.path}
+	if len(args) > 1 {
+		runtimeArgs = append(runtimeArgs, args[1:]...)
+	}
+
+	return s.execRuntime.Exec(runtimeArgs)
+}

internal/oci/runtime_path_test.go

@@ -0,0 +1,99 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+package oci
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPathRuntimeConstructor(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	r, err := NewRuntimeForPath(logger, "////an/invalid/path")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath(logger, "/tmp")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath(logger, "/dev/null")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath(logger, "/bin/sh")
+	require.NoError(t, err)
+
+	f, ok := r.(*pathRuntime)
+	require.True(t, ok)
+
+	require.Equal(t, "/bin/sh", f.path)
+}
+
+func TestPathRuntimeForwardsArgs(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		execRuntimeError error
+		args             []string
+	}{
+		{},
+		{
+			args: []string{"shouldBeReplaced"},
+		},
+		{
+			args: []string{"shouldBeReplaced", "arg1"},
+		},
+		{
+			execRuntimeError: fmt.Errorf("exec error"),
+		},
+	}
+
+	for _, tc := range testCases {
+		mockedRuntime := &RuntimeMock{
+			ExecFunc: func(strings []string) error {
+				return tc.execRuntimeError
+			},
+		}
+		r := pathRuntime{
+			logger:      logger,
+			path:        "runtime",
+			execRuntime: mockedRuntime,
+		}
+		err := r.Exec(tc.args)
+
+		require.ErrorIs(t, err, tc.execRuntimeError)
+
+		calls := mockedRuntime.ExecCalls()
+		require.Len(t, calls, 1)
+
+		numArgs := len(tc.args)
+		if numArgs == 0 {
+			numArgs = 1
+		}
+
+		require.Len(t, calls[0].Strings, numArgs)
+		require.Equal(t, "runtime", calls[0].Strings[0])
+
+		if numArgs > 1 {
+			require.EqualValues(t, tc.args[1:], calls[0].Strings[1:])
+		}
+	}
+}

internal/oci/runtime_syscall_exec.go

@@ -0,0 +1,38 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+type syscallExec struct{}
+
+var _ Runtime = (*syscallExec)(nil)
+
+func (r syscallExec) Exec(args []string) error {
+	err := syscall.Exec(args[0], args, os.Environ())
+	if err != nil {
+		return fmt.Errorf("could not exec '%v': %v", args[0], err)
+	}
+
+	// syscall.Exec is not expected to return. This is an error state regardless of whether
+	// err is nil or not.
+	return fmt.Errorf("unexpected return from exec '%v'", args[0])
+}

internal/oci/spec.go

@@ -17,86 +17,41 @@
 package oci
 
 import (
-	"encoding/json"
 	"fmt"
-	"os"
 
-	oci "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
 )
 
-// SpecModifier is a function that accepts a pointer to an OCI Srec and returns an
-// error. The intention is that the function would modify the spec in-place.
-type SpecModifier func(*oci.Spec) error
+// SpecModifier defines an interace for modifying a (raw) OCI spec
+type SpecModifier interface {
+	// Modify is a method that accepts a pointer to an OCI Srec and returns an
+	// error. The intention is that the function would modify the spec in-place.
+	Modify(*specs.Spec) error
+}
 
+//go:generate moq -stub -out spec_mock.go . Spec
 // Spec defines the operations to be performed on an OCI specification
 type Spec interface {
 	Load() error
 	Flush() error
 	Modify(SpecModifier) error
+	LookupEnv(string) (string, bool)
 }
 
-type fileSpec struct {
-	*oci.Spec
-	path string
-}
-
-var _ Spec = (*fileSpec)(nil)
-
-// NewSpecFromFile creates an object that encapsulates a file-backed OCI spec.
-// This can be used to read from the file, modify the spec, and write to the
-// same file.
-func NewSpecFromFile(filepath string) Spec {
-	oci := fileSpec{
-		path: filepath,
-	}
-
-	return &oci
-}
-
-// Load reads the contents of an OCI spec from file to be referenced internally.
-// The file is opened "read-only"
-func (s *fileSpec) Load() error {
-	specFile, err := os.Open(s.path)
+// NewSpec creates fileSpec based on the command line arguments passed to the
+// application using the specified logger.
+func NewSpec(logger *logrus.Logger, args []string) (Spec, error) {
+	bundleDir, err := GetBundleDir(args)
 	if err != nil {
-		return fmt.Errorf("error opening OCI specification file: %v", err)
+		return nil, fmt.Errorf("error getting bundle directory: %v", err)
 	}
-	defer specFile.Close()
+	logger.Infof("Using bundle directory: %v", bundleDir)
 
-	decoder := json.NewDecoder(specFile)
+	ociSpecPath := GetSpecFilePath(bundleDir)
+	logger.Infof("Using OCI specification file path: %v", ociSpecPath)
 
-	var spec oci.Spec
-	err = decoder.Decode(&spec)
-	if err != nil {
-		return fmt.Errorf("error reading OCI specification from file: %v", err)
-	}
+	ociSpec := NewFileSpec(ociSpecPath)
 
-	s.Spec = &spec
-	return nil
-}
-
-// Modify applies the specified SpecModifier to the stored OCI specification.
-func (s *fileSpec) Modify(f SpecModifier) error {
-	if s.Spec == nil {
-		return fmt.Errorf("no spec loaded for modification")
-	}
-	return f(s.Spec)
-}
-
-// Flush writes the stored OCI specification to the filepath specifed by the path member.
-// The file is truncated upon opening, overwriting any existing contents.
-func (s fileSpec) Flush() error {
-	specFile, err := os.Create(s.path)
-	if err != nil {
-		return fmt.Errorf("error opening OCI specification file: %v", err)
-	}
-	defer specFile.Close()
-
-	encoder := json.NewEncoder(specFile)
-
-	err = encoder.Encode(s.Spec)
-	if err != nil {
-		return fmt.Errorf("error writing OCI specification to file: %v", err)
-	}
-
-	return nil
+	return ociSpec, nil
 }

internal/oci/spec_file.go

@@ -0,0 +1,111 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type fileSpec struct {
+	memorySpec
+	path string
+}
+
+var _ Spec = (*fileSpec)(nil)
+
+// NewFileSpec creates an object that encapsulates a file-backed OCI spec.
+// This can be used to read from the file, modify the spec, and write to the
+// same file.
+func NewFileSpec(filepath string) Spec {
+	oci := fileSpec{
+		path: filepath,
+	}
+
+	return &oci
+}
+
+// Load reads the contents of an OCI spec from file to be referenced internally.
+// The file is opened "read-only"
+func (s *fileSpec) Load() error {
+	specFile, err := os.Open(s.path)
+	if err != nil {
+		return fmt.Errorf("error opening OCI specification file: %v", err)
+	}
+	defer specFile.Close()
+
+	spec, err := LoadFrom(specFile)
+	if err != nil {
+		return fmt.Errorf("error loading OCI specification from file: %v", err)
+	}
+	s.Spec = spec
+	return nil
+}
+
+// LoadFrom reads the contents of the OCI spec from the specified io.Reader.
+func LoadFrom(reader io.Reader) (*specs.Spec, error) {
+	decoder := json.NewDecoder(reader)
+
+	var spec specs.Spec
+
+	err := decoder.Decode(&spec)
+	if err != nil {
+		return nil, fmt.Errorf("error reading OCI specification: %v", err)
+	}
+
+	return &spec, nil
+}
+
+// Modify applies the specified SpecModifier to the stored OCI specification.
+func (s *fileSpec) Modify(m SpecModifier) error {
+	return s.memorySpec.Modify(m)
+}
+
+// Flush writes the stored OCI specification to the filepath specifed by the path member.
+// The file is truncated upon opening, overwriting any existing contents.
+func (s fileSpec) Flush() error {
+	if s.Spec == nil {
+		return fmt.Errorf("no OCI specification loaded")
+	}
+
+	specFile, err := os.Create(s.path)
+	if err != nil {
+		return fmt.Errorf("error opening OCI specification file: %v", err)
+	}
+	defer specFile.Close()
+
+	return flushTo(s.Spec, specFile)
+}
+
+// flushTo writes the stored OCI specification to the specified io.Writer.
+func flushTo(spec *specs.Spec, writer io.Writer) error {
+	if spec == nil {
+		return nil
+	}
+	encoder := json.NewEncoder(writer)
+
+	err := encoder.Encode(spec)
+	if err != nil {
+		return fmt.Errorf("error writing OCI specification: %v", err)
+	}
+
+	return nil
+}

internal/oci/spec_file_test.go

@@ -0,0 +1,103 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadFrom(t *testing.T) {
+	testCases := []struct {
+		contents []byte
+		isError  bool
+		spec     *specs.Spec
+	}{
+		{
+			contents: []byte{},
+			isError:  true,
+		},
+		{
+			contents: []byte("{}"),
+			isError:  false,
+			spec:     &specs.Spec{},
+		},
+	}
+
+	for i, tc := range testCases {
+		var spec *specs.Spec
+		spec, err := LoadFrom(bytes.NewReader(tc.contents))
+
+		if tc.isError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		if tc.spec == nil {
+			require.Nil(t, spec, "%d: %v", i, tc)
+		} else {
+			require.EqualValues(t, tc.spec, spec, "%d: %v", i, tc)
+		}
+	}
+}
+
+func TestFlushTo(t *testing.T) {
+	testCases := []struct {
+		isError  bool
+		spec     *specs.Spec
+		contents string
+	}{
+		{
+			spec: nil,
+		},
+		{
+			spec:     &specs.Spec{},
+			contents: "{\"ociVersion\":\"\"}\n",
+		},
+	}
+
+	for i, tc := range testCases {
+		buffer := bytes.Buffer{}
+
+		err := flushTo(tc.spec, &buffer)
+
+		if tc.isError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		require.EqualValues(t, tc.contents, buffer.String(), "%d: %v", i, tc)
+	}
+
+	// Add a simple test for a writer that returns an error when writing
+	err := flushTo(&specs.Spec{}, errorWriter{})
+	require.Error(t, err)
+}
+
+// errorWriter implements the io.Writer interface, always returning an error when
+// writing.
+type errorWriter struct{}
+
+func (e errorWriter) Write([]byte) (int, error) {
+	return 0, fmt.Errorf("error writing")
+}

internal/oci/spec_memory.go

@@ -0,0 +1,83 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package oci
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type memorySpec struct {
+	*specs.Spec
+}
+
+// NewMemorySpec creates a Spec instance from the specified OCI spec
+func NewMemorySpec(spec *specs.Spec) Spec {
+	s := memorySpec{
+		Spec: spec,
+	}
+
+	return &s
+}
+
+// Load is a no-op for the memorySpec spec
+func (s *memorySpec) Load() error {
+	return nil
+}
+
+// Flush is a no-op for the memorySpec spec
+func (s *memorySpec) Flush() error {
+	return nil
+}
+
+// Modify applies the specified SpecModifier to the stored OCI specification.
+func (s *memorySpec) Modify(m SpecModifier) error {
+	if s.Spec == nil {
+		return fmt.Errorf("cannot modify nil spec")
+	}
+	return m.Modify(s.Spec)
+}
+
+// LookupEnv mirrors os.LookupEnv for the OCI specification. It
+// retrieves the value of the environment variable named
+// by the key. If the variable is present in the environment the
+// value (which may be empty) is returned and the boolean is true.
+// Otherwise the returned value will be empty and the boolean will
+// be false.
+func (s memorySpec) LookupEnv(key string) (string, bool) {
+	if s.Spec == nil || s.Spec.Process == nil {
+		return "", false
+	}
+
+	for _, env := range s.Spec.Process.Env {
+		if !strings.HasPrefix(env, key) {
+			continue
+		}
+
+		parts := strings.SplitN(env, "=", 2)
+		if parts[0] == key {
+			if len(parts) < 2 {
+				return "", true
+			}
+			return parts[1], true
+		}
+	}
+
+	return "", false
+}

internal/oci/spec_memory_test.go

@@ -0,0 +1,178 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package oci
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLookupEnv(t *testing.T) {
+	const envName = "TEST_ENV"
+	testCases := []struct {
+		spec          *specs.Spec
+		expectedValue string
+		expectedExits bool
+	}{
+		{
+			// nil spec
+			spec:          nil,
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// nil process
+			spec:          &specs.Spec{},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// nil env
+			spec: &specs.Spec{
+				Process: &specs.Process{},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// empty env
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// different env set
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"SOMETHING_ELSE=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// same prefix
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV_BUT_NOT=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// same suffix
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"NOT_TEST_ENV=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// set blank
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV="}},
+			},
+			expectedValue: "",
+			expectedExits: true,
+		},
+		{
+			// set no-equals
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV"}},
+			},
+			expectedValue: "",
+			expectedExits: true,
+		},
+		{
+			// set value
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV=something"}},
+			},
+			expectedValue: "something",
+			expectedExits: true,
+		},
+		{
+			// set with equals
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV=something=somethingelse"}},
+			},
+			expectedValue: "something=somethingelse",
+			expectedExits: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := memorySpec{
+			Spec: tc.spec,
+		}
+
+		value, exists := spec.LookupEnv(envName)
+
+		require.Equal(t, tc.expectedValue, value, "%d: %v", i, tc)
+		require.Equal(t, tc.expectedExits, exists, "%d: %v", i, tc)
+	}
+}
+
+func TestModify(t *testing.T) {
+
+	testCases := []struct {
+		spec          *specs.Spec
+		modifierError error
+	}{
+		{
+			spec: nil,
+		},
+		{
+			spec: &specs.Spec{},
+		},
+		{
+			spec:          &specs.Spec{},
+			modifierError: fmt.Errorf("error in modifier"),
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := NewMemorySpec(tc.spec).(*memorySpec)
+
+		err := spec.Modify(modifier{tc.modifierError})
+
+		if tc.spec == nil {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else if tc.modifierError != nil {
+			require.EqualError(t, err, tc.modifierError.Error(), "%d: %v", i, tc)
+			require.EqualValues(t, &specs.Spec{}, spec.Spec, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+			require.Equal(t, "updated", spec.Spec.Version, "%d: %v", i, tc)
+		}
+	}
+}
+
+// TODO: Ideally we would generated a mock for the SpecModifer too. This causes
+// an import cycle and we define a local type as a work-around.
+type modifier struct {
+	modifierError error
+}
+
+func (m modifier) Modify(spec *specs.Spec) error {
+	if m.modifierError == nil {
+		spec.Version = "updated"
+	}
+	return m.modifierError
+}

internal/oci/spec_mock.go

@@ -1,70 +1,201 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
 
 package oci
 
 import (
-	oci "github.com/opencontainers/runtime-spec/specs-go"
+	"sync"
 )
 
-// MockSpec provides a simple mock for an OCI spec to be used in testing.
-// It also implements the SpecModifier interface.
-type MockSpec struct {
-	*oci.Spec
-	MockLoad   mockFunc
-	MockFlush  mockFunc
-	MockModify mockFunc
-}
+// Ensure, that SpecMock does implement Spec.
+// If this is not the case, regenerate this file with moq.
+var _ Spec = &SpecMock{}
 
-var _ Spec = (*MockSpec)(nil)
+// SpecMock is a mock implementation of Spec.
+//
+// 	func TestSomethingThatUsesSpec(t *testing.T) {
+//
+// 		// make and configure a mocked Spec
+// 		mockedSpec := &SpecMock{
+// 			FlushFunc: func() error {
+// 				panic("mock out the Flush method")
+// 			},
+// 			LoadFunc: func() error {
+// 				panic("mock out the Load method")
+// 			},
+// 			LookupEnvFunc: func(s string) (string, bool) {
+// 				panic("mock out the LookupEnv method")
+// 			},
+// 			ModifyFunc: func(specModifier SpecModifier) error {
+// 				panic("mock out the Modify method")
+// 			},
+// 		}
+//
+// 		// use mockedSpec in code that requires Spec
+// 		// and then make assertions.
+//
+// 	}
+type SpecMock struct {
+	// FlushFunc mocks the Flush method.
+	FlushFunc func() error
 
-// NewMockSpec constructs a MockSpec to be used in testing as a Spec
-func NewMockSpec(spec *oci.Spec, flushResult error, modifyResult error) *MockSpec {
-	s := MockSpec{
-		Spec:       spec,
-		MockFlush:  mockFunc{result: flushResult},
-		MockModify: mockFunc{result: modifyResult},
+	// LoadFunc mocks the Load method.
+	LoadFunc func() error
+
+	// LookupEnvFunc mocks the LookupEnv method.
+	LookupEnvFunc func(s string) (string, bool)
+
+	// ModifyFunc mocks the Modify method.
+	ModifyFunc func(specModifier SpecModifier) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Flush holds details about calls to the Flush method.
+		Flush []struct {
+		}
+		// Load holds details about calls to the Load method.
+		Load []struct {
+		}
+		// LookupEnv holds details about calls to the LookupEnv method.
+		LookupEnv []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Modify holds details about calls to the Modify method.
+		Modify []struct {
+			// SpecModifier is the specModifier argument value.
+			SpecModifier SpecModifier
+		}
 	}
-
-	return &s
+	lockFlush     sync.RWMutex
+	lockLoad      sync.RWMutex
+	lockLookupEnv sync.RWMutex
+	lockModify    sync.RWMutex
 }
 
-// Load invokes the mocked Load function to return the predefined error / result
-func (s *MockSpec) Load() error {
-	return s.MockLoad.call()
+// Flush calls FlushFunc.
+func (mock *SpecMock) Flush() error {
+	callInfo := struct {
+	}{}
+	mock.lockFlush.Lock()
+	mock.calls.Flush = append(mock.calls.Flush, callInfo)
+	mock.lockFlush.Unlock()
+	if mock.FlushFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.FlushFunc()
 }
 
-// Flush invokes the mocked Load function to return the predefined error / result
-func (s *MockSpec) Flush() error {
-	return s.MockFlush.call()
+// FlushCalls gets all the calls that were made to Flush.
+// Check the length with:
+//     len(mockedSpec.FlushCalls())
+func (mock *SpecMock) FlushCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockFlush.RLock()
+	calls = mock.calls.Flush
+	mock.lockFlush.RUnlock()
+	return calls
 }
 
-// Modify applies the specified SpecModifier to the spec and invokes the
-// mocked modify function to return the predefined error / result.
-func (s *MockSpec) Modify(f SpecModifier) error {
-	f(s.Spec)
-	return s.MockModify.call()
+// Load calls LoadFunc.
+func (mock *SpecMock) Load() error {
+	callInfo := struct {
+	}{}
+	mock.lockLoad.Lock()
+	mock.calls.Load = append(mock.calls.Load, callInfo)
+	mock.lockLoad.Unlock()
+	if mock.LoadFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.LoadFunc()
 }
 
-type mockFunc struct {
-	Callcount int
-	result    error
+// LoadCalls gets all the calls that were made to Load.
+// Check the length with:
+//     len(mockedSpec.LoadCalls())
+func (mock *SpecMock) LoadCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockLoad.RLock()
+	calls = mock.calls.Load
+	mock.lockLoad.RUnlock()
+	return calls
 }
 
-func (m *mockFunc) call() error {
-	m.Callcount++
-	return m.result
+// LookupEnv calls LookupEnvFunc.
+func (mock *SpecMock) LookupEnv(s string) (string, bool) {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockLookupEnv.Lock()
+	mock.calls.LookupEnv = append(mock.calls.LookupEnv, callInfo)
+	mock.lockLookupEnv.Unlock()
+	if mock.LookupEnvFunc == nil {
+		var (
+			sOut string
+			bOut bool
+		)
+		return sOut, bOut
+	}
+	return mock.LookupEnvFunc(s)
+}
+
+// LookupEnvCalls gets all the calls that were made to LookupEnv.
+// Check the length with:
+//     len(mockedSpec.LookupEnvCalls())
+func (mock *SpecMock) LookupEnvCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockLookupEnv.RLock()
+	calls = mock.calls.LookupEnv
+	mock.lockLookupEnv.RUnlock()
+	return calls
+}
+
+// Modify calls ModifyFunc.
+func (mock *SpecMock) Modify(specModifier SpecModifier) error {
+	callInfo := struct {
+		SpecModifier SpecModifier
+	}{
+		SpecModifier: specModifier,
+	}
+	mock.lockModify.Lock()
+	mock.calls.Modify = append(mock.calls.Modify, callInfo)
+	mock.lockModify.Unlock()
+	if mock.ModifyFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ModifyFunc(specModifier)
+}
+
+// ModifyCalls gets all the calls that were made to Modify.
+// Check the length with:
+//     len(mockedSpec.ModifyCalls())
+func (mock *SpecMock) ModifyCalls() []struct {
+	SpecModifier SpecModifier
+} {
+	var calls []struct {
+		SpecModifier SpecModifier
+	}
+	mock.lockModify.RLock()
+	calls = mock.calls.Modify
+	mock.lockModify.RUnlock()
+	return calls
 }

internal/oci/spec_test.go

@@ -0,0 +1,39 @@
+package oci
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMaintainSpec(t *testing.T) {
+	moduleRoot, err := test.GetModuleRoot()
+	require.NoError(t, err)
+
+	files := []string{
+		"config.clone3.json",
+	}
+
+	for _, f := range files {
+		inputSpecPath := filepath.Join(moduleRoot, "test/input", f)
+
+		spec := NewFileSpec(inputSpecPath).(*fileSpec)
+
+		spec.Load()
+
+		outputSpecPath := filepath.Join(moduleRoot, "test/output", f)
+		spec.path = outputSpecPath
+		spec.Flush()
+
+		inputContents, err := os.ReadFile(inputSpecPath)
+		require.NoError(t, err)
+
+		outputContents, err := os.ReadFile(outputSpecPath)
+		require.NoError(t, err)
+
+		require.JSONEq(t, string(inputContents), string(outputContents))
+	}
+}

internal/oci/state.go

@@ -0,0 +1,73 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package oci
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// State stores an OCI container state. This includes the spec path and the environment
+type State specs.State
+
+// LoadContainerState loads the container state from the specified filename. If the filename is empty or '-' the state is loaded from STDIN
+func LoadContainerState(filename string) (*State, error) {
+	if filename == "" || filename == "-" {
+		return ReadContainerState(os.Stdin)
+	}
+
+	inputFile, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+	defer inputFile.Close()
+
+	return ReadContainerState(inputFile)
+}
+
+// ReadContainerState reads the container state from the specified reader
+func ReadContainerState(reader io.Reader) (*State, error) {
+	var s State
+
+	d := json.NewDecoder(reader)
+	if err := d.Decode(&s); err != nil {
+		return nil, fmt.Errorf("failed to decode container state: %v", err)
+	}
+
+	return &s, nil
+}
+
+// LoadSpec loads the OCI spec associated with the container state
+func (s State) LoadSpec() (*specs.Spec, error) {
+	specFilePath := GetSpecFilePath(s.Bundle)
+	specFile, err := os.Open(specFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open OCI spec file: %v", err)
+	}
+	defer specFile.Close()
+
+	spec, err := LoadFrom(specFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OCI spec: %v", err)
+	}
+
+	return spec, nil
+}